From 8df09b71607bd301f60e6b9cc5d53ba7ebe0a587 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 19:13:49 +0200 Subject: [PATCH 001/114] Add toxic to config class and implement getter for it --- .../config/corpusConfig/CorpusAnnotationConfig.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java index ec588bd3..dc5ed7b0 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java @@ -24,6 +24,7 @@ public class CorpusAnnotationConfig { private boolean scope; private boolean xscope; private boolean unifiedTopic; + private boolean toxic; public boolean isGeoNames() { return geoNames; @@ -203,4 +204,7 @@ public boolean isUnifiedTopic() { return unifiedTopic; } + public boolean isToxic(){ + return toxic; + } } From 01baec2f83c1943a6679f6f6b39ec82cd4a70175 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 19:27:17 +0200 Subject: [PATCH 002/114] create toxic class --- .../texttechnologylab/models/toxic/Toxic.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java new file mode 100644 index 00000000..fe30ef63 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -0,0 +1,44 @@ +package org.texttechnologylab.models.toxic; + +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; + +import javax.persistence.*; + +@Entity +@Table(name = "toxic") +public class Toxic extends UIMAAnnotation implements WikiModel { + + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Toxic() { + super(-1, -1); + } + + public Toxic(int begin, int end) { + super(begin, end); + } + + public Toxic(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "UT" + "-" + this.getId(); + } + +} From 9e9c0a2baa028d0f6f0a5cca823c5b1d8aef68ac Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 19:38:27 +0200 Subject: [PATCH 003/114] add toxic and nonToxic properties as well as getters and setters --- .../texttechnologylab/models/toxic/Toxic.java | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index fe30ef63..868f1041 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -9,7 +9,10 @@ @Entity @Table(name = "toxic") public class Toxic extends UIMAAnnotation implements WikiModel { - + @Column(name = "toxic", nullable = false) + private double toxic; + @Column(name = "non_toxic", nullable = false) + private double nonToxic; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) @@ -28,6 +31,22 @@ public Toxic(int begin, int end, String coveredText) { setCoveredText(coveredText); } + public double getToxic() { + return toxic; + } + + public void setToxic(double newToxic) { + this.toxic = newToxic; + } + + public double getNonToxic() { + return nonToxic; + } + + public void setNonToxic(double nonToxic) { + this.nonToxic = nonToxic; + } + public Document getDocument() { return document; } @@ -40,5 +59,4 @@ public void setDocument(Document document) { public String getWikiId() { return "UT" + "-" + this.getId(); } - } From 1ebdb2a1afbd72dbce94f9c04be7abd9d9c64d40 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 19:42:04 +0200 Subject: [PATCH 004/114] typesystem toxic --- .../main/java/org/texttechnologylab/models/toxic/Toxic.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index 868f1041..eca25722 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -1,5 +1,6 @@ package org.texttechnologylab.models.toxic; +import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; @@ -8,6 +9,7 @@ @Entity @Table(name = "toxic") +@Typesystem(types = {org.texttechnologylab.annotation.Toxic.class}) public class Toxic extends UIMAAnnotation implements WikiModel { @Column(name = "toxic", nullable = false) private double toxic; @@ -34,7 +36,6 @@ public Toxic(int begin, int end, String coveredText) { public double getToxic() { return toxic; } - public void setToxic(double newToxic) { this.toxic = newToxic; } From 22e0028901810e1be0d8e4d41b7da39742ebb45b Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 20:12:14 +0200 Subject: [PATCH 005/114] Add toxic annotation handling method in Importer --- .../src/main/java/org/texttechnologylab/Importer.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 8a20f607..baad46f1 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -566,6 +566,11 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { () -> setUnifiedTopic(document, jCas), (ex) -> logImportWarn("This file should have contained UnifiedTopic annotations, but selecting them caused an error.", ex, filePath)); + if (corpusConfig.getAnnotations().isToxic()) + ExceptionUtils.tryCatchLog( + () -> setToxic(document, jCas), + (ex) -> logImportWarn("This file should have contained Toxic annotations, but selecting them caused an error.", ex, filePath)); + // Keep this at the end of the annotation setting, as they might require previous annotations. Order matter here! if (corpusConfig.getAnnotations().isLogicalLinks()) ExceptionUtils.tryCatchLog( @@ -1410,6 +1415,12 @@ private void setUnifiedTopic(Document document, JCas jCas) { document.setUnifiedTopics(unifiedTopics); } + /** + * Selects and sets the toxicities to a document. + */ + private void setToxic(Document document, JCas jCas) { + } + /** * Set the cleaned full text. That is the sum of all tokens except of all anomalies * Update: OBSOLETE for now, as this takes forever and makes the OCR worse. From ae5bd3880c4209ead4858f635cb2e413dc5278e8 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 20:18:33 +0200 Subject: [PATCH 006/114] Implement toxic selection and mapping in setToxic method --- .../src/main/java/org/texttechnologylab/Importer.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index baad46f1..4be826cc 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -49,6 +49,7 @@ import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.services.*; import org.texttechnologylab.utils.*; import org.texttechnologylab.models.negation.CompleteNegation; @@ -1419,6 +1420,16 @@ private void setUnifiedTopic(Document document, JCas jCas) { * Selects and sets the toxicities to a document. */ private void setToxic(Document document, JCas jCas) { + List toxics = new ArrayList<>(); + + JCasUtil.select(jCas, org.texttechnologylab.annotation.Toxic.class).forEach(t -> { + Toxic toxic = new Toxic(t.getBegin(), t.getEnd()); + toxic.setDocument(document); + toxic.setCoveredText(t.getCoveredText()); + toxic.setToxic(t.getToxic()); + toxic.setNonToxic(t.getNonToxic()); + toxics.add(toxic); + }); } /** From b2ec3381325b61d9397643a369d194295c2290e7 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 20:20:46 +0200 Subject: [PATCH 007/114] Add toxics property to Document class with appropriate annotations --- .../java/org/texttechnologylab/models/corpus/Document.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9d898b4..6d828e39 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -26,6 +26,7 @@ import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.utils.FreemarkerUtils; import org.texttechnologylab.utils.StringUtils; @@ -218,6 +219,12 @@ public long getPrimaryDbIdentifier() { @Fetch(value = FetchMode.SUBSELECT) private List unifiedTopics; + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List toxics; + public Document() { metadataTitleInfo = new MetadataTitleInfo(); } From 5540c7c5ac4ff7093b1f6f3b03dbab290c475a13 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 20:21:07 +0200 Subject: [PATCH 008/114] Set toxics in Document after processing import --- .../src/main/java/org/texttechnologylab/Importer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 4be826cc..f6eb8ce5 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1430,6 +1430,8 @@ private void setToxic(Document document, JCas jCas) { toxic.setNonToxic(t.getNonToxic()); toxics.add(toxic); }); + + document.setToxics(toxics); } /** From d4e0cfdca27fa81e953cc0dd1ab422e61e023402 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Wed, 9 Jul 2025 20:37:43 +0200 Subject: [PATCH 009/114] Add Toxic class to Hibernate metadata sources --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..905962ea 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -21,6 +21,7 @@ import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import java.util.HashMap; @@ -80,6 +81,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // toxic + metadataSources.addAnnotatedClass(Toxic.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From cc2dda6b46c7fe0838f6448a1a11ecb929398a9e Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 20:57:14 +0200 Subject: [PATCH 010/114] Add OffensiveSpeech entity for handling offensive speech annotations --- .../offensiveSpeech/OffensiveSpeech.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java new file mode 100644 index 00000000..2f7176b1 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -0,0 +1,45 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; + +import javax.persistence.*; + +@Entity +@Table(name = "offensive-speech") +public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { + + + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public OffensiveSpeech() { + super(-1, -1); + } + + public OffensiveSpeech(int begin, int end) { + super(begin, end); + } + + public OffensiveSpeech(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "UT" + "-" + this.getId(); + } + +} From f60d1a24c5a3aa757c4375dbc7a7636593b4dac6 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 20:57:32 +0200 Subject: [PATCH 011/114] Add Typesystem annotation to OffensiveSpeech entity --- .../models/offensiveSpeech/OffensiveSpeech.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 2f7176b1..f41f9ee2 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -1,5 +1,6 @@ package org.texttechnologylab.models.offensiveSpeech; +import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; @@ -8,6 +9,7 @@ @Entity @Table(name = "offensive-speech") +@Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { From 8ad779f9321c4539ebea961c2aa0504fdd03fd14 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:06:32 +0200 Subject: [PATCH 012/114] Add offensive and non-offensive flags to OffensiveSpeech entity --- .../models/offensiveSpeech/OffensiveSpeech.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index f41f9ee2..50bb8be5 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -1,18 +1,31 @@ package org.texttechnologylab.models.offensiveSpeech; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; import javax.persistence.*; +import java.util.List; @Entity @Table(name = "offensive-speech") @Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { + @Getter + @Setter + @Column(name = "offensive", nullable = false) + private boolean offensive; + @Getter + @Setter + @Column(name = "non_offensive", nullable = false) + private boolean nonOffensive; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) From 6be01b325a8dffba702dc907b8182c321f15f1da Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:07:43 +0200 Subject: [PATCH 013/114] add to HibernateConf --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..e331f1aa 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -17,6 +17,7 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -80,6 +81,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // offensive speech + metadataSources.addAnnotatedClass(OffensiveSpeech.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From ffb920b12a54ffc7dedabfdda3925950e53f78c3 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:09:15 +0200 Subject: [PATCH 014/114] add to CorpusAnnotationConfig --- .../config/corpusConfig/CorpusAnnotationConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java index ec588bd3..4a3300bc 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java @@ -24,6 +24,7 @@ public class CorpusAnnotationConfig { private boolean scope; private boolean xscope; private boolean unifiedTopic; + private boolean offensiveSpeech; public boolean isGeoNames() { return geoNames; @@ -203,4 +204,8 @@ public boolean isUnifiedTopic() { return unifiedTopic; } + public boolean isOffensiveSpeech() { + return offensiveSpeech; + } + } From bd4744f375c0c0e8b2c4b5758458fb82d25f112d Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:11:11 +0200 Subject: [PATCH 015/114] Add support for importing offensive speech annotations --- .../src/main/java/org/texttechnologylab/Importer.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 8a20f607..571ad4ea 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -566,6 +566,11 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { () -> setUnifiedTopic(document, jCas), (ex) -> logImportWarn("This file should have contained UnifiedTopic annotations, but selecting them caused an error.", ex, filePath)); + if (corpusConfig.getAnnotations().isOffensiveSpeech()) + ExceptionUtils.tryCatchLog( + () -> setOffensiveSpeech(document, jCas), + (ex) -> logImportWarn("This file should have contained OffensiveSpeech annotations, but selecting them caused an error.", ex, filePath)); + // Keep this at the end of the annotation setting, as they might require previous annotations. Order matter here! if (corpusConfig.getAnnotations().isLogicalLinks()) ExceptionUtils.tryCatchLog( @@ -1410,6 +1415,10 @@ private void setUnifiedTopic(Document document, JCas jCas) { document.setUnifiedTopics(unifiedTopics); } + private void setOffensiveSpeech(Document document, JCas jCas) { + + } + /** * Set the cleaned full text. That is the sum of all tokens except of all anomalies * Update: OBSOLETE for now, as this takes forever and makes the OCR worse. From 905e75f44c77b1595524584e453a418e2a8706c7 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:20:54 +0200 Subject: [PATCH 016/114] fix data type --- .../models/offensiveSpeech/OffensiveSpeech.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 50bb8be5..f3f94f3e 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -20,12 +20,12 @@ public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { @Getter @Setter @Column(name = "offensive", nullable = false) - private boolean offensive; + private double offensive; @Getter @Setter @Column(name = "non_offensive", nullable = false) - private boolean nonOffensive; + private double nonOffensive; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) From 2e3df8406ba08ce73c983959fbae76b3392b1b1d Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:21:27 +0200 Subject: [PATCH 017/114] Add method to set offensive speech annotations with score validation --- .../java/org/texttechnologylab/Importer.java | 58 +++++++++++++++---- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 571ad4ea..6c997be8 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -19,6 +19,7 @@ import org.apache.uima.util.CasIOUtils; import org.apache.uima.util.CasLoadMode; import org.springframework.context.ApplicationContext; +import org.texttechnologylab.annotation.AnnotationComment; import org.texttechnologylab.annotation.DocumentAnnotation; import org.texttechnologylab.annotation.geonames.GeoNamesEntity; import org.texttechnologylab.annotation.link.*; @@ -43,6 +44,7 @@ import org.texttechnologylab.models.imp.ImportStatus; import org.texttechnologylab.models.imp.LogStatus; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; import org.texttechnologylab.models.topic.TopicValueBase; @@ -140,13 +142,13 @@ public int getXMICountInPath() { public void start(int numThreads) throws DatabaseOperationException { logger.info( "\n _ _ _____ _____ _____ _ \n" + - "| | | / __ \\| ___| |_ _| | | \n" + - "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + - "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + - "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + - " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + - " | | \n" + - " |_|" + "| | | / __ \\| ___| |_ _| | | \n" + + "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + + "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + + "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + + " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + + " | | \n" + + " |_|" ); logger.info("===========> Global Import Id: " + importId); logger.info("===========> Importer Number: " + importerNumber); @@ -208,7 +210,7 @@ public void storeCorpusFromFolderAsync(String folderName, int numThreads) throws final var corpusConfig1 = corpusConfig; // This sucks so hard - why doesn't java just do this itself if needed? var existingCorpus = ExceptionUtils.tryCatchLog(() -> db.getCorpusByName(corpusConfig1.getName()), (ex) -> logger.error("Error getting an existing corpus by name. The corpus config should probably be changed " + - "to not add to existing corpus then.", ex)); + "to not add to existing corpus then.", ex)); if (existingCorpus != null) { // If we have the corpus, use that. Else store the new corpus. corpus = existingCorpus; @@ -468,7 +470,7 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { var exists = db.documentExists(corpus.getId(), document.getDocumentId()); if (exists) { logger.info("Document with id " + document.getDocumentId() - + " already exists in the corpus " + corpus.getId() + "."); + + " already exists in the corpus " + corpus.getId() + "."); logger.info("Checking if that document was also post-processed yet..."); var existingDoc = db.getDocumentByCorpusAndDocumentId(corpus.getId(), document.getDocumentId()); if (!existingDoc.isPostProcessed()) { @@ -824,8 +826,8 @@ private void setMetadataTitleInfo(Document document, JCas jCas, CorpusConfig cor if (documentAnnotation != null) { try { metadataTitleInfo.setPublished(documentAnnotation.getDateDay() + "." - + documentAnnotation.getDateMonth() + "." - + documentAnnotation.getDateYear()); + + documentAnnotation.getDateMonth() + "." + + documentAnnotation.getDateYear()); metadataTitleInfo.setAuthor(documentAnnotation.getAuthor()); } catch (Exception ex) { logger.warn("Tried extracting DocumentAnnotation type, it caused an error. Import will be continued as usual."); @@ -1416,7 +1418,41 @@ private void setUnifiedTopic(Document document, JCas jCas) { } private void setOffensiveSpeech(Document document, JCas jCas) { + List offensiveSpeeches = new ArrayList<>(); + JCasUtil.select(jCas, org.texttechnologylab.annotation.OffensiveSpeech.class).forEach(os -> { + OffensiveSpeech offensiveSpeech = new OffensiveSpeech(os.getBegin(), os.getEnd()); + offensiveSpeech.setDocument(document); + offensiveSpeech.setCoveredText(os.getCoveredText()); + + FSArray offensives = os.getOffensives(); + if (offensives == null) { + logger.warn("OffensiveSpeech annotation without offensives found. Skipping this annotation."); + return; + } + if (offensives.size() != 2) { + logger.warn("OffensiveSpeech annotation with " + offensives.size() + " offensives found. Expected 2. Skipping this annotation."); + return; + } + final String offensiveKey = "Offensive"; + final String nonOffensiveKey = "Not Offensive"; + OptionalDouble offensiveScore = offensives.stream() + .filter(oc -> oc.getKey().equals(offensiveKey)) + .map(AnnotationComment::getValue) + .mapToDouble(Double::parseDouble) + .findFirst(); + OptionalDouble nonOffensiveScore = offensives.stream() + .filter(oc -> oc.getKey().equals(nonOffensiveKey)) + .map(AnnotationComment::getValue) + .mapToDouble(Double::parseDouble).findFirst(); + if (offensiveScore.isEmpty() || nonOffensiveScore.isEmpty()) { + logger.warn("OffensiveSpeech annotation without offensive or non-offensive score found. Skipping this annotation."); + return; + } + offensiveSpeech.setOffensive(offensiveScore.getAsDouble()); + offensiveSpeech.setNonOffensive(nonOffensiveScore.getAsDouble()); + offensiveSpeeches.add(offensiveSpeech); + }); } /** From f81da7a7ed0fc336a6c75105c11306f3183e4cf0 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:22:41 +0200 Subject: [PATCH 018/114] add offensive speech to document entity --- .../java/org/texttechnologylab/models/corpus/Document.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9d898b4..d6225e33 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -21,6 +21,7 @@ import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; import org.texttechnologylab.models.corpus.links.Link; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.search.AnnotationSearchResult; import org.texttechnologylab.models.search.PageSnippet; import org.texttechnologylab.models.topic.TopicValueBase; @@ -218,6 +219,12 @@ public long getPrimaryDbIdentifier() { @Fetch(value = FetchMode.SUBSELECT) private List unifiedTopics; + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List offensiveSpeeches; + public Document() { metadataTitleInfo = new MetadataTitleInfo(); } From 3cad3e3201bc51ffbffc0c688035bbecd13af7f7 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:23:02 +0200 Subject: [PATCH 019/114] set offensive speeches in document after import --- .../src/main/java/org/texttechnologylab/Importer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 6c997be8..2ce83cc9 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1453,6 +1453,8 @@ private void setOffensiveSpeech(Document document, JCas jCas) { offensiveSpeech.setNonOffensive(nonOffensiveScore.getAsDouble()); offensiveSpeeches.add(offensiveSpeech); }); + + document.setOffensiveSpeeches(offensiveSpeeches); } /** From 50eea9c2779e8502789479acc6948f7fc47a1ac4 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Wed, 9 Jul 2025 21:24:08 +0200 Subject: [PATCH 020/114] fix table name --- .../models/offensiveSpeech/OffensiveSpeech.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index f3f94f3e..d3f3be4b 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -13,7 +13,7 @@ import java.util.List; @Entity -@Table(name = "offensive-speech") +@Table(name = "offensive_speech") @Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { From fe4099076736164bb253852aaf927940bf9ba42d Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:45:42 +0200 Subject: [PATCH 021/114] Add Emotion entity for emotion annotations in documents --- .../models/emotion/Emotion.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java new file mode 100644 index 00000000..1c64d887 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -0,0 +1,43 @@ +package org.texttechnologylab.models.emotion; + +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; + +import javax.persistence.*; + +@Entity +@Table(name = "emotion") +public class Emotion extends UIMAAnnotation implements WikiModel { + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Emotion() { + super(-1, -1); + } + + public Emotion(int begin, int end) { + super(begin, end); + } + + public Emotion(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "UT" + "-" + this.getId(); + } + +} From 6eee6f418c3aa34c1c628085f3757954e33f1074 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:46:05 +0200 Subject: [PATCH 022/114] Add Typesystem annotation to Emotion entity --- .../main/java/org/texttechnologylab/models/emotion/Emotion.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index 1c64d887..598493eb 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -1,5 +1,6 @@ package org.texttechnologylab.models.emotion; +import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; @@ -8,6 +9,7 @@ @Entity @Table(name = "emotion") +@Typesystem(types = {org.texttechnologylab.annotation.Emotion.class}) public class Emotion extends UIMAAnnotation implements WikiModel { @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) From b80fe97fec3082df57fdb9c26270939433f29830 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:53:02 +0200 Subject: [PATCH 023/114] Add EmotionType entity for managing emotion types --- .../models/emotion/EmotionType.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java new file mode 100644 index 00000000..038fa5e6 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java @@ -0,0 +1,26 @@ +package org.texttechnologylab.models.emotion; + +import lombok.Getter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "emotion_type") +public class EmotionType extends ModelBase { + + @Getter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public EmotionType() { + // Default constructor for JPA + } + + public EmotionType(String name) { + this.name = name; + } + +} From 66556784c79efc2e8c8a5e4d0f5ee2d51dfb3a67 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:57:22 +0200 Subject: [PATCH 024/114] Add EmotionValue entity for storing emotion values --- .../models/emotion/EmotionValue.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java new file mode 100644 index 00000000..549257e3 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java @@ -0,0 +1,38 @@ +package org.texttechnologylab.models.emotion; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "emotion_value") +public class EmotionValue extends ModelBase { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "emotion_type_id", nullable = false) + private EmotionType emotionType; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "emotion_id", nullable = false) + private Emotion emotion; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public EmotionValue() { + } + + public EmotionValue(EmotionType emotionType, Emotion emotion, double value) { + this.emotionType = emotionType; + this.emotion = emotion; + this.value = value; + } +} From 1159672883f82566d487c01bac22d6af2e91afdd Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:59:29 +0200 Subject: [PATCH 025/114] Add emotionValues field to Emotion entity for managing associated EmotionValue instances --- .../org/texttechnologylab/models/emotion/Emotion.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index 598493eb..c6c29b82 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -1,17 +1,28 @@ package org.texttechnologylab.models.emotion; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; import javax.persistence.*; +import java.util.List; @Entity @Table(name = "emotion") @Typesystem(types = {org.texttechnologylab.annotation.Emotion.class}) public class Emotion extends UIMAAnnotation implements WikiModel { + @Getter + @Setter + @OneToMany(mappedBy = "emotion", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + @Fetch(value = FetchMode.SUBSELECT) + private List emotionValues; + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) private Document document; From 1d4b317d02fd5e674ac50746005835b6ae6420fc Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 14:59:37 +0200 Subject: [PATCH 026/114] Add setter for name field in EmotionType entity --- .../java/org/texttechnologylab/models/emotion/EmotionType.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java index 038fa5e6..f3cc3061 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java @@ -1,6 +1,7 @@ package org.texttechnologylab.models.emotion; import lombok.Getter; +import lombok.Setter; import org.texttechnologylab.models.ModelBase; import javax.persistence.Column; @@ -12,6 +13,7 @@ public class EmotionType extends ModelBase { @Getter + @Setter @Column(name = "name", nullable = false, unique = true) private String name; From 865e2ff209573c0a4a9d655677e05d13494587b6 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:00:31 +0200 Subject: [PATCH 027/114] Add Emotion, EmotionValue, and EmotionType classes to Hibernate configuration --- .../java/org/texttechnologylab/config/HibernateConf.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..fbb7548b 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -13,6 +13,9 @@ import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; +import org.texttechnologylab.models.emotion.EmotionValue; import org.texttechnologylab.models.gbif.GbifOccurrence; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; @@ -80,6 +83,10 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // emotion + metadataSources.addAnnotatedClass(Emotion.class); + metadataSources.addAnnotatedClass(EmotionValue.class); + metadataSources.addAnnotatedClass(EmotionType.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From 316dfea13328d97eedc9cac817c33289ea8aa146 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:01:06 +0200 Subject: [PATCH 028/114] Add emotions field to Document entity for managing associated Emotion instances --- .../org/texttechnologylab/models/corpus/Document.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9d898b4..9b42c6cd 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -19,10 +19,8 @@ import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; -import org.texttechnologylab.models.corpus.links.Link; +import org.texttechnologylab.models.emotion.Emotion; import org.texttechnologylab.models.negation.*; -import org.texttechnologylab.models.search.AnnotationSearchResult; -import org.texttechnologylab.models.search.PageSnippet; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.UnifiedTopic; @@ -218,6 +216,12 @@ public long getPrimaryDbIdentifier() { @Fetch(value = FetchMode.SUBSELECT) private List unifiedTopics; + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List emotions; + public Document() { metadataTitleInfo = new MetadataTitleInfo(); } From 9071db656399ae9c9242ca7ffc3a6b2c16e60ed0 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:04:07 +0200 Subject: [PATCH 029/114] Add methods for retrieving and creating EmotionType instances in PostgresqlDataInterface_Impl --- .../PostgresqlDataInterface_Impl.java | 156 +++++++++++------- 1 file changed, 96 insertions(+), 60 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index dc9b4541..79d522ef 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -25,6 +25,8 @@ import org.texttechnologylab.models.dto.UCEMetadataFilterDto; import org.texttechnologylab.models.dto.map.MapClusterDto; import org.texttechnologylab.models.dto.map.PointDto; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; import org.texttechnologylab.models.gbif.GbifOccurrence; import org.texttechnologylab.models.globe.GlobeTaxon; import org.texttechnologylab.models.imp.ImportLog; @@ -423,12 +425,13 @@ public List getGlobeDataForDocument(long documentId) throws Database () -> getAllLinksOfLinkable(taxon.getId(), taxon.getClass(), List.of(AnnotationLink.class)) .stream() .filter(l -> l.getLinkId().equals("context") && l.getToAnnotationType().equals(GeoName.class.getName())).toList(), - (ex) -> { }); + (ex) -> { + }); // Foreach taxa, fetch a possible geoname link. if (links != null) for (var link : links) { var geoname = doc.getGeoNames().stream().filter(g -> g.getId() == link.getToId()).findFirst(); - if(geoname.isEmpty()) continue; + if (geoname.isEmpty()) continue; var globeTaxon = new GlobeTaxon(); globeTaxon.setLongitude(geoname.get().getLongitude()); globeTaxon.setLatitude(geoname.get().getLatitude()); @@ -970,7 +973,7 @@ public DocumentSearchResult defaultSearchForDocuments(int skip, return executeOperationSafely((session) -> session.doReturningWork((connection) -> { DocumentSearchResult search = null; try (var storedProcedure = connection.prepareCall("{call uce_search_layer_" + layer.name().toLowerCase() + - "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { + "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { storedProcedure.setInt(1, (int) corpusId); storedProcedure.setArray(2, connection.createArrayOf("text", searchTokens.stream().map(this::escapeSql).toArray())); storedProcedure.setString(3, ogSearchQuery); @@ -1190,9 +1193,9 @@ public List getDistinctTimesByCondition(String condition, long corpusId, return executeOperationSafely((session) -> { // Construct HQL dynamically (THIS IS UNSAFE BECAUSE OF THE CONDITION INSERTION) String hql = "SELECT DISTINCT t.coveredText " + - "FROM Time t " + - "JOIN Document d ON t.documentId = d.id " + - "WHERE " + condition + " AND d.corpusId = :corpusId"; + "FROM Time t " + + "JOIN Document d ON t.documentId = d.id " + + "WHERE " + condition + " AND d.corpusId = :corpusId"; var query = session.createQuery(hql, String.class); query.setParameter("corpusId", corpusId); @@ -1349,13 +1352,46 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera return executeOperationSafely((session) -> { var topic = session.get(UnifiedTopic.class, id); Hibernate.initialize(topic.getTopics()); - for(var t:topic.getTopics()){ + for (var t : topic.getTopics()) { Hibernate.initialize(t.getWords()); } return topic; }); } + public EmotionType getEmotionTypeById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> session.get(EmotionType.class, id)); + } + + public EmotionType getEmotionTypeByName(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(EmotionType.class); + var root = criteriaQuery.from(EmotionType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + return session.createQuery(criteriaQuery).uniqueResult(); + }); + } + + public EmotionType getOrCreateEmotionTypeByName(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(EmotionType.class); + var root = criteriaQuery.from(EmotionType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + + EmotionType emotionType; + try { + emotionType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + // If not found, create a new one + emotionType = new EmotionType(name); + session.save(emotionType); + } + return emotionType; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); @@ -1597,9 +1633,9 @@ public List getTopTopicsByDocument(long documentId, int limit) throws return executeOperationSafely((session) -> { // Use native SQL to query the document_topics_raw table String sql = "SELECT topiclabel, thetadt FROM documenttopicsraw " + - "WHERE document_id = :documentId " + - "ORDER BY thetadt DESC " + - "LIMIT :limit"; + "WHERE document_id = :documentId " + + "ORDER BY thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId) @@ -1613,9 +1649,9 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws return executeOperationSafely((session) -> { // Direct query using sentence_id String sql = "SELECT topiclabel, thetast FROM sentencetopics " + - "WHERE sentence_id = :sentenceId " + - "ORDER BY thetast DESC " + - "LIMIT :limit"; + "WHERE sentence_id = :sentenceId " + + "ORDER BY thetast DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("sentenceId", sentenceId) @@ -1628,12 +1664,12 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws public List getTopDocumentsByTopicLabel(String topicValue, long corpusId, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT d.id, d.documentid, dtr.thetadt " + - "FROM document d " + - "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + - "WHERE dtr.topiclabel = :topicValue " + - "AND d.corpusid = :corpusId " + - "ORDER BY dtr.thetadt DESC " + - "LIMIT :limit"; + "FROM document d " + + "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + + "WHERE dtr.topiclabel = :topicValue " + + "AND d.corpusid = :corpusId " + + "ORDER BY dtr.thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("topicValue", topicValue) @@ -1647,10 +1683,10 @@ public List getTopDocumentsByTopicLabel(String topicValue, long corpus public List getTopicWordsByTopicLabel(String topicValue, long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, probability " + - "FROM corpustopicwords " + - "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + - "ORDER BY probability DESC " + - "LIMIT 20"; + "FROM corpustopicwords " + + "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + + "ORDER BY probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("topicValue", topicValue); @@ -1687,12 +1723,12 @@ public List getSimilarTopicsbyTopicLabel(String topicValue, long corpu public List getNormalizedTopicWordsForCorpus(long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, " + - "AVG(probability) AS avg_probability, " + - "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + - "FROM corpustopicwords " + - "WHERE corpus_id = :corpusId " + - "GROUP BY word " + - "ORDER BY normalized_probability DESC"; + "AVG(probability) AS avg_probability, " + + "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + + "FROM corpustopicwords " + + "WHERE corpus_id = :corpusId " + + "GROUP BY word " + + "ORDER BY normalized_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("corpusId", corpusId); @@ -1739,11 +1775,11 @@ FROM get_normalized_topic_scores(:corpusId) public List getDocumentWordDistribution(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY word " + - "ORDER BY avg_probability DESC " + - "LIMIT 20"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY word " + + "ORDER BY avg_probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); @@ -1774,25 +1810,25 @@ public List getDocumentWordDistribution(long documentId) throws Datab public List getSimilarDocumentbyDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "WITH sourcewords AS (" + - " SELECT word " + - " FROM documenttopicwords " + - " WHERE document_id = :documentId " + - " GROUP BY word" + - "), " + - "similardocs AS (" + - " SELECT " + - " dtw.document_id, " + - " COUNT(DISTINCT dtw.word) AS sharedwords " + - " FROM documenttopicwords dtw " + - " JOIN sourcewords sw ON sw.word = dtw.word " + - " WHERE dtw.document_id != :documentId " + - " GROUP BY dtw.document_id " + - " ORDER BY sharedwords DESC" + - ") " + - "SELECT d.documentid, s.sharedwords " + - "FROM similardocs s " + - "JOIN document d ON s.document_id = d.id " + - "LIMIT 20"; + " SELECT word " + + " FROM documenttopicwords " + + " WHERE document_id = :documentId " + + " GROUP BY word" + + "), " + + "similardocs AS (" + + " SELECT " + + " dtw.document_id, " + + " COUNT(DISTINCT dtw.word) AS sharedwords " + + " FROM documenttopicwords dtw " + + " JOIN sourcewords sw ON sw.word = dtw.word " + + " WHERE dtw.document_id != :documentId " + + " GROUP BY dtw.document_id " + + " ORDER BY sharedwords DESC" + + ") " + + "SELECT d.documentid, s.sharedwords " + + "FROM similardocs s " + + "JOIN document d ON s.document_id = d.id " + + "LIMIT 20"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -1818,7 +1854,7 @@ public List getTaxonValuesAndCountByPageId(long documentId) throws Dat // Outer query: group by page_id, aggregate values and count String finalSql = "SELECT page_id, valuee AS taxon_value " + - "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; + "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; var query = session.createNativeQuery(finalSql) .setParameter("documentId", documentId) @@ -1834,8 +1870,8 @@ public List getNamedEntityValuesAndCountByPage(long documentId) throws return executeOperationSafely((session) -> { // Construct the SQL query to select page_id and coveredtext from the namedentity table String sql = "SELECT ne.page_id, ne.coveredtext AS named_entity_value, ne.typee AS named_entity_type " + - "FROM namedentity ne " + - "WHERE ne.document_id = :documentId"; + "FROM namedentity ne " + + "WHERE ne.document_id = :documentId"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -1956,10 +1992,10 @@ entities_in_sentences AS ( public List getTopicWordsByDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT topiclabel, word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY topiclabel, word " + - "ORDER BY avg_probability DESC"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY topiclabel, word " + + "ORDER BY avg_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); From 8e1a726d07c65da7fea4673032cd77936e5978b8 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:05:22 +0200 Subject: [PATCH 030/114] Add emotion field and getter method in CorpusAnnotationConfig --- .../config/corpusConfig/CorpusAnnotationConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java index ec588bd3..b5d3cea8 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java @@ -24,6 +24,7 @@ public class CorpusAnnotationConfig { private boolean scope; private boolean xscope; private boolean unifiedTopic; + private boolean emotion; public boolean isGeoNames() { return geoNames; @@ -203,4 +204,8 @@ public boolean isUnifiedTopic() { return unifiedTopic; } + public boolean isEmotion() { + return emotion; + } + } From c2e3a83a78b82bce76e42d90b9ec7d34ed1396df Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:06:17 +0200 Subject: [PATCH 031/114] Add emotion annotation handling in Importer --- .../main/java/org/texttechnologylab/Importer.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 8a20f607..f7831ccc 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -566,6 +566,11 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { () -> setUnifiedTopic(document, jCas), (ex) -> logImportWarn("This file should have contained UnifiedTopic annotations, but selecting them caused an error.", ex, filePath)); + if (corpusConfig.getAnnotations().isEmotion()) + ExceptionUtils.tryCatchLog( + () -> setEmotion(document, jCas), + (ex) -> logImportWarn("This file should have contained emotion annotations, but selecting them caused an error.", ex, filePath)); + // Keep this at the end of the annotation setting, as they might require previous annotations. Order matter here! if (corpusConfig.getAnnotations().isLogicalLinks()) ExceptionUtils.tryCatchLog( @@ -1410,6 +1415,13 @@ private void setUnifiedTopic(Document document, JCas jCas) { document.setUnifiedTopics(unifiedTopics); } + /** + * Selects and sets the emotions to a document + */ + private void setEmotion(Document document, JCas jCas) { + + } + /** * Set the cleaned full text. That is the sum of all tokens except of all anomalies * Update: OBSOLETE for now, as this takes forever and makes the OCR worse. From 755d476b43cf9b93623ffa9d4b5d9483dd965038 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:13:33 +0200 Subject: [PATCH 032/114] Add emotion processing in setEmotion method of Importer --- .../java/org/texttechnologylab/Importer.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index f7831ccc..3cce4ca0 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -19,6 +19,7 @@ import org.apache.uima.util.CasIOUtils; import org.apache.uima.util.CasLoadMode; import org.springframework.context.ApplicationContext; +import org.texttechnologylab.annotation.AnnotationComment; import org.texttechnologylab.annotation.DocumentAnnotation; import org.texttechnologylab.annotation.geonames.GeoNamesEntity; import org.texttechnologylab.annotation.link.*; @@ -39,6 +40,9 @@ import org.texttechnologylab.models.corpus.ocr.OCRPageAdapterImpl; import org.texttechnologylab.models.corpus.ocr.PageAdapter; import org.texttechnologylab.models.corpus.ocr.PageAdapterImpl; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; +import org.texttechnologylab.models.emotion.EmotionValue; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.ImportStatus; import org.texttechnologylab.models.imp.LogStatus; @@ -1419,7 +1423,40 @@ private void setUnifiedTopic(Document document, JCas jCas) { * Selects and sets the emotions to a document */ private void setEmotion(Document document, JCas jCas) { + List emotions = new ArrayList<>(); + JCasUtil.select(jCas, org.texttechnologylab.annotation.Emotion.class).forEach(e -> { + Emotion emotion = new Emotion(e.getBegin(), e.getEnd()); + emotion.setDocument(document); + + FSArray emotionAnnotations = e.getEmotions(); + List emotionValues = new ArrayList<>(); + for (AnnotationComment emotionAnnotation : emotionAnnotations) { + String emotionName = emotionAnnotation.getKey(); + EmotionType emotionType; + try { + emotionType = db.getOrCreateEmotionTypeByName(emotionName); + } catch (DatabaseOperationException ex) { + logger.error("Error while getting or creating emotion type for name: " + emotionName, ex); + continue; + } + String rawValue = emotionAnnotation.getValue(); + double value; + try { + value = Double.parseDouble(rawValue); + } catch (NumberFormatException ex) { + logger.warn("Invalid emotion value for " + emotionName + ": " + rawValue + ". Skipping this emotion.", ex); + continue; + } + EmotionValue emotionValue = new EmotionValue(emotionType, emotion, value); + emotionValues.add(emotionValue); + } + emotion.setEmotionValues(emotionValues); + + emotions.add(emotion); + }); + + document.setEmotions(emotions); } /** From 9b0a8b1778cecde9478f76768bf38906856feffb Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:23:06 +0200 Subject: [PATCH 033/114] Refactor EmotionValue class to extend UIMAAnnotation and implement WikiModel; add document association and constructors --- .../models/emotion/EmotionValue.java | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java index 549257e3..5ed6d93b 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java @@ -2,13 +2,15 @@ import lombok.Getter; import lombok.Setter; -import org.texttechnologylab.models.ModelBase; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; import javax.persistence.*; @Entity @Table(name = "emotion_value") -public class EmotionValue extends ModelBase { +public class EmotionValue extends UIMAAnnotation implements WikiModel { @Getter @Setter @@ -27,12 +29,34 @@ public class EmotionValue extends ModelBase { @Column(name = "value", nullable = false) private double value; + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + public EmotionValue() { + super(-1, -1); + } + + public EmotionValue(int begin, int end) { + super(begin, end); + } + + public EmotionValue(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); } - public EmotionValue(EmotionType emotionType, Emotion emotion, double value) { - this.emotionType = emotionType; - this.emotion = emotion; - this.value = value; + public Document getDocument() { + return document; } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "EV" + "-" + this.getId(); + } + } From 2c3475116aef82d9d94d3be76fe4d596ac5dcdcb Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:25:29 +0200 Subject: [PATCH 034/114] Enhance emotion handling in Importer by updating EmotionValue instantiation and setting additional properties --- .../src/main/java/org/texttechnologylab/Importer.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 3cce4ca0..9dcb8521 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1448,10 +1448,16 @@ private void setEmotion(Document document, JCas jCas) { logger.warn("Invalid emotion value for " + emotionName + ": " + rawValue + ". Skipping this emotion.", ex); continue; } - EmotionValue emotionValue = new EmotionValue(emotionType, emotion, value); + EmotionValue emotionValue = new EmotionValue(e.getBegin(), e.getEnd()); + emotionValue.setDocument(document); + emotionValue.setEmotion(emotion); + emotionValue.setEmotionType(emotionType); + emotionValue.setValue(value); + emotionValue.setCoveredText(e.getCoveredText()); emotionValues.add(emotionValue); } emotion.setEmotionValues(emotionValues); + emotion.setCoveredText(e.getCoveredText()); emotions.add(emotion); }); From f22aa3b131ef0c9c7030c43f6440bcc857056973 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 15:30:08 +0200 Subject: [PATCH 035/114] Update getWikiId method in Emotion to use 'E-' prefix for IDs --- .../main/java/org/texttechnologylab/models/emotion/Emotion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index c6c29b82..885bb968 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -50,7 +50,7 @@ public void setDocument(Document document) { @Override public String getWikiId() { - return "UT" + "-" + this.getId(); + return "E" + "-" + this.getId(); } } From 85fc804a2e517e7eebfe8c79c3671f5979018094 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Thu, 10 Jul 2025 15:31:19 +0200 Subject: [PATCH 036/114] fix wiki ID prefix in OffensiveSpeech entity --- .../models/offensiveSpeech/OffensiveSpeech.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index d3f3be4b..60243c6e 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -54,7 +54,7 @@ public void setDocument(Document document) { @Override public String getWikiId() { - return "UT" + "-" + this.getId(); + return "OS" + "-" + this.getId(); } } From d46f8a66601caf1cea6889ee60946d702d0a70ca Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Thu, 10 Jul 2025 15:32:42 +0200 Subject: [PATCH 037/114] Fix wiki ID format in Toxic class --- .../src/main/java/org/texttechnologylab/models/toxic/Toxic.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index eca25722..46aa6556 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -58,6 +58,6 @@ public void setDocument(Document document) { @Override public String getWikiId() { - return "UT" + "-" + this.getId(); + return "T" + "-" + this.getId(); } } From a60b136e4828e432e93d8a10823767fd34fa4973 Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 15:52:26 +0200 Subject: [PATCH 038/114] Create basic sentiment entity class --- .../models/sentiment/Sentiment.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java new file mode 100644 index 00000000..7adae13a --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -0,0 +1,42 @@ +package org.texttechnologylab.models.sentiment; + +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; + +import javax.persistence.*; + +@Entity +@Table(name = "sentiment") +public class Sentiment extends UIMAAnnotation implements WikiModel { + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Sentiment() { + super(-1, -1); + } + + public Sentiment(int begin, int end) { + super(begin, end); + } + + public Sentiment(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "S" + "-" + this.getId(); + } +} From d2f33664a7f530486abf0b6e8f5ef84911104df8 Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:00:02 +0200 Subject: [PATCH 039/114] New Columns and get set methods for columns added --- .../models/sentiment/Sentiment.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index 7adae13a..2791b80d 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -1,5 +1,7 @@ package org.texttechnologylab.models.sentiment; +import lombok.Getter; +import lombok.Setter; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; @@ -10,6 +12,27 @@ @Table(name = "sentiment") public class Sentiment extends UIMAAnnotation implements WikiModel { + @Getter + @Setter + @Column(name = "sentiment", nullable = false) + private int sentiment; + + @Getter + @Setter + @Column(name = "probability_positive", nullable = false) + private double probabilityPositive; + + @Getter + @Setter + @Column(name = "probability_neutral", nullable = false) + private double probabilityNeutral; + + @Getter + @Setter + @Column(name = "probability_negative", nullable = false) + private double probabilityNegative; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) private Document document; From 1691219d6bb3445440eb04affb471aec6f4f0021 Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:01:50 +0200 Subject: [PATCH 040/114] added typesystem annotation --- .../java/org/texttechnologylab/models/sentiment/Sentiment.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index 2791b80d..0d8a0a5d 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -2,6 +2,7 @@ import lombok.Getter; import lombok.Setter; +import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; @@ -10,6 +11,7 @@ @Entity @Table(name = "sentiment") +@Typesystem(types = {org.texttechnologylab.annotation.SentimentBert.class}) public class Sentiment extends UIMAAnnotation implements WikiModel { @Getter From 32278901579e2a603f1a0699d64f61887c4a01de Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:04:47 +0200 Subject: [PATCH 041/114] added sentiment to HibernateConf --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..d7d31b82 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -17,6 +17,7 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -80,6 +81,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // sentiment + metadataSources.addAnnotatedClass(Sentiment.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From 6545f9f481dafff158132ef9a746e670842c979a Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:07:19 +0200 Subject: [PATCH 042/114] Added Sentiment to Document --- .../java/org/texttechnologylab/models/corpus/Document.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9d898b4..a1027bbd 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -23,6 +23,7 @@ import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.search.AnnotationSearchResult; import org.texttechnologylab.models.search.PageSnippet; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.UnifiedTopic; @@ -218,6 +219,12 @@ public long getPrimaryDbIdentifier() { @Fetch(value = FetchMode.SUBSELECT) private List unifiedTopics; + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List sentiments; + public Document() { metadataTitleInfo = new MetadataTitleInfo(); } From 05110398b6d86ba623ce99f917cae69b89227d4e Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:09:39 +0200 Subject: [PATCH 043/114] Added Sentiment to CorpusAnnotationConfig --- .../config/corpusConfig/CorpusAnnotationConfig.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java index ec588bd3..8835ac21 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java @@ -24,6 +24,7 @@ public class CorpusAnnotationConfig { private boolean scope; private boolean xscope; private boolean unifiedTopic; + private boolean sentiment; public boolean isGeoNames() { return geoNames; @@ -202,5 +203,8 @@ public void setXscope(boolean xscope) { public boolean isUnifiedTopic() { return unifiedTopic; } + public boolean isSentiment() { + return sentiment; + } } From cb255e6aacc079ec71d5ece06abee3226cf82ecc Mon Sep 17 00:00:00 2001 From: imer Date: Thu, 10 Jul 2025 16:12:46 +0200 Subject: [PATCH 044/114] added sentiment stuff to Importer --- .../src/main/java/org/texttechnologylab/Importer.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 8a20f607..d37c1c69 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -566,6 +566,12 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { () -> setUnifiedTopic(document, jCas), (ex) -> logImportWarn("This file should have contained UnifiedTopic annotations, but selecting them caused an error.", ex, filePath)); + if (corpusConfig.getAnnotations().isSentiment()) + ExceptionUtils.tryCatchLog( + () -> setSentiment(document, jCas), + (ex) -> logImportWarn("This file should have contained sentiment annotations, but selecting them caused an error.", ex, filePath)); + + // Keep this at the end of the annotation setting, as they might require previous annotations. Order matter here! if (corpusConfig.getAnnotations().isLogicalLinks()) ExceptionUtils.tryCatchLog( @@ -1410,6 +1416,10 @@ private void setUnifiedTopic(Document document, JCas jCas) { document.setUnifiedTopics(unifiedTopics); } + private void setSentiment(Document document, JCas jCas){ + + } + /** * Set the cleaned full text. That is the sum of all tokens except of all anomalies * Update: OBSOLETE for now, as this takes forever and makes the OCR worse. From 2798ecb764ccc0f1856da9c811ca8c789250204b Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 16:19:47 +0200 Subject: [PATCH 045/114] Add emotion checkbox to corpus annotations template --- .../templates/corpus/components/corpusAnnotations.ftl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl index cfd27136..0dcc7102 100644 --- a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl +++ b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl @@ -186,5 +186,16 @@ + +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isEmotion() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
\ No newline at end of file From 1f9c5d29bbfacb89e1f12f833cb4a65d4fedd3d8 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 16:50:21 +0200 Subject: [PATCH 046/114] Add emotion handling to WikiService and create emotion annotation page --- .../resources/templates/css/lexicon.css | 4 ++ .../wiki/pages/emotionAnnotationPage.ftl | 60 +++++++++++++++++++ .../models/emotion/Emotion.java | 5 ++ .../wiki/EmotionWikiPageViewModel.java | 6 ++ .../services/LexiconService.java | 9 +-- .../PostgresqlDataInterface_Impl.java | 11 ++++ .../services/WikiService.java | 12 ++++ .../org/texttechnologylab/routes/WikiApi.java | 3 + 8 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java diff --git a/uce.portal/resources/templates/css/lexicon.css b/uce.portal/resources/templates/css/lexicon.css index a025a286..4c4021d2 100644 --- a/uce.portal/resources/templates/css/lexicon.css +++ b/uce.portal/resources/templates/css/lexicon.css @@ -88,6 +88,10 @@ border-left-color: lightpink; } +.lexicon-view .lexicon-entry[data-type="emotion"] { + border-left-color: darkorange; +} + .lexicon-view .lexicon-entry[data-type="geoname"] { border-left-color: dodgerblue; } diff --git a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl new file mode 100644 index 00000000..2d4451c0 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl @@ -0,0 +1,60 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ +
+ + +
+ <#assign emotion = vm.getWikiModel()> +
+ <#list emotion.collectEmotionValues() as emotionValuePair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ + +
+ <#include "*/wiki/components/kwic.ftl"> +
+ +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index 885bb968..fe31fd3b 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -8,6 +8,7 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.utils.Pair; import javax.persistence.*; import java.util.List; @@ -48,6 +49,10 @@ public void setDocument(Document document) { this.document = document; } + public List> collectEmotionValues() { + return this.emotionValues.stream().map(ev -> new Pair<>(ev.getEmotionType().getName(), ev.getValue())).toList(); + } + @Override public String getWikiId() { return "E" + "-" + this.getId(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java new file mode 100644 index 00000000..17514f20 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java @@ -0,0 +1,6 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class EmotionWikiPageViewModel extends AnnotationWikiPageViewModel { + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java index b6cf739e..285fa2f2 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java @@ -1,6 +1,5 @@ package org.texttechnologylab.services; -import io.micrometer.common.lang.Nullable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Service; @@ -8,19 +7,16 @@ import org.texttechnologylab.exceptions.DatabaseOperationException; import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.models.UIMAAnnotation; -import org.texttechnologylab.models.biofid.BiofidTaxon; import org.texttechnologylab.models.biofid.GazetteerTaxon; import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; +import org.texttechnologylab.models.emotion.Emotion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.viewModels.lexicon.LexiconOccurrenceViewModel; -import org.texttechnologylab.utils.SystemStatus; import javax.persistence.Table; -import java.sql.Array; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; @Service @@ -47,7 +43,8 @@ public class LexiconService { Cue.class, Scope.class, XScope.class, - UnifiedTopic.class)); + UnifiedTopic.class, + Emotion.class)); public LexiconService(PostgresqlDataInterface_Impl db) { this.db = db; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 79d522ef..47a83304 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1359,6 +1359,17 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public Emotion getInitializedEmotionById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var emotion = session.get(Emotion.class, id); + Hibernate.initialize(emotion.getEmotionValues()); + for (var ev : emotion.getEmotionValues()) { + Hibernate.initialize(ev.getEmotionType()); + } + return emotion; + }); + } + public EmotionType getEmotionTypeById(long id) throws DatabaseOperationException { return executeOperationSafely((session) -> session.get(EmotionType.class, id)); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java index 8fdb832c..02b32fae 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java @@ -109,6 +109,18 @@ public UnifiedTopicWikiPageViewModel buildUnifiedTopicWikiPageViewModel(long id, return viewModel; } + public EmotionWikiPageViewModel buildEmotionWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new EmotionWikiPageViewModel(); + var emotion = db.getInitializedEmotionById(id); + viewModel.setWikiModel(emotion); + viewModel.setDocument(db.getDocumentById(emotion.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Emotion"); + + return viewModel; + } + /** * Gets a DocumentTopicDistributionWikiPageViewModel to render a Wikipage for that topic distribution */ diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java index 38f1b077..6584a5fb 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java @@ -165,6 +165,9 @@ public WikiApi(ApplicationContext serviceContext, Configuration freemarkerConfig } else if (type.startsWith("DTR")) { model.put("vm", wikiService.buildTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/topicPage.ftl"; + } else if (type.startsWith("E")) { + model.put("vm", wikiService.buildEmotionWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/emotionAnnotationPage.ftl"; } else { // The type part of the wikiId was unknown. Throw an error. logger.warn("Someone tried to query a wiki page of a type that does not exist in UCE. This shouldn't happen."); From 2453aeceec026a3c81e2cfad02930d3601e18ccd Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 16:54:05 +0200 Subject: [PATCH 047/114] Remove KWIC view section from emotion annotation page --- .../resources/templates/wiki/pages/emotionAnnotationPage.ftl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl index 2d4451c0..1b012c8c 100644 --- a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl +++ b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl @@ -52,9 +52,4 @@ - -
- <#include "*/wiki/components/kwic.ftl"> -
- From 6d526771cd6539925193ec4fec9c35acecf91799 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 17:21:53 +0200 Subject: [PATCH 048/114] Add emotion markers and styling to document annotations --- .../templates/css/document-reader.css | 22 +++++++++++++ .../models/UIMAAnnotation.java | 28 ++++++++++++++--- .../models/corpus/Document.java | 2 ++ .../models/emotion/Emotion.java | 31 +++++++++++++++++++ .../PostgresqlDataInterface_Impl.java | 10 ++++++ 5 files changed, 89 insertions(+), 4 deletions(-) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 8c88a990..1ca744e2 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -338,6 +338,11 @@ body { position: relative; } +.document-content .colorable-emotion { + transition: background-color 0.3s ease; + position: relative; +} + .document-content .animated-topic-scroll { transition: box-shadow 0.3s ease-in-out; box-shadow: 0 0 10px 5px rgba(255, 255, 0, 0.5); @@ -360,6 +365,23 @@ body { margin-left: 4px; } +.document-content .emotion-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + .side-bar { width: 500px; transition: 0.5s; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index 96f53ca3..a532c5f8 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -1,21 +1,18 @@ package org.texttechnologylab.models; -import io.micrometer.common.lang.Nullable; import lombok.Getter; import lombok.Setter; -import org.texttechnologylab.models.biofid.BiofidTaxon; import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.corpus.links.AnnotationLink; import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; -import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; +import org.texttechnologylab.models.emotion.Emotion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.utils.StringUtils; import javax.persistence.*; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; @MappedSuperclass public class UIMAAnnotation extends ModelBase implements Linkable { @@ -128,6 +125,9 @@ public String buildHTMLString(List annotations, String coveredTe Map topicMarkers = new TreeMap<>(); Map topicCoverWrappersStart = new TreeMap<>(); Map topicCoverWrappersEnd = new TreeMap<>(); + Map emotionMarkers = new TreeMap<>(); + Map emotionCoverWrappersStart = new TreeMap<>(); + Map emotionCoverWrappersEnd = new TreeMap<>(); for (var annotation : annotations) { @@ -168,6 +168,17 @@ public String buildHTMLString(List annotations, String coveredTe continue; } + if (annotation instanceof Emotion emotion) { + var start = emotion.getBegin() - offset - errorOffset; + var end = emotion.getEnd() - offset - errorOffset; + + emotionCoverWrappersStart.put(start, emotion.generateEmotionCoveredStartSpan()); + emotionCoverWrappersEnd.put(end, ""); + + emotionMarkers.put(end, emotion.generateEmotionMarker()); + continue; + } + var start = annotation.getBegin() - offset - errorOffset; var end = annotation.getEnd() - offset - errorOffset; @@ -187,6 +198,9 @@ public String buildHTMLString(List annotations, String coveredTe if (topicCoverWrappersEnd.containsKey(i)) { finalText.append(topicCoverWrappersEnd.get(i)); } + if (emotionCoverWrappersEnd.containsKey(i)) { + finalText.append(emotionCoverWrappersEnd.get(i)); + } if (endTags.containsKey(i)) { //finalText.append(endTags.get(i).getFirst()); for (var tag : endTags.get(i)) { @@ -195,6 +209,9 @@ public String buildHTMLString(List annotations, String coveredTe } // Insert start spans + if (emotionCoverWrappersStart.containsKey(i)) { + finalText.append(emotionCoverWrappersStart.get(i)); + } if (topicCoverWrappersStart.containsKey(i)) { finalText.append(topicCoverWrappersStart.get(i)); } @@ -203,6 +220,9 @@ public String buildHTMLString(List annotations, String coveredTe if (topicMarkers.containsKey(i)) { finalText.append(topicMarkers.get(i)); } + if (emotionMarkers.containsKey(i)) { + finalText.append(emotionMarkers.get(i)); + } if (startTags.containsKey(i)) { finalText.append(generateMultiHTMLTag(startTags.get(i))); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index 9b42c6cd..bf6ff836 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -368,6 +368,8 @@ public List getAllAnnotations(int pagesSkip, int pagesTake) { annotations.addAll(events.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); // unifiedTopics annotations.addAll(unifiedTopics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); + // emotions + annotations.addAll(emotions.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index fe31fd3b..513b9fbf 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -11,7 +11,9 @@ import org.texttechnologylab.utils.Pair; import javax.persistence.*; +import java.util.Comparator; import java.util.List; +import java.util.UUID; @Entity @Table(name = "emotion") @@ -49,6 +51,35 @@ public void setDocument(Document document) { this.document = document; } + private EmotionValue getRepresentativeEmotionValue() { + if (this.emotionValues != null && !this.emotionValues.isEmpty()) { + return this.emotionValues.stream().max(Comparator.comparingDouble(EmotionValue::getValue)).orElse(null); + } + return null; + } + + public String generateEmotionMarker() { + String repEmotionValue = ""; + if (this.getEmotionValues() != null && !this.getEmotionValues().isEmpty()) { + var repEmotion = this.getRepresentativeEmotionValue(); + if (repEmotion != null) { + repEmotionValue = repEmotion.getEmotionType().getName(); + } + } + return String.format("e", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repEmotionValue); + } + + public String generateEmotionCoveredStartSpan() { + String repEmotionValue = ""; + if (this.getEmotionValues() != null && !this.getEmotionValues().isEmpty()) { + var repEmotion = this.getRepresentativeEmotionValue(); + if (repEmotion != null) { + repEmotionValue = repEmotion.getEmotionType().getName(); + } + } + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repEmotionValue); + } + public List> collectEmotionValues() { return this.emotionValues.stream().map(ev -> new Pair<>(ev.getEmotionType().getName(), ev.getValue())).toList(); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 47a83304..8f0a7906 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -2116,6 +2116,16 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag Hibernate.initialize(topic.getTopics()); } + // emotion + Hibernate.initialize(doc.getEmotions()); + + for (var emotion : doc.getEmotions()) { + Hibernate.initialize(emotion.getEmotionValues()); + for (var ev : emotion.getEmotionValues()) { + Hibernate.initialize(ev.getEmotionType()); + } + } + for (var link : doc.getWikipediaLinks()) { Hibernate.initialize(link.getWikiDataHyponyms()); } From 716e2268b5ae6d19d1f6d79eecaa876dbe1ec100 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:08:48 +0200 Subject: [PATCH 049/114] Add emotion development visualization and API endpoint --- .../resources/templates/js/documentReader.js | 246 ++++++++++++------ .../templates/reader/documentReaderView.ftl | 4 + .../PostgresqlDataInterface_Impl.java | 64 +++++ .../main/java/org/texttechnologylab/App.java | 9 +- .../texttechnologylab/routes/DocumentApi.java | 28 +- 5 files changed, 266 insertions(+), 85 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index ac7fb3c7..7036110c 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -370,8 +370,7 @@ function colorUnifiedTopics(selectedTopic) { if ($selectedTopicTag.length === 0) { color = topicColorMap[selectedTopic]; - } - else{ + } else { color = $selectedTopicTag.css('background-color'); } @@ -477,16 +476,16 @@ function updateTopicNavButtonStates() { function initScrollbarMinimap() { setTimeout(updateMinimapMarkers, 500); - $(window).on('scroll', function() { + $(window).on('scroll', function () { updateMinimapScroll(); }); - $(window).on('resize', function() { + $(window).on('resize', function () { updateMinimapMarkers(); updateMinimapScroll(); }); - $('.scrollbar-minimap').on('click', function(e) { + $('.scrollbar-minimap').on('click', function (e) { const clickPosition = (e.pageY - $(this).offset().top) / $(this).height(); const dimensions = getMinimapDimensions(); const documentPosition = minimapToDocumentPosition(clickPosition * dimensions.minimapHeight, dimensions); @@ -497,7 +496,7 @@ function initScrollbarMinimap() { }, 300); }); - $(document).on('mouseenter', '.minimap-marker', function(e) { + $(document).on('mouseenter', '.minimap-marker', function (e) { const $marker = $(this); const $preview = $('.minimap-preview'); const $previewContent = $('.preview-content'); @@ -513,14 +512,14 @@ function initScrollbarMinimap() { const markerTop = parseFloat($marker.css('top')); const approximateDocumentPosition = minimapToDocumentPosition(markerTop, dimensions); - const $topicElements = $('.colorable-topic').filter(function() { + const $topicElements = $('.colorable-topic').filter(function () { return $(this).data('topic-value') === topicValue; }); let closestElement = null; let minDistance = Number.MAX_VALUE; - $topicElements.each(function() { + $topicElements.each(function () { const $element = $(this); const elementOffset = $element.offset(); @@ -547,7 +546,7 @@ function initScrollbarMinimap() { const coveredText = closestElement.data('wcovered'); if (coveredText) { - previewText = coveredText; + previewText = coveredText; } } if (previewTitle) { @@ -580,11 +579,11 @@ function initScrollbarMinimap() { } }); - $(document).on('mouseleave', '.minimap-marker', function() { + $(document).on('mouseleave', '.minimap-marker', function () { $('.minimap-preview').hide(); }); - $('.scrollbar-minimap').on('mouseleave', function() { + $('.scrollbar-minimap').on('mouseleave', function () { $('.minimap-preview').hide(); }); } @@ -592,10 +591,10 @@ function initScrollbarMinimap() { function updateMinimapMarkers() { const $minimap = $('.minimap-markers'); const dimensions = getMinimapDimensions(); - + $minimap.empty(); - $('.document-content .page').each(function(index) { + $('.document-content .page').each(function (index) { const $page = $(this); if (!$page.attr('id')) { @@ -628,7 +627,7 @@ function addAllTopicMarkersToMinimap() { const dimensions = getMinimapDimensions(); const topicPositions = {}; - $('.colorable-topic').each(function() { + $('.colorable-topic').each(function () { const $topic = $(this); const topicValue = $topic.data('topic-value'); @@ -651,7 +650,7 @@ function addAllTopicMarkersToMinimap() { } }); - Object.keys(topicPositions).forEach(function(position) { + Object.keys(topicPositions).forEach(function (position) { const pos = parseInt(position); const topicData = topicPositions[pos]; const topicValues = Object.keys(topicData.topics); @@ -673,7 +672,7 @@ function addAllTopicMarkersToMinimap() { }); } -function updateTopicMarkersOnMinimap(selectedTopic=null) { +function updateTopicMarkersOnMinimap(selectedTopic = null) { const $minimap = $('.minimap-markers'); const dimensions = getMinimapDimensions(); @@ -685,7 +684,7 @@ function updateTopicMarkersOnMinimap(selectedTopic=null) { const activeTopic = selectedTopic ? selectedTopic : $activeTopic.data('topic'); const topicColor = topicColorMap[activeTopic]; - $('.colorable-topic').each(function() { + $('.colorable-topic').each(function () { const $topic = $(this); const topicValue = $topic.data('topic-value'); @@ -718,18 +717,18 @@ function getMinimapDimensions() { } function documentToMinimapPosition(documentPos, dimensions) { - const { documentHeight, minimapHeight } = dimensions || getMinimapDimensions(); + const {documentHeight, minimapHeight} = dimensions || getMinimapDimensions(); return (documentPos / documentHeight) * minimapHeight; } function minimapToDocumentPosition(minimapPos, dimensions) { - const { documentHeight, minimapHeight } = dimensions || getMinimapDimensions(); + const {documentHeight, minimapHeight} = dimensions || getMinimapDimensions(); return (minimapPos / minimapHeight) * documentHeight; } function createMinimapMarker(options) { - const { top, height, color, elementId, topic, className } = options; - + const {top, height, color, elementId, topic, className} = options; + const $marker = $('
') .addClass('minimap-marker') .addClass(className || '') @@ -738,10 +737,10 @@ function createMinimapMarker(options) { 'height': Math.max(2, height) + 'px', 'background-color': color || '#ccc' }); - + if (elementId) $marker.attr('data-element-id', elementId); if (topic) $marker.attr('data-topic', topic); - + return $marker; } @@ -814,7 +813,7 @@ document.querySelectorAll('.tab-btn').forEach(btn => { $('.scrollbar-minimap').hide(); sideBar.classList.add('visualization-expanded'); } else { - setTimeout(updateFloatingUIPositions,500) ; + setTimeout(updateFloatingUIPositions, 500); currentSelectedTopic = null; sideBar.classList.remove('visualization-expanded'); $('.scrollbar-minimap').show(); @@ -863,10 +862,12 @@ $(document).on('click', '.viz-nav-btn', function () { setTimeout(() => renderSentenceTopicSankey('vp-5'), 500); } + if (target === '#viz-panel-6') { + setTimeout(() => renderEmotionDevelopment('vp-6'), 500); + } }); - function renderSentenceTopicNetwork(containerId) { const container = document.getElementById(containerId); if (!container || container.classList.contains('rendered')) return; @@ -878,13 +879,13 @@ function renderSentenceTopicNetwork(containerId) { $.ajax({ url: `/api/document/unifiedTopicSentenceMap`, method: 'GET', - data: { documentId }, + data: {documentId}, dataType: 'json', success: function (utToSentenceMapList) { const utToSentenceMap = new Map(); const topicToSentences = {}; - utToSentenceMapList.forEach(({ unifiedtopicId, sentenceId }) => { + utToSentenceMapList.forEach(({unifiedtopicId, sentenceId}) => { const utId = unifiedtopicId.toString(); const sId = sentenceId.toString(); utToSentenceMap.set(utId, sId); @@ -895,7 +896,7 @@ function renderSentenceTopicNetwork(containerId) { $.ajax({ url: `/api/rag/sentenceEmbeddings`, method: 'GET', - data: { documentId }, + data: {documentId}, dataType: 'json', success: function (embeddings) { $('.visualization-spinner').hide() @@ -908,7 +909,7 @@ function renderSentenceTopicNetwork(containerId) { return; } const sentenceEmbeddingMap = new Map(); - embeddings.forEach(({ sentenceId, tsne2d }) => { + embeddings.forEach(({sentenceId, tsne2d}) => { sentenceEmbeddingMap.set(sentenceId.toString(), tsne2d); }); @@ -931,11 +932,11 @@ function renderSentenceTopicNetwork(containerId) { nodes.push({ id: sentenceId, - name: `Sentence `+utId, + name: `Sentence ` + utId, symbolSize: 20, x, y, - itemStyle: { color }, - label: { show: false }, + itemStyle: {color}, + label: {show: false}, tooltip: { confine: true, // prevent overflow @@ -970,15 +971,15 @@ function renderSentenceTopicNetwork(containerId) { const neighbors = embeddingArray .filter(([id2]) => id1 !== id2 && nodeSet.has(id2)) - .map(([id2, vec2]) => ({ id2, dist: euclidean(vec1, vec2) })) + .map(([id2, vec2]) => ({id2, dist: euclidean(vec1, vec2)})) .sort((a, b) => a.dist - b.dist) .slice(0, k); - neighbors.forEach(({ id2 }) => { + neighbors.forEach(({id2}) => { links.push({ source: id1, target: id2, - lineStyle: { color: '#bbb', opacity: 0.2 } + lineStyle: {color: '#bbb', opacity: 0.2} }); }); }); @@ -999,27 +1000,27 @@ function renderSentenceTopicNetwork(containerId) { '', nodes, links, - null, - function (params) { - if (params.dataType === 'node') { - const name = params.name.split('Sentence ')[1]; - $('.scrollbar-minimap').hide(); - hideTopicNavButtons(); - clearTopicColoring(); - $('.colorable-topic').each(function () { - const topicValue = $(this).data('topic-value'); - const utId = this.id.replace('utopic-UT-', ''); - if (utId === name) { - $(this).css({ - 'background-color': topicColorMap[topicValue], - 'border-radius': '3px', - 'padding': '0 2px' - }); - this.scrollIntoView({ behavior: 'smooth', block: 'center' }); - } - }); - } - }); + null, + function (params) { + if (params.dataType === 'node') { + const name = params.name.split('Sentence ')[1]; + $('.scrollbar-minimap').hide(); + hideTopicNavButtons(); + clearTopicColoring(); + $('.colorable-topic').each(function () { + const topicValue = $(this).data('topic-value'); + const utId = this.id.replace('utopic-UT-', ''); + if (utId === name) { + $(this).css({ + 'background-color': topicColorMap[topicValue], + 'border-radius': '3px', + 'padding': '0 2px' + }); + this.scrollIntoView({behavior: 'smooth', block: 'center'}); + } + }); + } + }); container.classList.add('rendered'); }, @@ -1074,17 +1075,16 @@ function computeTopicSimilarityMatrix(data, type = "cosine") { } - function renderTopicSimilarityMatrix(containerId) { const container = document.getElementById(containerId); - if (!container || container.classList.contains('rendered')){ + if (!container || container.classList.contains('rendered')) { $('.selector-container').show(); return; } $('.visualization-spinner').show() const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - $.get('/api/document/page/topicWords', { documentId: docId }) + $.get('/api/document/page/topicWords', {documentId: docId}) .then(data => { $('.visualization-spinner').hide() if (!data || !Array.isArray(data) || data.length === 0) { @@ -1101,13 +1101,13 @@ function renderTopicSimilarityMatrix(containerId) { function updateChart() { const type = similarityTypeSelector.value; - const { labels, matrix } = computeTopicSimilarityMatrix(data, type); + const {labels, matrix} = computeTopicSimilarityMatrix(data, type); const tooltipFormatter = function (params) { const xLabel = labels[params.data[0]]; const yLabel = labels[params.data[1]]; const value = type === "count" ? params.data[2] : params.data[2].toFixed(3); - return xLabel + " & " + yLabel + "
" + (type.charAt(0).toUpperCase() + type.slice(1)) + ": " + value; + return xLabel + " & " + yLabel + "
" + (type.charAt(0).toUpperCase() + type.slice(1)) + ": " + value; }; window.graphVizHandler.createHeatMap( @@ -1136,7 +1136,7 @@ function renderTopicEntityChordDiagram(containerId) { const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - $.get('/api/document/page/topicEntityRelation', { documentId: docId }) + $.get('/api/document/page/topicEntityRelation', {documentId: docId}) .then(data => { $('.visualization-spinner').hide() if (!data || !Array.isArray(data) || data.length === 0) { @@ -1154,8 +1154,8 @@ function renderTopicEntityChordDiagram(containerId) { let nodeIndex = 0; const categories = [ - { name: 'Topic', itemStyle: { color: '#5470C6' } }, - { name: 'Entity', itemStyle: { color: '#91CC75' } } + {name: 'Topic', itemStyle: {color: '#5470C6'}}, + {name: 'Entity', itemStyle: {color: '#91CC75'}} ]; function getCategory(name, isEntity) { @@ -1169,11 +1169,11 @@ function renderTopicEntityChordDiagram(containerId) { if (topic && !nodeMap.has(topic)) { nodeMap.set(topic, nodeIndex++); - nodes.push({ name: topic, value: 0, category: getCategory(topic, false) }); + nodes.push({name: topic, value: 0, category: getCategory(topic, false)}); } if (entityType && !nodeMap.has(entityType)) { nodeMap.set(entityType, nodeIndex++); - nodes.push({ name: entityType, value: 0, category: getCategory(entityType, true) }); + nodes.push({name: entityType, value: 0, category: getCategory(entityType, true)}); } // Step 2: count link frequency as weight @@ -1262,7 +1262,7 @@ function renderTopicEntityChordDiagram(containerId) { const pageNumber = params.name; const pageElement = document.querySelector('.page[data-id="' + pageNumber + '"]'); if (pageElement) { - pageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }); + pageElement.scrollIntoView({behavior: 'smooth', block: 'start'}); } else { console.error(`Page ` + pageNumber + ` not found.`); } @@ -1364,7 +1364,7 @@ function renderSentenceTopicSankey(containerId) { 'border-radius': '3px', 'padding': '0 2px' }); - this.scrollIntoView({ behavior: 'smooth', block: 'center' }); + this.scrollIntoView({behavior: 'smooth', block: 'center'}); } }); } @@ -1377,6 +1377,94 @@ function renderSentenceTopicSankey(containerId) { }); } +function renderEmotionDevelopment(containerId) { + const container = document.getElementById(containerId); + if (!container || container.classList.contains('rendered')) return; + + $('.visualization-spinner').show(); + const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); + + const emotionReq = $.get('/api/document/page/emotionDev', {documentId: docId}); + emotionReq.then(emotionData => { + $('.visualization-spinner').hide(); + if (!emotionData || typeof emotionData !== 'object' || !("emotionTypes" in emotionData) || !Array.isArray(emotionData.emotionTypes) || emotionData.emotionTypes.length === 0 || !("emotionData" in emotionData) || !Array.isArray(emotionData.emotionData) || emotionData.emotionData.length === 0) { + const container = document.getElementById(containerId); + if (container) { + container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; + } + container.classList.add('rendered'); + return; + } + + const emotionColors = {}; + emotionData.emotionTypes.forEach((emotionType, index) => { + const hue = (index * 360 / emotionData.emotionTypes.length) % 360; + emotionColors[emotionType.id] = 'hsl(' + hue + ', 70%, 50%)'; + }); + + const seriesData = emotionData.emotionTypes.map(emotionType => { + const emotionId = emotionType.id; + const emotionName = emotionType.name; + const emotionValues = emotionData.emotionData.map(pageData => { + const emotionValue = pageData.find(e => e.emotionType === emotionId); + return emotionValue ? (emotionValue.emotionValue * 100) : 0; + }); + return { + name: emotionName, + type: 'bar', + data: emotionValues, + color: emotionColors[emotionId] || '#888', + smooth: true, + lineStyle: { + width: 2 + }, + itemStyle: { + borderWidth: 1 + } + }; + }); + const xData = emotionData.emotionData.map((_, index) => index + 1); + const chartConfig = { + xData: xData, + seriesData: seriesData, + yLabel: 'Emotion Value', + }; + console.log(chartConfig); + const tooltipFormatter = function (params) { + console.log(params); + return 'Test'; + }; + + window.graphVizHandler.createBarLineChart( + containerId, + '', + chartConfig, + tooltipFormatter, + function (params) { + // do nothing + } + ).then(chart => { + console.log('Emotion development chart created successfully:', chart); + }).catch(err => { + console.error('Error creating emotion development chart:', err); + $('.visualization-spinner').hide(); + const container = document.getElementById(containerId); + if (container) { + container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; + } + }); + + container.classList.add('rendered'); + }).catch(() => { + $('.visualization-spinner').hide(); + const container = document.getElementById(containerId); + if (container) { + container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; + } + container.classList.add('rendered'); + }); +} + function renderTemporalExplorer(containerId) { const container = document.getElementById(containerId); @@ -1384,11 +1472,11 @@ function renderTemporalExplorer(containerId) { $('.visualization-spinner').show() const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - const taxonReq = $.get('/api/document/page/taxon', { documentId: docId }); - const topicReq = $.get('/api/document/page/topics', { documentId: docId }); - const entityReq = $.get('/api/document/page/namedEntities', { documentId: docId }); - const lemmaReq = $.get('/api/document/page/lemma', { documentId: docId }); - const geonameReq = $.get('/api/document/page/geoname', { documentId: docId }); + const taxonReq = $.get('/api/document/page/taxon', {documentId: docId}); + const topicReq = $.get('/api/document/page/topics', {documentId: docId}); + const entityReq = $.get('/api/document/page/namedEntities', {documentId: docId}); + const lemmaReq = $.get('/api/document/page/lemma', {documentId: docId}); + const geonameReq = $.get('/api/document/page/geoname', {documentId: docId}); Promise.all([taxonReq, topicReq, entityReq, lemmaReq, geonameReq]).then(([taxon, topics, entities, lemma, geoname]) => { $('.visualization-spinner').hide() @@ -1447,7 +1535,7 @@ function renderTemporalExplorer(containerId) { // Collect unique sorted page IDs const rawPageIds = []; - annotationSources.forEach(({ data, pageField }) => { + annotationSources.forEach(({data, pageField}) => { data.forEach(d => { const pid = parseInt(d[pageField]); if (!isNaN(pid)) rawPageIds.push(pid); @@ -1462,7 +1550,7 @@ function renderTemporalExplorer(containerId) { const dataMap = new Map(); - annotationSources.forEach(({ key, data, pageField, valueField, transformValue }) => { + annotationSources.forEach(({key, data, pageField, valueField, transformValue}) => { data.forEach(item => { const pid = parseInt(item[pageField]); const page = pageIdToPageNumber.get(pid); @@ -1490,10 +1578,10 @@ function renderTemporalExplorer(containerId) { const pages = sorted.map(row => row.page); const seriesData = annotationSources - .map(({ key, label, color }) => { + .map(({key, label, color}) => { const data = sorted.map(row => row[key]?.length || 0); const hasNonZero = data.some(count => count > 0); - return hasNonZero ? { name: label, data, color } : null; + return hasNonZero ? {name: label, data, color} : null; }) .filter(d => d !== null); @@ -1514,7 +1602,7 @@ function renderTemporalExplorer(containerId) { let tooltipHtml = '
Page ' + page + '
'; - annotationSources.forEach(({ key, label, color }) => { + annotationSources.forEach(({key, label, color}) => { if (!seriesNames.has(label)) return; const items = record[key]; if (!items || items.length === 0) return; @@ -1555,7 +1643,7 @@ function renderTemporalExplorer(containerId) { const pageNumber = params.name; const pageElement = document.querySelector('.page[data-id="' + pageNumber + '"]'); if (pageElement) { - pageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }); + pageElement.scrollIntoView({behavior: 'smooth', block: 'start'}); } else { console.error(`Page ` + pageNumber + ` not found.`); } diff --git a/uce.portal/resources/templates/reader/documentReaderView.ftl b/uce.portal/resources/templates/reader/documentReaderView.ftl index ee05a3f5..ce0ac06d 100644 --- a/uce.portal/resources/templates/reader/documentReaderView.ftl +++ b/uce.portal/resources/templates/reader/documentReaderView.ftl @@ -300,6 +300,9 @@
+
+
+
@@ -308,6 +311,7 @@ +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 8f0a7906..a0f322e4 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1905,6 +1905,70 @@ public List getLemmaByPage(long documentId) throws DatabaseOperationEx }); } + public List getDocumentEmotionsOrdered(long documentId) throws DatabaseOperationException { + class EmotionEntry { + long emotionType; + double emotionValue; + } + return executeOperationSafely((session) -> { + String sql = """ + SELECT e.id AS emotionId, + ev.emotion_type_id AS emotionType, + ev.value AS emotionValue + FROM emotion e + JOIN emotion_value ev ON e.id = ev.emotion_id + WHERE e.document_id = :documentId + ORDER BY e.beginn, e.endd + """; + var query = session.createNativeQuery(sql) + .setParameter("documentId", documentId); + + var result = query.getResultList(); + List emotionIds = new ArrayList<>(); + Map> emotionMap = new HashMap<>(); + for (Object raw : result) { + Object[] row = (Object[]) raw; + long emotionId = ((Number) row[0]).longValue(); + long emotionType = ((Number) row[1]).longValue(); + double emotionValue = ((Number) row[2]).doubleValue(); + + if (!emotionIds.contains(emotionId)) { + emotionIds.add(emotionId); + } + + EmotionEntry entry = new EmotionEntry(); + entry.emotionType = emotionType; + entry.emotionValue = emotionValue; + + emotionMap.computeIfAbsent(emotionId, k -> new ArrayList<>()).add(entry); + } + return emotionIds.stream() + .map(emotionMap::get) + .map(e -> e.stream().map(entry -> Map.of( + "emotionType", entry.emotionType, + "emotionValue", entry.emotionValue + )).toArray()) + .toList(); + }); + } + + public List getEmotionTypes() throws DatabaseOperationException { + return executeOperationSafely((session) -> { + String sql = "SELECT id, name FROM emotion_type ORDER BY name"; + var query = session.createNativeQuery(sql); + //noinspection unchecked + return query.getResultList().stream() + .map(result -> { + Object[] row = (Object[]) result; + return Map.of( + "id", ((Number) row[0]).longValue(), + "name", (String) row[1] + ); + }) + .toList(); + }); + } + public List getGeonameByPage(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT gn.page_id, gn.coveredtext AS geoname_value " + diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java index f374a964..5628bfa1 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java @@ -8,8 +8,6 @@ import org.apache.commons.cli.ParseException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.keycloak.authorization.client.AuthzClient; -import org.keycloak.representations.idm.authorization.AuthorizationRequest; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.texttechnologylab.auth.AuthenticationRouteRegister; @@ -19,7 +17,6 @@ import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.freeMarker.Renderer; import org.texttechnologylab.freeMarker.RequestContextHolder; -import org.texttechnologylab.models.authentication.UceUser; import org.texttechnologylab.models.corpus.Corpus; import org.texttechnologylab.models.corpus.UCELog; import org.texttechnologylab.modules.ModelGroup; @@ -39,7 +36,10 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.StandardOpenOption; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import static spark.Spark.*; @@ -420,6 +420,7 @@ private static void initSparkRoutes(ApplicationContext context) throws IOExcepti get("/page/namedEntities", (registry.get(DocumentApi.class)).getDocumentNamedEntitiesByPage); get("/page/lemma", (registry.get(DocumentApi.class)).getDocumentLemmaByPage); get("/page/geoname", (registry.get(DocumentApi.class)).getDocumentGeonameByPage); + get("/page/emotionDev", (registry.get(DocumentApi.class)).getDocumentEmotionDevelopment); }); path("/rag", () -> { diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java index db5c4b15..4cdc443a 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java @@ -157,7 +157,7 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo var id = ExceptionUtils.tryCatchLog(() -> request.queryParams("id"), (ex) -> logger.error("Error: the url for the document reader requires an 'id' query parameter. " + - "Document reader can't be built.", ex)); + "Document reader can't be built.", ex)); if (id == null) return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); @@ -406,7 +406,6 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo } }); - public Route getSentenceTopicsWithEntities = ((request, response) -> { var documentId = ExceptionUtils.tryCatchLog(() -> Long.parseLong(request.queryParams("documentId")), (ex) -> logger.error("Error: couldn't determine the documentId for sentence topics with entities. ", ex)); @@ -514,5 +513,30 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo } }; + public Route getDocumentEmotionDevelopment = (request, response) -> { + var documentId = ExceptionUtils.tryCatchLog(() -> Long.parseLong(request.queryParams("documentId")), + (ex) -> logger.error("Error: couldn't determine the documentId for emotion development.", ex)); + + if (documentId == null) { + response.status(400); + return new CustomFreeMarkerEngine(this.freemarkerConfig) + .render(new ModelAndView(Map.of("information", "Missing documentId parameter for emotion development"), "defaultError.ftl")); + } + + try { + var emotionData = db.getDocumentEmotionsOrdered(documentId); + var emotionTypes = db.getEmotionTypes(); + Map emotionDevelopment = new HashMap<>(); + emotionDevelopment.put("emotionTypes", emotionTypes); + emotionDevelopment.put("emotionData", emotionData); + response.type("application/json"); + return new Gson().toJson(emotionDevelopment); + } catch (Exception ex) { + logger.error("Error retrieving document emotion development.", ex); + response.status(500); + return new CustomFreeMarkerEngine(this.freemarkerConfig) + .render(new ModelAndView(Map.of("information", "Error retrieving document emotion development."), "defaultError.ftl")); + } + }; } From 7d33565543c89a5b63c675ec918fe90ae81e5f84 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:14:07 +0200 Subject: [PATCH 050/114] why does this not use a class?? --- uce.portal/resources/templates/css/document-reader.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 1ca744e2..5536eec4 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -763,7 +763,7 @@ body { outline: none; } -#vp-3, #vp-4, #vp-5, #vp-2, #vp-1 { +#vp-3, #vp-4, #vp-5, #vp-2, #vp-1, #vp-6 { display: flex; align-items: center; justify-content: center; From de5dd582818c26b5eef22590a4ea157771a859bd Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:15:15 +0200 Subject: [PATCH 051/114] Add createLineChart method to GraphVizHandler for emotion data visualization --- .../resources/templates/js/documentReader.js | 14 +--- uce.portal/resources/templates/js/graphViz.js | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 7036110c..6690844c 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1429,13 +1429,12 @@ function renderEmotionDevelopment(containerId) { seriesData: seriesData, yLabel: 'Emotion Value', }; - console.log(chartConfig); const tooltipFormatter = function (params) { console.log(params); return 'Test'; }; - window.graphVizHandler.createBarLineChart( + window.graphVizHandler.createLineChart( containerId, '', chartConfig, @@ -1443,16 +1442,7 @@ function renderEmotionDevelopment(containerId) { function (params) { // do nothing } - ).then(chart => { - console.log('Emotion development chart created successfully:', chart); - }).catch(err => { - console.error('Error creating emotion development chart:', err); - $('.visualization-spinner').hide(); - const container = document.getElementById(containerId); - if (container) { - container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; - } - }); + ); container.classList.add('rendered'); }).catch(() => { diff --git a/uce.portal/resources/templates/js/graphViz.js b/uce.portal/resources/templates/js/graphViz.js index 366de940..686e1691 100644 --- a/uce.portal/resources/templates/js/graphViz.js +++ b/uce.portal/resources/templates/js/graphViz.js @@ -379,6 +379,81 @@ var GraphVizHandler = (function () { return echart; }; + GraphVizHandler.prototype.createLineChart = async function ( + target, + title, + config, + tooltipFormatter, + onClick = null + ) { + const chartId = generateUUID(); + + const { + xData, + seriesData, + yLabel = 'Count' + } = config; + + const option = { + tooltip: { + trigger: 'axis', + enterable: true, + backgroundColor: '#fff', + borderColor: '#ccc', + borderWidth: 1, + textStyle: { + color: '#000', + fontSize: 12 + }, + formatter: tooltipFormatter + }, + + title: { + text: title, + left: 'center' + }, + + legend: { + data: seriesData.map(s => s.name), + top: 'auto' + }, + + xAxis: { + type: 'category', + name: 'X', + data: xData + }, + + yAxis: { + type: 'value', + name: yLabel + }, + + series: [] + }; + + seriesData.forEach(s => { + option.series.push({ + name: s.name, + type: 'line', + data: s.data, + symbol: 'circle', + symbolSize: 10, + lineStyle: { width: 3, color: s.color }, + itemStyle: { color: s.color }, + z: 2 + }); + }); + + const echart = new ECharts(target, option); + this.activeCharts[chartId] = echart; + + if (onClick && typeof onClick === 'function') { + echart.getInstance().on('click', onClick); + } + + return echart; + }; GraphVizHandler.prototype.createChordChart = async function ( From f35621a1673281bd9d374594f045d0aae0098705 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:22:29 +0200 Subject: [PATCH 052/114] Refactor emotion value calculation and enhance tooltip formatting in document reader --- uce.portal/resources/templates/js/documentReader.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 6690844c..5939e145 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1407,7 +1407,7 @@ function renderEmotionDevelopment(containerId) { const emotionName = emotionType.name; const emotionValues = emotionData.emotionData.map(pageData => { const emotionValue = pageData.find(e => e.emotionType === emotionId); - return emotionValue ? (emotionValue.emotionValue * 100) : 0; + return emotionValue ? emotionValue.emotionValue : 0; }); return { name: emotionName, @@ -1430,8 +1430,14 @@ function renderEmotionDevelopment(containerId) { yLabel: 'Emotion Value', }; const tooltipFormatter = function (params) { - console.log(params); - return 'Test'; + const index = params[0].dataIndex; + const data = [...emotionData.emotionData[index]].sort((a, b) => b.emotionValue - a.emotionValue); + let tooltipContent = 'Emotion Change '+ (index + 1) + '
'; + for (const emotion of data) { + const emotionType = emotionData.emotionTypes.find(e => e.id === emotion.emotionType); + tooltipContent += '' + emotionType.name + ': ' + emotion.emotionValue.toFixed(2) + '
'; + } + return tooltipContent; }; window.graphVizHandler.createLineChart( From 20c494dd39e5e4b5c566127168361587f9f26b60 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:26:09 +0200 Subject: [PATCH 053/114] Enhance tooltip formatting and implement smooth scrolling for emotion elements in document reader --- uce.portal/resources/templates/js/documentReader.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 5939e145..41896212 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1432,7 +1432,7 @@ function renderEmotionDevelopment(containerId) { const tooltipFormatter = function (params) { const index = params[0].dataIndex; const data = [...emotionData.emotionData[index]].sort((a, b) => b.emotionValue - a.emotionValue); - let tooltipContent = 'Emotion Change '+ (index + 1) + '
'; + let tooltipContent = 'Emotion Change ' + (index + 1) + '
'; for (const emotion of data) { const emotionType = emotionData.emotionTypes.find(e => e.id === emotion.emotionType); tooltipContent += '' + emotionType.name + ': ' + emotion.emotionValue.toFixed(2) + '
'; @@ -1446,7 +1446,15 @@ function renderEmotionDevelopment(containerId) { chartConfig, tooltipFormatter, function (params) { - // do nothing + const index = params.dataIndex; + const elementId = 'emot-E-' + (index + 1); + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } } ); From 8c6094c657e0b9a7f2e35437673c9b0b25e0bc66 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:35:30 +0200 Subject: [PATCH 054/114] Enhance tooltip content in document reader to include emotion segment details and associated content --- uce.portal/resources/templates/js/documentReader.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 41896212..9f136834 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1432,11 +1432,18 @@ function renderEmotionDevelopment(containerId) { const tooltipFormatter = function (params) { const index = params[0].dataIndex; const data = [...emotionData.emotionData[index]].sort((a, b) => b.emotionValue - a.emotionValue); - let tooltipContent = 'Emotion Change ' + (index + 1) + '
'; + let tooltipContent = 'Emotion Segment ' + (index + 1) + '
'; for (const emotion of data) { const emotionType = emotionData.emotionTypes.find(e => e.id === emotion.emotionType); tooltipContent += '' + emotionType.name + ': ' + emotion.emotionValue.toFixed(2) + '
'; } + const elementId = 'emot-E-' + (index + 1); + const element = document.getElementById(elementId); + if (element) { + tooltipContent += 'Content:
'; + const content = $(element).attr('data-wcovered'); + tooltipContent += '' + content + ''; + } return tooltipContent; }; From 762f1d2d0897e7a78854a681315b112481e203ff Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:48:49 +0200 Subject: [PATCH 055/114] Add emotionId to EmotionEntry and update tooltip element IDs in document reader --- uce.portal/resources/templates/js/documentReader.js | 5 +++-- .../services/PostgresqlDataInterface_Impl.java | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 9f136834..88dfd1b5 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1437,7 +1437,7 @@ function renderEmotionDevelopment(containerId) { const emotionType = emotionData.emotionTypes.find(e => e.id === emotion.emotionType); tooltipContent += '' + emotionType.name + ': ' + emotion.emotionValue.toFixed(2) + '
'; } - const elementId = 'emot-E-' + (index + 1); + const elementId = 'emot-E-' + data[0].emotionId; const element = document.getElementById(elementId); if (element) { tooltipContent += 'Content:
'; @@ -1454,7 +1454,8 @@ function renderEmotionDevelopment(containerId) { tooltipFormatter, function (params) { const index = params.dataIndex; - const elementId = 'emot-E-' + (index + 1); + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; const element = document.getElementById(elementId); if (element) { element.scrollIntoView({behavior: 'smooth', block: 'center'}); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index a0f322e4..ce04b2e7 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1907,6 +1907,7 @@ public List getLemmaByPage(long documentId) throws DatabaseOperationEx public List getDocumentEmotionsOrdered(long documentId) throws DatabaseOperationException { class EmotionEntry { + long emotionId; long emotionType; double emotionValue; } @@ -1937,6 +1938,7 @@ class EmotionEntry { } EmotionEntry entry = new EmotionEntry(); + entry.emotionId = emotionId; entry.emotionType = emotionType; entry.emotionValue = emotionValue; @@ -1946,7 +1948,8 @@ class EmotionEntry { .map(emotionMap::get) .map(e -> e.stream().map(entry -> Map.of( "emotionType", entry.emotionType, - "emotionValue", entry.emotionValue + "emotionValue", entry.emotionValue, + "emotionId", entry.emotionId )).toArray()) .toList(); }); From e05b246dbb785422bfa0390985cb2236d643d1d4 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 20:50:10 +0200 Subject: [PATCH 056/114] Sort emotion types alphabetically in document reader for consistent color assignment --- uce.portal/resources/templates/js/documentReader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 88dfd1b5..1c5662a3 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1397,7 +1397,7 @@ function renderEmotionDevelopment(containerId) { } const emotionColors = {}; - emotionData.emotionTypes.forEach((emotionType, index) => { + [...emotionData.emotionTypes].sort((a, b) => a.name.localeCompare(b.name)).forEach((emotionType, index) => { const hue = (index * 360 / emotionData.emotionTypes.length) % 360; emotionColors[emotionType.id] = 'hsl(' + hue + ', 70%, 50%)'; }); From 929a45d6a3202aba0905fc0514d04da3fe470408 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 21:06:32 +0200 Subject: [PATCH 057/114] Refactor CSS for improved formatting and add highlight functionality for emotion elements in document reader --- .../templates/css/document-reader.css | 55 ++++++++++++------- .../resources/templates/js/documentReader.js | 17 +++++- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 5536eec4..56512fa8 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -141,21 +141,21 @@ body { .header-btn:hover { background-color: lightgray; transition: 0.15s; - color:white; + color: white; } .document-content { padding: 2rem 4.5rem; } -.document-content *{ - font-family: Courier !important; +.document-content * { + font-family: Courier !important; word-break: break-word; } .document-content p, .document-content label, -.document-content span{ +.document-content span { font-size: 16px; } @@ -163,8 +163,8 @@ body { .document-content h4, .document-content h3, .document-content h2, -.document-content h1{ - color:rgba(0, 0, 0, 0.6); +.document-content h1 { + color: rgba(0, 0, 0, 0.6); font-weight: bold; } @@ -352,9 +352,9 @@ body { display: inline-flex; align-items: center; justify-content: center; - min-width: 20px; /* minimum size */ + min-width: 20px; /* minimum size */ height: 20px; - padding: 0 4px; /* horizontal padding */ + padding: 0 4px; /* horizontal padding */ border-radius: 50%; border: 1px solid #333; font-size: 14px; @@ -369,9 +369,9 @@ body { display: inline-flex; align-items: center; justify-content: center; - min-width: 20px; /* minimum size */ + min-width: 20px; /* minimum size */ height: 20px; - padding: 0 4px; /* horizontal padding */ + padding: 0 4px; /* horizontal padding */ border-radius: 20%; border: 1px solid #333; font-size: 14px; @@ -382,6 +382,12 @@ body { margin-left: 4px; } +.document-content .emotion-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + .side-bar { width: 500px; transition: 0.5s; @@ -490,27 +496,27 @@ body { font-size: 0.7rem; color: #333; font-weight: 600; - box-shadow: 0 2px 4px rgba(0,0,0,0.1); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); transition: transform 0.2s ease, box-shadow 0.2s ease; text-align: center; width: 100%; white-space: normal; word-wrap: break-word; hyphens: auto; - border: 1px solid rgba(0,0,0,0.1); + border: 1px solid rgba(0, 0, 0, 0.1); min-height: 32px; } .side-bar-content .document-topics-list .topic-tag:hover { transform: translateY(-2px); - box-shadow: 0 4px 8px rgba(0,0,0,0.15); + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.15); cursor: pointer; } .side-bar-content .document-topics-list .topic-tag.active-topic { transform: translateY(-2px); - box-shadow: 0 4px 8px rgba(0,0,0,0.2); - border: 2px solid rgba(0,0,0,0.3); + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + border: 2px solid rgba(0, 0, 0, 0.3); font-weight: 700; opacity: 1.2; } @@ -588,14 +594,14 @@ body { --c: #ffffff; padding: 1em; - border-radius: var(--r)/var(--r) min(var(--r),var(--p) - var(--b)/2) min(var(--r),100% - var(--p) - var(--b)/2) var(--r); - clip-path: polygon(100% 100%,0 100%,0 0,100% 0, - 100% max(0% ,var(--p) - var(--b)/2), + border-radius: var(--r)/var(--r) min(var(--r), var(--p) - var(--b) / 2) min(var(--r), 100% - var(--p) - var(--b) / 2) var(--r); + clip-path: polygon(100% 100%, 0 100%, 0 0, 100% 0, + 100% max(0%, var(--p) - var(--b) / 2), calc(100% + var(--h)) var(--p), - 100% min(100%,var(--p) + var(--b)/2)); + 100% min(100%, var(--p) + var(--b) / 2)); background: var(--c); border-image: conic-gradient(var(--c) 0 0) fill 0/ - calc(var(--p) - var(--b)/2) 0 calc(100% - var(--p) - var(--b)/2) var(--r)/ + calc(var(--p) - var(--b) / 2) 0 calc(100% - var(--p) - var(--b) / 2) var(--r)/ 0 var(--h) 0 0; } @@ -644,6 +650,7 @@ body { .tab-content .tab-pane.active { display: block; } + .side-bar.visualization-expanded { width: 150vw !important; transition: width 0.3s ease; @@ -655,6 +662,7 @@ body { height: 100%; position: relative; } + .visualization-wrapper .visualization-content { flex: 1; overflow-y: auto; @@ -673,13 +681,16 @@ body { width: 100%; } + .visualization-wrapper .visualization-spinner__icon { margin-bottom: 16px; } + .visualization-wrapper .visualization-spinner__icon i { font-size: 2.5rem; color: #0078d4; } + .visualization-wrapper .visualization-spinner__text { font-size: 1.1rem; color: #444; @@ -691,6 +702,7 @@ body { .visualization-content .viz-panel { display: none; } + .viz-panel.active { display: block; } @@ -707,7 +719,7 @@ body { display: flex; justify-content: space-around; border-radius: 24px; - box-shadow: 0 4px 24px rgba(0,0,0,0.12); + box-shadow: 0 4px 24px rgba(0, 0, 0, 0.12); background: #fff; border: 1px solid #e0e0e0; padding: 0.5rem 1.5rem; @@ -726,6 +738,7 @@ body { color: #555; transition: color 0.2s, background 0.2s; } + .viz-nav-btn.active { color: #007bff; background: none; diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 1c5662a3..0d03db55 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1464,7 +1464,22 @@ function renderEmotionDevelopment(containerId) { $('.scrollbar-minimap').hide(); } } - ); + ).then(chart=> { + chart.getInstance().on('mouseover', function (params) { + // remove previous highlights to be safe + $('.emotion-covered.highlight').removeClass('highlight'); + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.addClass('highlight'); + }); + chart.getInstance().on('mouseout', function (params) { + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.removeClass('highlight'); + }); + }); container.classList.add('rendered'); }).catch(() => { From e69f8335b59cbe860789c70761ee6941ee157d83 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 10 Jul 2025 21:08:57 +0200 Subject: [PATCH 058/114] Add mouseout event to remove highlight from emotion elements in document reader --- uce.portal/resources/templates/js/documentReader.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 0d03db55..f7114252 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1479,6 +1479,9 @@ function renderEmotionDevelopment(containerId) { const element = $('#' + elementId); element.removeClass('highlight'); }); + $(container).on('mouseout', function () { + $('.emotion-covered.highlight').removeClass('highlight'); + }); }); container.classList.add('rendered'); From 5ce91f99f20c1ef64a5b4a439bc8f097a61aa629 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 11 Jul 2025 13:21:20 +0200 Subject: [PATCH 059/114] Add emotion page association in Importer for filtered emotions --- .../src/main/java/org/texttechnologylab/Importer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 9dcb8521..b245f89c 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1000,6 +1000,12 @@ private void updateAnnotationsWithPageId(Document document, Page page, boolean i anno.setPage(page); } } + if (document.getEmotions() != null) { + for (var anno : document.getEmotions().stream().filter(t -> + (t.getBegin() >= page.getBegin() && t.getEnd() <= page.getEnd()) || (t.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } } /** From 1f1e790051878dc91980025c2c79132b3071a90c Mon Sep 17 00:00:00 2001 From: imer Date: Fri, 11 Jul 2025 13:44:11 +0200 Subject: [PATCH 060/114] correct type class --- .../java/org/texttechnologylab/models/sentiment/Sentiment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index 0d8a0a5d..c2ae6ecd 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -11,7 +11,7 @@ @Entity @Table(name = "sentiment") -@Typesystem(types = {org.texttechnologylab.annotation.SentimentBert.class}) +@Typesystem(types = {org.texttechnologylab.annotation.SentimentModel.class}) public class Sentiment extends UIMAAnnotation implements WikiModel { @Getter From ed1249b5e61b9fcbafa820acdacdda9d5db85285 Mon Sep 17 00:00:00 2001 From: imer Date: Fri, 11 Jul 2025 13:55:17 +0200 Subject: [PATCH 061/114] added sentiment loading to importer :) --- .../main/java/org/texttechnologylab/Importer.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index d37c1c69..d8b9a2d4 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -45,6 +45,7 @@ import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -1417,7 +1418,21 @@ private void setUnifiedTopic(Document document, JCas jCas) { } private void setSentiment(Document document, JCas jCas){ + List sentiments = new ArrayList<>(); + JCasUtil.select(jCas, org.texttechnologylab.annotation.SentimentModel.class).forEach(s -> { + Sentiment sentiment = new Sentiment(s.getBegin(), s.getEnd()); + sentiment.setDocument(document); + + sentiment.setSentiment(s.getSentiment()); + sentiment.setProbabilityPositive(s.getProbabilityPositive()); + sentiment.setProbabilityNegative(s.getProbabilityNegative()); + sentiment.setProbabilityNeutral(s.getProbabilityNeutral()); + + sentiment.setCoveredText(s.getCoveredText()); + sentiments.add(sentiment); + }); + document.setSentiments(sentiments); } /** From b94fbe4685426c6a2a9d9abc5aaed521a460ebdf Mon Sep 17 00:00:00 2001 From: imer Date: Fri, 11 Jul 2025 14:03:08 +0200 Subject: [PATCH 062/114] added pageid to sentiment <3 --- .../src/main/java/org/texttechnologylab/Importer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index d8b9a2d4..66097393 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -998,6 +998,12 @@ private void updateAnnotationsWithPageId(Document document, Page page, boolean i anno.setPage(page); } } + if (document.getSentiments() != null) { + for (var anno: document.getSentiments().stream().filter(s -> + (s.getBegin() >= page.getBegin() && s.getEnd() <= page.getEnd()) || (s.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } } /** From a455395418e903f01fc08fcc85a86f9b0f22b0c3 Mon Sep 17 00:00:00 2001 From: imer Date: Fri, 11 Jul 2025 14:50:45 +0200 Subject: [PATCH 063/114] added sentiment to lexicon --- uce.portal/resources/templates/css/lexicon.css | 4 ++++ .../java/org/texttechnologylab/services/LexiconService.java | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/css/lexicon.css b/uce.portal/resources/templates/css/lexicon.css index a025a286..a742ee3f 100644 --- a/uce.portal/resources/templates/css/lexicon.css +++ b/uce.portal/resources/templates/css/lexicon.css @@ -87,6 +87,10 @@ .lexicon-view .lexicon-entry[data-type="unifiedtopic"] { border-left-color: lightpink; } +.lexicon-view .lexicon-entry[data-type="sentiment"] { + /* off-white */ + border-left-color: #f8f0e3; +} .lexicon-view .lexicon-entry[data-type="geoname"] { border-left-color: dodgerblue; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java index b6cf739e..c1dc06ff 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java @@ -13,6 +13,7 @@ import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.viewModels.lexicon.LexiconOccurrenceViewModel; import org.texttechnologylab.utils.SystemStatus; @@ -47,7 +48,8 @@ public class LexiconService { Cue.class, Scope.class, XScope.class, - UnifiedTopic.class)); + UnifiedTopic.class, + Sentiment.class)); public LexiconService(PostgresqlDataInterface_Impl db) { this.db = db; From 61107667268ad3f10127db54c24949ac80a6cc36 Mon Sep 17 00:00:00 2001 From: imer Date: Fri, 11 Jul 2025 15:13:26 +0200 Subject: [PATCH 064/114] Random stuff in Web for Sentiment, WIki etc. --- .../wiki/pages/sentimentAnnotationPage.ftl | 56 +++++++++++++++++++ .../models/sentiment/Sentiment.java | 11 ++++ .../wiki/SentimentWikiPageViewModel.java | 9 +++ .../PostgresqlDataInterface_Impl.java | 8 +++ .../services/WikiService.java | 12 ++++ .../org/texttechnologylab/routes/WikiApi.java | 3 + 6 files changed, 99 insertions(+) create mode 100644 uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java diff --git a/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl new file mode 100644 index 00000000..15baabcb --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl @@ -0,0 +1,56 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ +
+ + +
+ <#assign sentiment = vm.getWikiModel()> +
+ <#list sentiment.loopThroughProperties() as propertyPair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ + +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index c2ae6ecd..2751e726 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -6,8 +6,11 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.utils.Pair; import javax.persistence.*; +import java.util.ArrayList; +import java.util.List; @Entity @Table(name = "sentiment") @@ -64,4 +67,12 @@ public void setDocument(Document document) { public String getWikiId() { return "S" + "-" + this.getId(); } + + public List> loopThroughProperties() { + return List.of( + new Pair<>("probability positive", Double.toString(probabilityPositive)), + new Pair<>("probability negative", Double.toString(probabilityNegative)), + new Pair<>("probability neutral", Double.toString(probabilityNeutral)) + ); + } } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java new file mode 100644 index 00000000..fbe01749 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java @@ -0,0 +1,9 @@ +package org.texttechnologylab.models.viewModels.wiki; + +import org.texttechnologylab.models.topic.TopicValueBase; + +import java.util.List; + +public class SentimentWikiPageViewModel extends AnnotationWikiPageViewModel { + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index dc9b4541..509c5328 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -31,6 +31,7 @@ import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.CompleteNegation; import org.texttechnologylab.models.search.*; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; @@ -1356,6 +1357,13 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public Sentiment getInitializedSentimentById(long id) throws DatabaseOperationException{ + return executeOperationSafely((session) -> { + var sent = session.get(Sentiment.class, id); + return sent; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java index 8fdb832c..eca51496 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java @@ -109,6 +109,18 @@ public UnifiedTopicWikiPageViewModel buildUnifiedTopicWikiPageViewModel(long id, return viewModel; } + public SentimentWikiPageViewModel buildSentimentWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new SentimentWikiPageViewModel(); + var sentiment = db.getInitializedSentimentById(id); + viewModel.setWikiModel(sentiment); + viewModel.setDocument(db.getDocumentById(sentiment.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Sentiment"); + + return viewModel; + } + /** * Gets a DocumentTopicDistributionWikiPageViewModel to render a Wikipage for that topic distribution */ diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java index 38f1b077..014c7163 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java @@ -165,6 +165,9 @@ public WikiApi(ApplicationContext serviceContext, Configuration freemarkerConfig } else if (type.startsWith("DTR")) { model.put("vm", wikiService.buildTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/topicPage.ftl"; + } else if (type.startsWith("S")) { + model.put("vm", wikiService.buildSentimentWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/sentimentAnnotationPage.ftl"; } else { // The type part of the wikiId was unknown. Throw an error. logger.warn("Someone tried to query a wiki page of a type that does not exist in UCE. This shouldn't happen."); From e44ca3add66243515c14ec0f36df801e3ebc7b0c Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Thu, 17 Jul 2025 14:14:44 +0200 Subject: [PATCH 065/114] Synchronize getOrCreateEmotionTypeByName method to ensure thread safety --- .../services/PostgresqlDataInterface_Impl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 79d522ef..b92990cf 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1373,7 +1373,7 @@ public EmotionType getEmotionTypeByName(String name) throws DatabaseOperationExc }); } - public EmotionType getOrCreateEmotionTypeByName(String name) throws DatabaseOperationException { + public synchronized EmotionType getOrCreateEmotionTypeByName(String name) throws DatabaseOperationException { return executeOperationSafely((session) -> { var criteriaBuilder = session.getCriteriaBuilder(); var criteriaQuery = criteriaBuilder.createQuery(EmotionType.class); From 34033f6c00f83581483c53caaa3c24982ffb1955 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 13:49:24 +0200 Subject: [PATCH 066/114] Add Model and ModelVersion entities for model information management --- .../models/modelInfo/Model.java | 27 +++++++++++++++ .../models/modelInfo/ModelVersion.java | 33 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java new file mode 100644 index 00000000..2cd0538b --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java @@ -0,0 +1,27 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "model") +public class Model extends ModelBase { + + @Getter + @Setter + @Column(name = "model_name", nullable = false, unique = true) + private String modelName; + + public Model() { + // Default constructor for JPA + } + public Model(String modelName) { + this.modelName = modelName; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java new file mode 100644 index 00000000..db591759 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "model_version") +public class ModelVersion extends ModelBase { + + @Getter + @Setter + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "model_id", nullable = false) + private Model model; + + @Getter + @Setter + @Column(name = "version", nullable = false) + private String version; + + public ModelVersion() { + // Default constructor for JPA + } + + public ModelVersion(Model model, String version) { + this.model = model; + this.version = version; + } + +} From 731299898f3bca32ebba8cfc8bc8b5509a1274ba Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 13:51:07 +0200 Subject: [PATCH 067/114] Add Model and ModelVersion classes to Hibernate configuration --- .../java/org/texttechnologylab/config/HibernateConf.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..940abe6d 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -16,6 +16,8 @@ import org.texttechnologylab.models.gbif.GbifOccurrence; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; @@ -80,6 +82,9 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // model info + metadataSources.addAnnotatedClass(Model.class); + metadataSources.addAnnotatedClass(ModelVersion.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From d1df871f3fc953875d22a48757d5e5e75df52103 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 13:53:29 +0200 Subject: [PATCH 068/114] Add unique constraint to ModelVersion table for model_id and version --- .../org/texttechnologylab/models/modelInfo/ModelVersion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java index db591759..e189c680 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java @@ -7,7 +7,7 @@ import javax.persistence.*; @Entity -@Table(name = "model_version") +@Table(name = "model_version", uniqueConstraints = @UniqueConstraint(columnNames = {"model_id", "version"})) public class ModelVersion extends ModelBase { @Getter From 0c225f6edb6d5b69fb6f93e57792382bf2c01b6d Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 13:56:48 +0200 Subject: [PATCH 069/114] Add methods for managing Model and ModelVersion entities --- .../PostgresqlDataInterface_Impl.java | 175 +++++++++++------- 1 file changed, 113 insertions(+), 62 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index a7482171..1e1baa06 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -20,7 +20,6 @@ import org.texttechnologylab.models.biofid.GazetteerTaxon; import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; -import org.texttechnologylab.models.corpus.Time; import org.texttechnologylab.models.corpus.links.*; import org.texttechnologylab.models.dto.UCEMetadataFilterDto; import org.texttechnologylab.models.dto.map.MapClusterDto; @@ -29,6 +28,8 @@ import org.texttechnologylab.models.globe.GlobeTaxon; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.CompleteNegation; import org.texttechnologylab.models.search.*; import org.texttechnologylab.models.topic.TopicValueBase; @@ -43,7 +44,10 @@ import javax.persistence.criteria.Order; import javax.persistence.criteria.Path; import javax.persistence.criteria.Predicate; -import java.sql.*; +import java.sql.Array; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.*; import java.util.function.Function; @@ -434,12 +438,13 @@ public List getGlobeDataForDocument(long documentId) throws Database () -> getAllLinksOfLinkable(taxon.getId(), taxon.getClass(), List.of(AnnotationLink.class)) .stream() .filter(l -> l.getLinkId().equals("context") && l.getToAnnotationType().equals(GeoName.class.getName())).toList(), - (ex) -> { }); + (ex) -> { + }); // Foreach taxa, fetch a possible geoname link. if (links != null) for (var link : links) { var geoname = doc.getGeoNames().stream().filter(g -> g.getId() == link.getToId()).findFirst(); - if(geoname.isEmpty()) continue; + if (geoname.isEmpty()) continue; var globeTaxon = new GlobeTaxon(); globeTaxon.setLongitude(geoname.get().getLongitude()); globeTaxon.setLatitude(geoname.get().getLatitude()); @@ -981,7 +986,7 @@ public DocumentSearchResult defaultSearchForDocuments(int skip, return executeOperationSafely((session) -> session.doReturningWork((connection) -> { DocumentSearchResult search = null; try (var storedProcedure = connection.prepareCall("{call uce_search_layer_" + layer.name().toLowerCase() + - "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { + "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { storedProcedure.setInt(1, (int) corpusId); storedProcedure.setArray(2, connection.createArrayOf("text", searchTokens.stream().map(this::escapeSql).toArray())); storedProcedure.setString(3, ogSearchQuery); @@ -1200,9 +1205,9 @@ public List getDistinctTimesByCondition(String condition, long corpusId, return executeOperationSafely((session) -> { // Construct HQL dynamically (THIS IS UNSAFE BECAUSE OF THE CONDITION INSERTION) String hql = "SELECT DISTINCT t.coveredText " + - "FROM Time t " + - "JOIN Document d ON t.documentId = d.id " + - "WHERE " + condition + " AND d.corpusId = :corpusId"; + "FROM Time t " + + "JOIN Document d ON t.documentId = d.id " + + "WHERE " + condition + " AND d.corpusId = :corpusId"; var query = session.createQuery(hql, String.class); query.setParameter("corpusId", corpusId); @@ -1393,13 +1398,59 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera return executeOperationSafely((session) -> { var topic = session.get(UnifiedTopic.class, id); Hibernate.initialize(topic.getTopics()); - for(var t:topic.getTopics()){ + for (var t : topic.getTopics()) { Hibernate.initialize(t.getWords()); } return topic; }); } + public synchronized Model getOrCreateModel(String modelName) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(Model.class); + var root = criteriaQuery.from(Model.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), modelName)); + + List models = session.createQuery(criteriaQuery).getResultList(); + if (models.isEmpty()) { + Model newModel = new Model(modelName); + session.save(newModel); + return newModel; + } else { + return models.getFirst(); + } + }); + } + + public synchronized ModelVersion getOrCreateModelVersion(Model model, String version) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ModelVersion.class); + var root = criteriaQuery.from(ModelVersion.class); + criteriaQuery.select(root).where( + criteriaBuilder.and( + criteriaBuilder.equal(root.get("model"), model), + criteriaBuilder.equal(root.get("version"), version) + ) + ); + + List versions = session.createQuery(criteriaQuery).getResultList(); + if (versions.isEmpty()) { + ModelVersion newVersion = new ModelVersion(model, version); + session.save(newVersion); + return newVersion; + } else { + return versions.getFirst(); + } + }); + } + + public ModelVersion getOrCreateModelVersion(String modelName, String version) throws DatabaseOperationException { + Model model = getOrCreateModel(modelName); + return getOrCreateModelVersion(model, version); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); @@ -1641,9 +1692,9 @@ public List getTopTopicsByDocument(long documentId, int limit) throws return executeOperationSafely((session) -> { // Use native SQL to query the document_topics_raw table String sql = "SELECT topiclabel, thetadt FROM documenttopicsraw " + - "WHERE document_id = :documentId " + - "ORDER BY thetadt DESC " + - "LIMIT :limit"; + "WHERE document_id = :documentId " + + "ORDER BY thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId) @@ -1657,9 +1708,9 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws return executeOperationSafely((session) -> { // Direct query using sentence_id String sql = "SELECT topiclabel, thetast FROM sentencetopics " + - "WHERE sentence_id = :sentenceId " + - "ORDER BY thetast DESC " + - "LIMIT :limit"; + "WHERE sentence_id = :sentenceId " + + "ORDER BY thetast DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("sentenceId", sentenceId) @@ -1672,12 +1723,12 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws public List getTopDocumentsByTopicLabel(String topicValue, long corpusId, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT d.id, d.documentid, dtr.thetadt " + - "FROM document d " + - "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + - "WHERE dtr.topiclabel = :topicValue " + - "AND d.corpusid = :corpusId " + - "ORDER BY dtr.thetadt DESC " + - "LIMIT :limit"; + "FROM document d " + + "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + + "WHERE dtr.topiclabel = :topicValue " + + "AND d.corpusid = :corpusId " + + "ORDER BY dtr.thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("topicValue", topicValue) @@ -1691,10 +1742,10 @@ public List getTopDocumentsByTopicLabel(String topicValue, long corpus public List getTopicWordsByTopicLabel(String topicValue, long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, probability " + - "FROM corpustopicwords " + - "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + - "ORDER BY probability DESC " + - "LIMIT 20"; + "FROM corpustopicwords " + + "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + + "ORDER BY probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("topicValue", topicValue); @@ -1731,12 +1782,12 @@ public List getSimilarTopicsbyTopicLabel(String topicValue, long corpu public List getNormalizedTopicWordsForCorpus(long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, " + - "AVG(probability) AS avg_probability, " + - "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + - "FROM corpustopicwords " + - "WHERE corpus_id = :corpusId " + - "GROUP BY word " + - "ORDER BY normalized_probability DESC"; + "AVG(probability) AS avg_probability, " + + "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + + "FROM corpustopicwords " + + "WHERE corpus_id = :corpusId " + + "GROUP BY word " + + "ORDER BY normalized_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("corpusId", corpusId); @@ -1783,11 +1834,11 @@ FROM get_normalized_topic_scores(:corpusId) public List getDocumentWordDistribution(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY word " + - "ORDER BY avg_probability DESC " + - "LIMIT 20"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY word " + + "ORDER BY avg_probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); @@ -1818,25 +1869,25 @@ public List getDocumentWordDistribution(long documentId) throws Datab public List getSimilarDocumentbyDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "WITH sourcewords AS (" + - " SELECT word " + - " FROM documenttopicwords " + - " WHERE document_id = :documentId " + - " GROUP BY word" + - "), " + - "similardocs AS (" + - " SELECT " + - " dtw.document_id, " + - " COUNT(DISTINCT dtw.word) AS sharedwords " + - " FROM documenttopicwords dtw " + - " JOIN sourcewords sw ON sw.word = dtw.word " + - " WHERE dtw.document_id != :documentId " + - " GROUP BY dtw.document_id " + - " ORDER BY sharedwords DESC" + - ") " + - "SELECT d.documentid, s.sharedwords " + - "FROM similardocs s " + - "JOIN document d ON s.document_id = d.id " + - "LIMIT 20"; + " SELECT word " + + " FROM documenttopicwords " + + " WHERE document_id = :documentId " + + " GROUP BY word" + + "), " + + "similardocs AS (" + + " SELECT " + + " dtw.document_id, " + + " COUNT(DISTINCT dtw.word) AS sharedwords " + + " FROM documenttopicwords dtw " + + " JOIN sourcewords sw ON sw.word = dtw.word " + + " WHERE dtw.document_id != :documentId " + + " GROUP BY dtw.document_id " + + " ORDER BY sharedwords DESC" + + ") " + + "SELECT d.documentid, s.sharedwords " + + "FROM similardocs s " + + "JOIN document d ON s.document_id = d.id " + + "LIMIT 20"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -1862,7 +1913,7 @@ public List getTaxonValuesAndCountByPageId(long documentId) throws Dat // Outer query: group by page_id, aggregate values and count String finalSql = "SELECT page_id, valuee AS taxon_value " + - "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; + "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; var query = session.createNativeQuery(finalSql) .setParameter("documentId", documentId) @@ -1878,8 +1929,8 @@ public List getNamedEntityValuesAndCountByPage(long documentId) throws return executeOperationSafely((session) -> { // Construct the SQL query to select page_id and coveredtext from the namedentity table String sql = "SELECT ne.page_id, ne.coveredtext AS named_entity_value, ne.typee AS named_entity_type " + - "FROM namedentity ne " + - "WHERE ne.document_id = :documentId"; + "FROM namedentity ne " + + "WHERE ne.document_id = :documentId"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -2000,10 +2051,10 @@ entities_in_sentences AS ( public List getTopicWordsByDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT topiclabel, word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY topiclabel, word " + - "ORDER BY avg_probability DESC"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY topiclabel, word " + + "ORDER BY avg_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); From a62facf62fa47b32e4c5db7fb88eb7761570a220 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 14:00:45 +0200 Subject: [PATCH 070/114] Add modelVersion field to UIMAAnnotation for model information linkage --- .../org/texttechnologylab/models/UIMAAnnotation.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index 96f53ca3..5fee44cb 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -1,21 +1,18 @@ package org.texttechnologylab.models; -import io.micrometer.common.lang.Nullable; import lombok.Getter; import lombok.Setter; -import org.texttechnologylab.models.biofid.BiofidTaxon; import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.corpus.links.AnnotationLink; import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; -import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.utils.StringUtils; import javax.persistence.*; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; @MappedSuperclass public class UIMAAnnotation extends ModelBase implements Linkable { @@ -51,6 +48,12 @@ public long getPrimaryDbIdentifier() { @Column(name = "page_id", insertable = false, updatable = false) private Long pageId; + @Getter + @Setter + @ManyToOne(fetch = FetchType.LAZY, optional = true) + @JoinColumn(name = "model_version_id", nullable = true) + private ModelVersion modelVersion; + public String getCoveredText() { if (coveredText == null) { return ""; // Or return an empty string "" if that's preferred From 56b27e3be5639740df96bb3613eab0e3205d482a Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 18 Jul 2025 14:20:58 +0200 Subject: [PATCH 071/114] Rename modelName field to name in Model class --- .../org/texttechnologylab/models/modelInfo/Model.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java index 2cd0538b..b9bd4608 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java @@ -14,14 +14,15 @@ public class Model extends ModelBase { @Getter @Setter - @Column(name = "model_name", nullable = false, unique = true) - private String modelName; + @Column(name = "name", nullable = false, unique = true) + private String name; public Model() { // Default constructor for JPA } - public Model(String modelName) { - this.modelName = modelName; + + public Model(String name) { + this.name = name; } } From 3f75d383cd5bd5f9f7be4e2c354a7799a652e0d4 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 16:24:17 +0200 Subject: [PATCH 072/114] Add ModelCategory class for categorizing models --- .../models/modelInfo/ModelCategory.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java new file mode 100644 index 00000000..474dca0c --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java @@ -0,0 +1,28 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "model_category") +public class ModelCategory extends ModelBase { + + @Getter + @Setter + @Column(name = "category_name", nullable = false, unique = true) + private String categoryName; + + public ModelCategory() { + // Default constructor for JPA + } + + public ModelCategory(String categoryName) { + this.categoryName = categoryName; + } + +} From 8c70e1668ce77c8685334276c6a9f10180a8421c Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 16:47:23 +0200 Subject: [PATCH 073/114] Add many-to-many relationship between Model and ModelCategory --- .../org/texttechnologylab/models/modelInfo/Model.java | 10 ++++++++++ .../models/modelInfo/ModelCategory.java | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java index b9bd4608..ff572529 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java @@ -2,11 +2,15 @@ import lombok.Getter; import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.models.ModelBase; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.ManyToMany; import javax.persistence.Table; +import java.util.Set; @Entity @Table(name = "model") @@ -17,6 +21,12 @@ public class Model extends ModelBase { @Column(name = "name", nullable = false, unique = true) private String name; + @Getter + @Setter + @ManyToMany(mappedBy = "models") + @Fetch(FetchMode.JOIN) + private Set categories; + public Model() { // Default constructor for JPA } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java index 474dca0c..37c6f68d 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java @@ -2,11 +2,15 @@ import lombok.Getter; import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.models.ModelBase; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.ManyToMany; import javax.persistence.Table; +import java.util.Set; @Entity @Table(name = "model_category") @@ -17,6 +21,12 @@ public class ModelCategory extends ModelBase { @Column(name = "category_name", nullable = false, unique = true) private String categoryName; + @Getter + @Setter + @ManyToMany(mappedBy = "categories") + @Fetch(FetchMode.JOIN) + private Set models; + public ModelCategory() { // Default constructor for JPA } From 470475a150534fd8e3836a2e113ea6acc0fd9a65 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 16:47:40 +0200 Subject: [PATCH 074/114] Add ModelCategory class to Hibernate configuration --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 940abe6d..68b9e5b2 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -17,6 +17,7 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.TopicValueBase; @@ -85,6 +86,7 @@ public static SessionFactory buildSessionFactory() { // model info metadataSources.addAnnotatedClass(Model.class); metadataSources.addAnnotatedClass(ModelVersion.class); + metadataSources.addAnnotatedClass(ModelCategory.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From d4b48a85204592e04eb3f077d955df5b275f2f5e Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 16:54:15 +0200 Subject: [PATCH 075/114] Add methods for managing ModelCategory associations in PostgresqlDataInterface_Impl --- .../PostgresqlDataInterface_Impl.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 1e1baa06..ddba7dc5 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -29,6 +29,7 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.CompleteNegation; import org.texttechnologylab.models.search.*; @@ -1451,6 +1452,38 @@ public ModelVersion getOrCreateModelVersion(String modelName, String version) th return getOrCreateModelVersion(model, version); } + public synchronized ModelCategory getOrCreateModelCategory(String categoryName) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ModelCategory.class); + var root = criteriaQuery.from(ModelCategory.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("categoryName"), categoryName)); + + List categories = session.createQuery(criteriaQuery).getResultList(); + if (categories.isEmpty()) { + ModelCategory newCategory = new ModelCategory(categoryName); + session.save(newCategory); + return newCategory; + } else { + return categories.getFirst(); + } + }); + } + + public synchronized void registerModelToCategoryAssociation(Model model, ModelCategory category) throws DatabaseOperationException { + executeOperationSafely((session) -> { + // Add the association + model.getCategories().add(category); + category.getModels().add(model); + + // Save the changes + session.saveOrUpdate(model); + session.saveOrUpdate(category); + return null; + }); + } + + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); From 9ebac77bdb132a2faa2c3d9019036a37ea7e884d Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:11:37 +0200 Subject: [PATCH 076/114] fix update Model class --- .../main/java/org/texttechnologylab/models/modelInfo/Model.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java index ff572529..38ffc831 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java @@ -23,7 +23,7 @@ public class Model extends ModelBase { @Getter @Setter - @ManyToMany(mappedBy = "models") + @ManyToMany @Fetch(FetchMode.JOIN) private Set categories; From 0ea356c34cd16e4481aba0ae02ce7463f45d723d Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:11:43 +0200 Subject: [PATCH 077/114] Ensure managed entities in registerModelToCategoryAssociation method --- .../services/PostgresqlDataInterface_Impl.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index ddba7dc5..8d0b60a0 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1472,13 +1472,18 @@ public synchronized ModelCategory getOrCreateModelCategory(String categoryName) public synchronized void registerModelToCategoryAssociation(Model model, ModelCategory category) throws DatabaseOperationException { executeOperationSafely((session) -> { - // Add the association - model.getCategories().add(category); - category.getModels().add(model); - + // First, ensure the model and category are managed entities + Model managedModel = session.get(Model.class, model.getId()); + ModelCategory managedCategory = session.get(ModelCategory.class, category.getId()); + if (managedModel == null || managedCategory == null) { + throw new IllegalArgumentException("Model or Category not found in the database."); + } + // Add the category to the model's categories + managedModel.getCategories().add(managedCategory); + // Add the model to the category's models + managedCategory.getModels().add(managedModel); // Save the changes - session.saveOrUpdate(model); - session.saveOrUpdate(category); + session.saveOrUpdate(managedModel); return null; }); } From ec2fc90f46ecc51df6418bffd8d2705a229c7924 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:14:59 +0200 Subject: [PATCH 078/114] Add NamedModel annotation for model naming consistency --- .../models/modelInfo/NamedModel.java | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java new file mode 100644 index 00000000..f90344f2 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java @@ -0,0 +1,25 @@ +package org.texttechnologylab.models.modelInfo; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to specify the name of a model. + * This can be used to annotate classes that represent models + * in the Text Technology Lab framework. + * Use this to ensure consistency with the name of the model + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface NamedModel { + + /** + * The name of the model. + * + * @return the name of the model + */ + String name(); + +} From 4c559ef141b0538849530b8cd8fb3042ca2f386b Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:17:05 +0200 Subject: [PATCH 079/114] Add getOrCreateModelCategory method to handle model category retrieval --- .../services/PostgresqlDataInterface_Impl.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 8d0b60a0..df38c407 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -8,6 +8,7 @@ import org.hibernate.criterion.Restrictions; import org.hibernate.type.LongType; import org.hibernate.type.StandardBasicTypes; +import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Service; import org.texttechnologylab.annotations.Searchable; import org.texttechnologylab.config.HibernateConf; @@ -31,6 +32,7 @@ import org.texttechnologylab.models.modelInfo.Model; import org.texttechnologylab.models.modelInfo.ModelCategory; import org.texttechnologylab.models.modelInfo.ModelVersion; +import org.texttechnologylab.models.modelInfo.NamedModel; import org.texttechnologylab.models.negation.CompleteNegation; import org.texttechnologylab.models.search.*; import org.texttechnologylab.models.topic.TopicValueBase; @@ -1470,6 +1472,14 @@ public synchronized ModelCategory getOrCreateModelCategory(String categoryName) }); } + public ModelCategory getOrCreateModelCategory(@NotNull Class modelClass) throws DatabaseOperationException { + NamedModel namedModel = modelClass.getAnnotation(NamedModel.class); + if (namedModel == null) { + throw new IllegalArgumentException("The class " + modelClass.getName() + " is not annotated with @NamedModel."); + } + return getOrCreateModelCategory(namedModel.name()); + } + public synchronized void registerModelToCategoryAssociation(Model model, ModelCategory category) throws DatabaseOperationException { executeOperationSafely((session) -> { // First, ensure the model and category are managed entities From 04324d64ab34bfe880039ad1748919915aaf9c81 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:21:22 +0200 Subject: [PATCH 080/114] Add NamedModel annotation to Emotion class for naming consistency --- .../main/java/org/texttechnologylab/models/emotion/Emotion.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java index 885bb968..30ac890a 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -8,6 +8,7 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; import javax.persistence.*; import java.util.List; @@ -15,6 +16,7 @@ @Entity @Table(name = "emotion") @Typesystem(types = {org.texttechnologylab.annotation.Emotion.class}) +@NamedModel(name = "emotion") public class Emotion extends UIMAAnnotation implements WikiModel { @Getter From 3d722de83b2f15bcd49d558de9c71311098c1b2a Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 22 Jul 2025 17:24:00 +0200 Subject: [PATCH 081/114] Add model category handling and associations in Emotion processing --- .../java/org/texttechnologylab/Importer.java | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index ba998ff3..ade32575 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -22,8 +22,14 @@ import org.texttechnologylab.annotation.AnnotationComment; import org.texttechnologylab.annotation.DocumentAnnotation; import org.texttechnologylab.annotation.geonames.GeoNamesEntity; -import org.texttechnologylab.annotation.link.*; -import org.texttechnologylab.annotation.ocr.*; +import org.texttechnologylab.annotation.link.ADLink; +import org.texttechnologylab.annotation.link.DALink; +import org.texttechnologylab.annotation.link.DLink; +import org.texttechnologylab.annotation.model.MetaData; +import org.texttechnologylab.annotation.ocr.OCRBlock; +import org.texttechnologylab.annotation.ocr.OCRLine; +import org.texttechnologylab.annotation.ocr.OCRPage; +import org.texttechnologylab.annotation.ocr.OCRToken; import org.texttechnologylab.config.CommonConfig; import org.texttechnologylab.config.CorpusConfig; import org.texttechnologylab.exceptions.DatabaseOperationException; @@ -46,6 +52,9 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.ImportStatus; import org.texttechnologylab.models.imp.LogStatus; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; @@ -55,7 +64,6 @@ import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.services.*; import org.texttechnologylab.utils.*; -import org.texttechnologylab.models.negation.CompleteNegation; import java.io.*; import java.nio.file.Files; @@ -144,13 +152,13 @@ public int getXMICountInPath() { public void start(int numThreads) throws DatabaseOperationException { logger.info( "\n _ _ _____ _____ _____ _ \n" + - "| | | / __ \\| ___| |_ _| | | \n" + - "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + - "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + - "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + - " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + - " | | \n" + - " |_|" + "| | | / __ \\| ___| |_ _| | | \n" + + "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + + "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + + "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + + " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + + " | | \n" + + " |_|" ); logger.info("===========> Global Import Id: " + importId); logger.info("===========> Importer Number: " + importerNumber); @@ -212,7 +220,7 @@ public void storeCorpusFromFolderAsync(String folderName, int numThreads) throws final var corpusConfig1 = corpusConfig; // This sucks so hard - why doesn't java just do this itself if needed? var existingCorpus = ExceptionUtils.tryCatchLog(() -> db.getCorpusByName(corpusConfig1.getName()), (ex) -> logger.error("Error getting an existing corpus by name. The corpus config should probably be changed " + - "to not add to existing corpus then.", ex)); + "to not add to existing corpus then.", ex)); if (existingCorpus != null) { // If we have the corpus, use that. Else store the new corpus. corpus = existingCorpus; @@ -472,7 +480,7 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { var exists = db.documentExists(corpus.getId(), document.getDocumentId()); if (exists) { logger.info("Document with id " + document.getDocumentId() - + " already exists in the corpus " + corpus.getId() + "."); + + " already exists in the corpus " + corpus.getId() + "."); logger.info("Checking if that document was also post-processed yet..."); var existingDoc = db.getDocumentByCorpusAndDocumentId(corpus.getId(), document.getDocumentId()); if (!existingDoc.isPostProcessed()) { @@ -829,8 +837,8 @@ private void setMetadataTitleInfo(Document document, JCas jCas, CorpusConfig cor if (documentAnnotation != null) { try { metadataTitleInfo.setPublished(documentAnnotation.getDateDay() + "." - + documentAnnotation.getDateMonth() + "." - + documentAnnotation.getDateYear()); + + documentAnnotation.getDateMonth() + "." + + documentAnnotation.getDateYear()); metadataTitleInfo.setAuthor(documentAnnotation.getAuthor()); } catch (Exception ex) { logger.warn("Tried extracting DocumentAnnotation type, it caused an error. Import will be continued as usual."); @@ -1430,11 +1438,29 @@ private void setUnifiedTopic(Document document, JCas jCas) { * Selects and sets the emotions to a document */ private void setEmotion(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Emotion.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Emotion", e); + return; + } + List emotions = new ArrayList<>(); JCasUtil.select(jCas, org.texttechnologylab.annotation.Emotion.class).forEach(e -> { Emotion emotion = new Emotion(e.getBegin(), e.getEnd()); emotion.setDocument(document); + try { + MetaData modelMetaData = e.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + emotion.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException ex) { + logger.error("Error while getting or creating model for Emotion", ex); + return; + } FSArray emotionAnnotations = e.getEmotions(); List emotionValues = new ArrayList<>(); From 36d1c5d06f43af69274dda31336e84992c078b66 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 23 Jul 2025 13:39:23 +0200 Subject: [PATCH 082/114] Implement model selection feature in document reader with UI updates and backend integration for emotions --- .../templates/css/document-reader.css | 41 ++++++ .../resources/templates/js/documentReader.js | 133 +++++++++++++----- uce.portal/resources/templates/js/graphViz.js | 114 ++++++++++++--- .../templates/reader/documentReaderView.ftl | 26 ++++ .../models/corpus/Document.java | 14 +- .../models/modelInfo/ModelNameHelper.java | 23 +++ .../PostgresqlDataInterface_Impl.java | 36 ++++- .../texttechnologylab/routes/DocumentApi.java | 33 ++++- .../main/resources/languageTranslations.json | 4 + .../public/js/visualization/echarts.js | 10 +- 10 files changed, 367 insertions(+), 67 deletions(-) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 56512fa8..e62a6793 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -521,6 +521,47 @@ body { opacity: 1.2; } +/* Sidebar Model Selection */ +.side-bar-content .model-category { + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; +} +.side-bar-content .model-category label { + font-weight: bold; + font-size: 1.1rem; + color: #333; +} +.side-bar-content .model-category select { + padding: 6px 10px; + font-size: 14px; + border-radius: 5px; + border: 1px solid #ccc; + background-color: #f9f9f9; + transition: border-color 0.2s ease-in-out; +} +.side-bar-content .model-category select:focus { + border-color: #007bff; + outline: none; +} +.side-bar-content .model-category select option { + padding: 6px 10px; + font-size: 14px; + background-color: #fff; + color: #333; +} +.side-bar-content .model-category select option:hover { + background-color: #f0f0f0; +} +.side-bar-content .model-category select option:checked { + background-color: #007bff; + color: white; +} +.side-bar-content .model-category select option:checked:hover { + background-color: #0056b3; +} + /* Scrollbar Minimap Styles */ .scrollbar-minimap { position: fixed; diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index f7114252..0e991504 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -272,6 +272,35 @@ function attachTopicClickHandlers() { dot.style.top = event.clientY - 9 + "px"; });*/ +/** + * Retrieve the model selection from the sidebar. + * @return Record + */ +function retrieveModelSelection() { + const modelSelects = $('.side-bar select.model-select'); + const modelSelection = {}; + for (let i = 0; i < modelSelects.length; i++) { + const modelSelect = $(modelSelects[i]); + const category = modelSelect.attr('model-category-name'); + const model = parseInt(modelSelect.val()); + if (category !== undefined && model !== undefined) { + modelSelection[category] = model; + } + } + return modelSelection; +} + +function modelCategoriesChanged(element) { + lazyLoadPages() + + const $element = $(element); + if ($element.attr('model-category-name') === 'emotion') { + const containerId = 'vp-6'; + const $container = $('#' + containerId); + $container.addClass('dirty'); + } +} + /** * Handle the lazy loading of more pages */ @@ -280,6 +309,12 @@ async function lazyLoadPages() { const id = $readerContainer.data('id'); const pagesCount = $readerContainer.data('pagescount'); + // clear out the existing pages + $('.reader-container .document-content').empty(); + + const modelSelection = retrieveModelSelection(); + const modelSelectionString = JSON.stringify(modelSelection); + for (let i = 0; i <= pagesCount; i += 10) { const $loadedPagesCount = $('.site-container .loaded-pages-count'); $loadedPagesCount.html(i); @@ -288,7 +323,7 @@ async function lazyLoadPages() { $loadedPagesCount.html(i); } else { await $.ajax({ - url: "/api/document/reader/pagesList?id=" + id + "&skip=" + i, + url: "/api/document/reader/pagesList?id=" + id + "&skip=" + i + "&modelSelection=" + encodeURIComponent(modelSelectionString), type: "GET", success: function (response) { // Render the new pages @@ -1379,15 +1414,18 @@ function renderSentenceTopicSankey(containerId) { function renderEmotionDevelopment(containerId) { const container = document.getElementById(containerId); - if (!container || container.classList.contains('rendered')) return; + if (!container || (container.classList.contains('rendered') && !container.classList.contains('dirty'))) return; $('.visualization-spinner').show(); const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - - const emotionReq = $.get('/api/document/page/emotionDev', {documentId: docId}); + const modelSelection = retrieveModelSelection(); + const emotionReq = $.get('/api/document/page/emotionDev', {documentId: docId, modelId: modelSelection['emotion']}); emotionReq.then(emotionData => { $('.visualization-spinner').hide(); if (!emotionData || typeof emotionData !== 'object' || !("emotionTypes" in emotionData) || !Array.isArray(emotionData.emotionTypes) || emotionData.emotionTypes.length === 0 || !("emotionData" in emotionData) || !Array.isArray(emotionData.emotionData) || emotionData.emotionData.length === 0) { + if (container.classList.contains('rendered') && container.classList.contains('dirty')) { + return; + } const container = document.getElementById(containerId); if (container) { container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; @@ -1447,44 +1485,61 @@ function renderEmotionDevelopment(containerId) { return tooltipContent; }; - window.graphVizHandler.createLineChart( - containerId, - '', - chartConfig, - tooltipFormatter, - function (params) { - const index = params.dataIndex; - const emotionId = emotionData.emotionData[index][0].emotionId; - const elementId = 'emot-E-' + emotionId; - const element = document.getElementById(elementId); - if (element) { - element.scrollIntoView({behavior: 'smooth', block: 'center'}); - clearTopicColoring(); - hideTopicNavButtons(); - $('.scrollbar-minimap').hide(); - } + if (container.classList.contains('dirty')) { + const chartId = container.getAttribute('chart-id'); + if (!chartId) { + console.error('No chart ID found for container:', containerId); + return; } - ).then(chart=> { - chart.getInstance().on('mouseover', function (params) { - // remove previous highlights to be safe - $('.emotion-covered.highlight').removeClass('highlight'); - const index = params.dataIndex; - const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; - const element = $('#' + elementId); - element.addClass('highlight'); - }); - chart.getInstance().on('mouseout', function (params) { - const index = params.dataIndex; - const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; - const element = $('#' + elementId); - element.removeClass('highlight'); - }); - $(container).on('mouseout', function () { - $('.emotion-covered.highlight').removeClass('highlight'); + window.graphVizHandler.updateLineChart( + chartId, + '', + chartConfig, + tooltipFormatter + ); + container.classList.remove('dirty'); + } + else { + window.graphVizHandler.createLineChart( + containerId, + '', + chartConfig, + tooltipFormatter, + function (params) { + const index = params.dataIndex; + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } + } + ).then(chart => { + chart.getInstance().on('mouseover', function (params) { + // remove previous highlights to be safe + $('.emotion-covered.highlight').removeClass('highlight'); + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.addClass('highlight'); + }); + chart.getInstance().on('mouseout', function (params) { + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.removeClass('highlight'); + }); + $(container).on('mouseout', function () { + $('.emotion-covered.highlight').removeClass('highlight'); + }); + $(container).attr('chart-id', chart.getChartId()); }); - }); - container.classList.add('rendered'); + container.classList.add('rendered'); + } }).catch(() => { $('.visualization-spinner').hide(); const container = document.getElementById(containerId); diff --git a/uce.portal/resources/templates/js/graphViz.js b/uce.portal/resources/templates/js/graphViz.js index 686e1691..c08d70a6 100644 --- a/uce.portal/resources/templates/js/graphViz.js +++ b/uce.portal/resources/templates/js/graphViz.js @@ -132,7 +132,7 @@ var GraphVizHandler = (function () { return getColorForWeight(weight); } - GraphVizHandler.prototype.createSankeyChart = async function (target, title, linksData, nodesData,onClick = null) { + GraphVizHandler.prototype.createSankeyChart = async function (target, title, linksData, nodesData, onClick = null) { const chartId = generateUUID(); const option = { @@ -223,11 +223,10 @@ var GraphVizHandler = (function () { }] }); - } - else { + } else { const resetNodes = nodesData.map(node => ({ ...node, - label: { show: false }, + label: {show: false}, // itemStyle: { // ...(node.itemStyle || {}), // opacity: 0.5 @@ -364,8 +363,8 @@ var GraphVizHandler = (function () { data: s.data, symbol: 'circle', symbolSize: 10, - lineStyle: { width: 3, color: s.color }, - itemStyle: { color: s.color }, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, z: 2 }); }); @@ -439,14 +438,15 @@ var GraphVizHandler = (function () { data: s.data, symbol: 'circle', symbolSize: 10, - lineStyle: { width: 3, color: s.color }, - itemStyle: { color: s.color }, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, z: 2 }); }); const echart = new ECharts(target, option); this.activeCharts[chartId] = echart; + echart.setChartId(chartId); if (onClick && typeof onClick === 'function') { echart.getInstance().on('click', onClick); @@ -455,6 +455,67 @@ var GraphVizHandler = (function () { return echart; }; + GraphVizHandler.prototype.updateLineChart = function ( + chartId, + title, + config, + tooltipFormatter) { + const echart = this.getChartById(chartId); + if (!echart) { + console.warn('No chart found with ID:', chartId); + return; + } + const { + xData, + seriesData, + yLabel = 'Count' + } = config; + const option = { + title: { + text: title, + left: 'center' + }, + tooltip: { + trigger: 'axis', + enterable: true, + backgroundColor: '#fff', + borderColor: '#ccc', + borderWidth: 1, + textStyle: { + color: '#000', + fontSize: 12 + }, + formatter: tooltipFormatter + }, + legend: { + data: seriesData.map(s => s.name), + top: 'auto' + }, + xAxis: { + type: 'category', + name: 'X', + data: xData + }, + yAxis: { + type: 'value', + name: yLabel + }, + series: [] + }; + seriesData.forEach(s => { + option.series.push({ + name: s.name, + type: 'line', + data: s.data, + symbol: 'circle', + symbolSize: 10, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, + z: 2 + }); + }); + echart.updateOption(option); + } GraphVizHandler.prototype.createChordChart = async function ( target, @@ -467,7 +528,7 @@ var GraphVizHandler = (function () { const hasGraphData = data.nodes && data.links && data.categories; const option = { - title: { text: title, left: 'center' }, + title: {text: title, left: 'center'}, tooltip: { trigger: 'item', enterable: hasGraphData, @@ -489,15 +550,15 @@ var GraphVizHandler = (function () { series: hasGraphData ? [{ type: 'graph', layout: 'circular', - circular: { rotateLabel: true }, + circular: {rotateLabel: true}, data: data.nodes, links: data.links, categories: data.categories, roam: true, - label: { rotate: 90, show: true }, - itemStyle: { borderWidth: 1, borderColor: '#aaa' }, - lineStyle: { opacity: 0.5, width: 2, curveness: 0.3 }, - emphasis: { focus: 'adjacency', label: { show: true } } + label: {rotate: 90, show: true}, + itemStyle: {borderWidth: 1, borderColor: '#aaa'}, + lineStyle: {opacity: 0.5, width: 2, curveness: 0.3}, + emphasis: {focus: 'adjacency', label: {show: true}} }] : { type: 'chord', data: data.nodes, @@ -533,7 +594,7 @@ var GraphVizHandler = (function () { title, matrix, labels, - series_name= null, + series_name = null, tooltipFormatter = null, onClick = null ) { @@ -616,10 +677,10 @@ var GraphVizHandler = (function () { ) { const chartId = generateUUID(); const option = { - title: { text: '', left: 'center' }, + title: {text: '', left: 'center'}, tooltip: {}, - xAxis: { show: false, min: 'dataMin', max: 'dataMax' }, - yAxis: { show: false, min: 'dataMin', max: 'dataMax' }, + xAxis: {show: false, min: 'dataMin', max: 'dataMax'}, + yAxis: {show: false, min: 'dataMin', max: 'dataMax'}, animationDuration: 1500, animationEasingUpdate: 'quinticInOut', series: [{ @@ -635,7 +696,7 @@ var GraphVizHandler = (function () { edges: links, roam: true, symbolSize: 10, - label: { show: false }, + label: {show: false}, emphasis: { focus: 'adjacency', lineStyle: { @@ -643,7 +704,7 @@ var GraphVizHandler = (function () { } }, - itemStyle: { borderColor: '#fff', borderWidth: 1 } + itemStyle: {borderColor: '#fff', borderWidth: 1} }] }; @@ -660,6 +721,19 @@ var GraphVizHandler = (function () { return echart; }; + GraphVizHandler.prototype.getActiveCharts = function () { + return this.activeCharts; + } + + GraphVizHandler.prototype.getChartById = function (chartId) { + if (chartId in this.activeCharts) { + return this.activeCharts[chartId]; + } else { + console.warn('No chart found with ID:', chartId); + return null; + } + } + return GraphVizHandler; }()); diff --git a/uce.portal/resources/templates/reader/documentReaderView.ftl b/uce.portal/resources/templates/reader/documentReaderView.ftl index ce0ac06d..77788436 100644 --- a/uce.portal/resources/templates/reader/documentReaderView.ftl +++ b/uce.portal/resources/templates/reader/documentReaderView.ftl @@ -259,6 +259,32 @@

+ + + <#if modelCategories?has_content && modelCategories?size gt 0> +
+

${languageResource.get("modelSelection")}

+
+ <#list modelCategories as category> +
+ + +
+ +
+
+ diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index bf6ff836..7775ac26 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -20,6 +20,7 @@ import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.modelInfo.ModelNameHelper; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; @@ -347,7 +348,7 @@ public List getAllTaxa(){ * * @return */ - public List getAllAnnotations(int pagesSkip, int pagesTake) { + public List getAllAnnotations(int pagesSkip, int pagesTake, Map modelSelection) { var pagesBegin = getPages().stream().skip(pagesSkip).limit(1).findFirst().get().getBegin(); var pagesEnd = getPages().stream().skip(Math.min(pagesSkip + pagesTake, getPages().size() - 1)).limit(1).findFirst().get().getEnd(); @@ -369,7 +370,8 @@ public List getAllAnnotations(int pagesSkip, int pagesTake) { // unifiedTopics annotations.addAll(unifiedTopics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); // emotions - annotations.addAll(emotions.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); + int emotionModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Emotion.class), -1); + annotations.addAll(emotions.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> emotionModelId == -1 || e.getModelVersion().getModel().getId() == emotionModelId).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; @@ -434,4 +436,12 @@ public List getDocumentUnifiedTopicDistribution(Integer topN) { .collect(Collectors.toList()); } + public List getModelCategories() { + List categories = new ArrayList<>(); + if (!emotions.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Emotion.class)); + } + return categories; + } + } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java new file mode 100644 index 00000000..10d47da6 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java @@ -0,0 +1,23 @@ +package org.texttechnologylab.models.modelInfo; + +import org.jetbrains.annotations.NotNull; + +public class ModelNameHelper { + + private ModelNameHelper() { + // Utility class, no instantiation allowed + } + + @NotNull + public static String getModelName(@NotNull Class modelClass) { + NamedModel namedModel = modelClass.getAnnotation(NamedModel.class); + if (namedModel == null) { + throw new IllegalArgumentException("Class " + modelClass.getName() + " is not annotated with @NamedModel"); + } + String modelName = namedModel.name(); + if (modelName.isEmpty()) { + throw new IllegalArgumentException("Model name cannot be empty for class " + modelClass.getName()); + } + return modelName; + } +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index da1fff14..d8c7589f 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -2047,7 +2047,7 @@ public List getLemmaByPage(long documentId) throws DatabaseOperationEx }); } - public List getDocumentEmotionsOrdered(long documentId) throws DatabaseOperationException { + public List getDocumentEmotionsOrdered(long documentId, int modelId) throws DatabaseOperationException { class EmotionEntry { long emotionId; long emotionType; @@ -2060,11 +2060,14 @@ class EmotionEntry { ev.value AS emotionValue FROM emotion e JOIN emotion_value ev ON e.id = ev.emotion_id + JOIN model_version mv ON e.model_version_id = mv.id WHERE e.document_id = :documentId + AND mv.model_id = :modelId ORDER BY e.beginn, e.endd """; var query = session.createNativeQuery(sql) - .setParameter("documentId", documentId); + .setParameter("documentId", documentId) + .setParameter("modelId", modelId); var result = query.getResultList(); List emotionIds = new ArrayList<>(); @@ -2114,6 +2117,33 @@ public List getEmotionTypes() throws DatabaseOperationException { }); } + public List getEmotionTypesForDocument(long documentId, long modelId) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + String sql = """ + SELECT DISTINCT et.id AS emotionTypeId, et.name AS emotionTypeName + FROM emotion e + JOIN emotion_value ev ON e.id = ev.emotion_id + JOIN emotion_type et ON ev.emotion_type_id = et.id + JOIN model_version mv ON e.model_version_id = mv.id + WHERE e.document_id = :documentId + AND mv.model_id = :modelId + """; + var query = session.createNativeQuery(sql) + .setParameter("documentId", documentId) + .setParameter("modelId", modelId); + //noinspection unchecked + return query.getResultList().stream() + .map(result -> { + Object[] row = (Object[]) result; + return Map.of( + "id", ((Number) row[0]).longValue(), + "name", (String) row[1] + ); + }) + .toList(); + }); + } + public List getGeonameByPage(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT gn.page_id, gn.coveredtext AS geoname_value " + @@ -2330,6 +2360,8 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag for (var emotion : doc.getEmotions()) { Hibernate.initialize(emotion.getEmotionValues()); + Hibernate.initialize(emotion.getModelVersion()); + Hibernate.initialize(emotion.getModelVersion().getModel()); for (var ev : emotion.getEmotionValues()) { Hibernate.initialize(ev.getEmotionType()); } diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java index 09051178..9c5586b3 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java @@ -2,6 +2,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import freemarker.template.Configuration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -13,6 +15,7 @@ import org.texttechnologylab.config.CorpusConfig; import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.models.corpus.UCEMetadataValueType; +import org.texttechnologylab.models.modelInfo.ModelCategory; import org.texttechnologylab.models.search.SearchType; import org.texttechnologylab.services.PostgresqlDataInterface_Impl; import org.texttechnologylab.services.S3StorageService; @@ -186,6 +189,14 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo model.put("searchTokens", String.join("[TOKEN]", activeSearchState.getSearchTokens())); } } + + List modelCategories = new ArrayList<>(); + List modelCategoryNames = doc.getModelCategories(); + for (String categoryName : modelCategoryNames) { + ModelCategory modelCategory = db.getOrCreateModelCategory(categoryName); + modelCategories.add(modelCategory); + } + model.put("modelCategories", modelCategories); } catch (Exception ex) { logger.error("Error creating the document reader view for document with id: " + id, ex); return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); @@ -203,10 +214,24 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo if (id == null) return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); + String modelSelectionString = request.queryParams("modelSelection"); + Map modelSelection = new HashMap<>(); + if (modelSelectionString != null && !modelSelectionString.isEmpty()) { + try { + JsonObject jsonObject = JsonParser.parseString(modelSelectionString).getAsJsonObject(); + for (var entry : jsonObject.entrySet()) { + modelSelection.put(entry.getKey(), entry.getValue().getAsInt()); + } + } catch (Exception ex) { + logger.error("Error parsing model selection JSON: " + modelSelectionString, ex); + } + } + + try { var skip = Integer.parseInt(request.queryParams("skip")); var doc = db.getCompleteDocumentById(Long.parseLong(id), skip, 10); - var annotations = doc.getAllAnnotations(skip, 10); + var annotations = doc.getAllAnnotations(skip, 10, modelSelection); model.put("documentAnnotations", annotations); model.put("documentText", doc.getFullText()); model.put("documentPages", doc.getPages(10, skip)); @@ -518,6 +543,8 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo public Route getDocumentEmotionDevelopment = (request, response) -> { var documentId = ExceptionUtils.tryCatchLog(() -> Long.parseLong(request.queryParams("documentId")), (ex) -> logger.error("Error: couldn't determine the documentId for emotion development.", ex)); + String modelIdString = request.queryParams("modelId"); + int modelId = modelIdString == null ? -1 : Integer.parseInt(modelIdString); if (documentId == null) { response.status(400); @@ -526,8 +553,8 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo } try { - var emotionData = db.getDocumentEmotionsOrdered(documentId); - var emotionTypes = db.getEmotionTypes(); + var emotionData = db.getDocumentEmotionsOrdered(documentId, modelId); + var emotionTypes = db.getEmotionTypesForDocument(documentId, modelId); Map emotionDevelopment = new HashMap<>(); emotionDevelopment.put("emotionTypes", emotionTypes); emotionDevelopment.put("emotionData", emotionData); diff --git a/uce.portal/uce.web/src/main/resources/languageTranslations.json b/uce.portal/uce.web/src/main/resources/languageTranslations.json index fe109024..9549e573 100644 --- a/uce.portal/uce.web/src/main/resources/languageTranslations.json +++ b/uce.portal/uce.web/src/main/resources/languageTranslations.json @@ -451,6 +451,10 @@ "de-DE": "Wichtige Themen", "en-EN": "Key Topics" }, + "modelSelection": { + "de-DE": "Modellauswahl", + "en-EN": "Model Selection" + }, "corpusWords": { "de-DE": "Top Wörter aus dem Korpus", "en-EN": "Top words from Corpus" diff --git a/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js b/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js index 446bdd74..787a6074 100644 --- a/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js +++ b/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js @@ -23,6 +23,14 @@ class ECharts { this.option = newOption; this.render(); } + + setChartId(id) { + this.chartId = id; + } + + getChartId() { + return this.chartId; + } } -export { ECharts }; +export {ECharts}; From cd92c8d1b5939994328643c8cc627f692040b5b1 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 23 Jul 2025 13:49:26 +0200 Subject: [PATCH 083/114] fix events --- .../resources/templates/js/documentReader.js | 63 +++++++++++++++---- uce.portal/resources/templates/js/graphViz.js | 10 +-- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 0e991504..754acb88 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1497,6 +1497,43 @@ function renderEmotionDevelopment(containerId) { chartConfig, tooltipFormatter ); + const chart = window.graphVizHandler.getChartById(chartId); + if (!chart) { + console.error('Chart not found for ID:', chartId); + return; + } + // remove old event listeners + chart.getInstance().off('mouseover'); + chart.getInstance().off('mouseout'); + chart.getInstance().off('click'); + + // add new event listeners + chart.getInstance().on('mouseover', function (params) { + // remove previous highlights to be safe + $('.emotion-covered.highlight').removeClass('highlight'); + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.addClass('highlight'); + }); + chart.getInstance().on('mouseout', function (params) { + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.removeClass('highlight'); + }); + chart.getInstance().on('click', function (params) { + const index = params.dataIndex; + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } + }); container.classList.remove('dirty'); } else { @@ -1504,19 +1541,7 @@ function renderEmotionDevelopment(containerId) { containerId, '', chartConfig, - tooltipFormatter, - function (params) { - const index = params.dataIndex; - const emotionId = emotionData.emotionData[index][0].emotionId; - const elementId = 'emot-E-' + emotionId; - const element = document.getElementById(elementId); - if (element) { - element.scrollIntoView({behavior: 'smooth', block: 'center'}); - clearTopicColoring(); - hideTopicNavButtons(); - $('.scrollbar-minimap').hide(); - } - } + tooltipFormatter ).then(chart => { chart.getInstance().on('mouseover', function (params) { // remove previous highlights to be safe @@ -1532,6 +1557,18 @@ function renderEmotionDevelopment(containerId) { const element = $('#' + elementId); element.removeClass('highlight'); }); + chart.getInstance().on('click', function (params) { + const index = params.dataIndex; + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } + }); $(container).on('mouseout', function () { $('.emotion-covered.highlight').removeClass('highlight'); }); diff --git a/uce.portal/resources/templates/js/graphViz.js b/uce.portal/resources/templates/js/graphViz.js index c08d70a6..f7401364 100644 --- a/uce.portal/resources/templates/js/graphViz.js +++ b/uce.portal/resources/templates/js/graphViz.js @@ -295,8 +295,7 @@ var GraphVizHandler = (function () { target, title, config, - tooltipFormatter, - onClick = null + tooltipFormatter ) { const chartId = generateUUID(); @@ -448,10 +447,6 @@ var GraphVizHandler = (function () { this.activeCharts[chartId] = echart; echart.setChartId(chartId); - if (onClick && typeof onClick === 'function') { - echart.getInstance().on('click', onClick); - } - return echart; }; @@ -459,7 +454,8 @@ var GraphVizHandler = (function () { chartId, title, config, - tooltipFormatter) { + tooltipFormatter + ) { const echart = this.getChartById(chartId); if (!echart) { console.warn('No chart found with ID:', chartId); From c828de065e962d28ce877eea490f0af7fe87682c Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 15:18:23 +0200 Subject: [PATCH 084/114] Refactor OffensiveSpeech entity and add OffensiveSpeechType and OffensiveSpeechValue classes --- .../offensiveSpeech/OffensiveSpeech.java | 10 ++---- .../offensiveSpeech/OffensiveSpeechType.java | 27 +++++++++++++++ .../offensiveSpeech/OffensiveSpeechValue.java | 33 +++++++++++++++++++ 3 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 60243c6e..2a9023a0 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -19,13 +19,9 @@ public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { @Getter @Setter - @Column(name = "offensive", nullable = false) - private double offensive; - - @Getter - @Setter - @Column(name = "non_offensive", nullable = false) - private double nonOffensive; + @OneToMany(mappedBy = "offensiveSpeech", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + @Fetch(FetchMode.SUBSELECT) + private List offensiveSpeechValues; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java new file mode 100644 index 00000000..de77b8bd --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java @@ -0,0 +1,27 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "offensive_speech_type") +public class OffensiveSpeechType extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public OffensiveSpeechType() { + } + + public OffensiveSpeechType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java new file mode 100644 index 00000000..64bd0bd4 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "offensive_speech_value") +public class OffensiveSpeechValue extends ModelBase { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "offensive_speech_type_id", nullable = false) + private OffensiveSpeechType offensiveSpeechType; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "offensive_speech_id", nullable = false) + private OffensiveSpeech offensiveSpeech; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public OffensiveSpeechValue() { + } + +} From 52a902e4c6e0a91faee07f2dd3cc47dfd8ae323b Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 15:39:32 +0200 Subject: [PATCH 085/114] Add OffensiveSpeechType and OffensiveSpeechValue handling in Hibernate configuration and importer --- .../config/HibernateConf.java | 4 ++ .../PostgresqlDataInterface_Impl.java | 21 +++++++++ .../java/org/texttechnologylab/Importer.java | 47 +++++++++++-------- 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index e331f1aa..490cec79 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -18,6 +18,8 @@ import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -83,6 +85,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); // offensive speech metadataSources.addAnnotatedClass(OffensiveSpeech.class); + metadataSources.addAnnotatedClass(OffensiveSpeechValue.class); + metadataSources.addAnnotatedClass(OffensiveSpeechType.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index dc9b4541..03ef50ed 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -30,6 +30,7 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.CompleteNegation; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; import org.texttechnologylab.models.search.*; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; @@ -1356,6 +1357,26 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public synchronized OffensiveSpeechType getOrCreateOffensiveSpeechType(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(OffensiveSpeechType.class); + var root = criteriaQuery.from(OffensiveSpeechType.class); + criteriaQuery.select(root) + .where(criteriaBuilder.equal(root.get("name"), name)); + + OffensiveSpeechType offensiveSpeechType; + try { + offensiveSpeechType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + // If not found, create a new one + offensiveSpeechType = new OffensiveSpeechType(name); + session.save(offensiveSpeechType); + } + return offensiveSpeechType; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 2ce83cc9..62529be4 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -45,6 +45,8 @@ import org.texttechnologylab.models.imp.LogStatus; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; import org.texttechnologylab.models.topic.TopicValueBase; @@ -1430,27 +1432,32 @@ private void setOffensiveSpeech(Document document, JCas jCas) { logger.warn("OffensiveSpeech annotation without offensives found. Skipping this annotation."); return; } - if (offensives.size() != 2) { - logger.warn("OffensiveSpeech annotation with " + offensives.size() + " offensives found. Expected 2. Skipping this annotation."); - return; - } - final String offensiveKey = "Offensive"; - final String nonOffensiveKey = "Not Offensive"; - OptionalDouble offensiveScore = offensives.stream() - .filter(oc -> oc.getKey().equals(offensiveKey)) - .map(AnnotationComment::getValue) - .mapToDouble(Double::parseDouble) - .findFirst(); - OptionalDouble nonOffensiveScore = offensives.stream() - .filter(oc -> oc.getKey().equals(nonOffensiveKey)) - .map(AnnotationComment::getValue) - .mapToDouble(Double::parseDouble).findFirst(); - if (offensiveScore.isEmpty() || nonOffensiveScore.isEmpty()) { - logger.warn("OffensiveSpeech annotation without offensive or non-offensive score found. Skipping this annotation."); - return; + + List offensivesList = new ArrayList<>(); + for (AnnotationComment offensive : offensives) { + String offensiveSpeechName = offensive.getKey(); + OffensiveSpeechType offensiveSpeechType; + try { + offensiveSpeechType = db.getOrCreateOffensiveSpeechType(offensiveSpeechName); + } catch (DatabaseOperationException e) { + logger.warn("Unknown OffensiveSpeechType: " + offensiveSpeechName + ". Skipping this annotation."); + continue; + } + double value; + try { + value = Double.parseDouble(offensive.getValue()); + } + catch (NumberFormatException e) { + logger.warn("Invalid value for OffensiveSpeechType: " + offensiveSpeechName + ". Skipping this annotation."); + continue; + } + OffensiveSpeechValue offensiveSpeechValue = new OffensiveSpeechValue(); + offensiveSpeechValue.setOffensiveSpeechType(offensiveSpeechType); + offensiveSpeechValue.setValue(value); + offensiveSpeechValue.setOffensiveSpeech(offensiveSpeech); + offensivesList.add(offensiveSpeechValue); } - offensiveSpeech.setOffensive(offensiveScore.getAsDouble()); - offensiveSpeech.setNonOffensive(nonOffensiveScore.getAsDouble()); + offensiveSpeech.setOffensiveSpeechValues(offensivesList); offensiveSpeeches.add(offensiveSpeech); }); From 32be6312c784aad6b9196e2965459394b663d21d Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 17:37:12 +0200 Subject: [PATCH 086/114] Rename database table names for OffensiveSpeech, OffensiveSpeechType, and OffensiveSpeechValue to remove underscores --- .../models/offensiveSpeech/OffensiveSpeech.java | 2 +- .../models/offensiveSpeech/OffensiveSpeechType.java | 2 +- .../models/offensiveSpeech/OffensiveSpeechValue.java | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 2a9023a0..29c45045 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -13,7 +13,7 @@ import java.util.List; @Entity -@Table(name = "offensive_speech") +@Table(name = "offensivespeech") @Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java index de77b8bd..d1bbad48 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java @@ -9,7 +9,7 @@ import javax.persistence.Table; @Entity -@Table(name = "offensive_speech_type") +@Table(name = "offensivespeech_type") public class OffensiveSpeechType extends ModelBase { @Getter diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java index 64bd0bd4..15bc1f1a 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java @@ -7,19 +7,19 @@ import javax.persistence.*; @Entity -@Table(name = "offensive_speech_value") +@Table(name = "offensivespeech_value") public class OffensiveSpeechValue extends ModelBase { @Getter @Setter @ManyToOne - @JoinColumn(name = "offensive_speech_type_id", nullable = false) + @JoinColumn(name = "offensivespeech_type_id", nullable = false) private OffensiveSpeechType offensiveSpeechType; @Getter @Setter @ManyToOne - @JoinColumn(name = "offensive_speech_id", nullable = false) + @JoinColumn(name = "offensivespeech_id", nullable = false) private OffensiveSpeech offensiveSpeech; @Getter From 7d52b6612aab2a0665f53fe98629771738752305 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 17:21:40 +0200 Subject: [PATCH 087/114] Add Offensive Speech annotation in corpusAnnotations.ftl --- .../templates/corpus/components/corpusAnnotations.ftl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl index cfd27136..fdf69176 100644 --- a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl +++ b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl @@ -186,5 +186,16 @@ + +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isOffensiveSpeech() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
\ No newline at end of file From 240eccad7980b19892692cea6b63c6131580bb35 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 17:39:34 +0200 Subject: [PATCH 088/114] Add OffensiveSpeech model to LexiconService --- .../java/org/texttechnologylab/services/LexiconService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java index b6cf739e..74e967e2 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java @@ -13,6 +13,7 @@ import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.viewModels.lexicon.LexiconOccurrenceViewModel; import org.texttechnologylab.utils.SystemStatus; @@ -47,7 +48,8 @@ public class LexiconService { Cue.class, Scope.class, XScope.class, - UnifiedTopic.class)); + UnifiedTopic.class, + OffensiveSpeech.class)); public LexiconService(PostgresqlDataInterface_Impl db) { this.db = db; From 1fe22564e3f2b327c8f7c1da678591a25fef8c55 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 17:42:31 +0200 Subject: [PATCH 089/114] Add styling for Offensive Speech entries in lexicon view --- uce.portal/resources/templates/css/lexicon.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uce.portal/resources/templates/css/lexicon.css b/uce.portal/resources/templates/css/lexicon.css index a025a286..b351aefe 100644 --- a/uce.portal/resources/templates/css/lexicon.css +++ b/uce.portal/resources/templates/css/lexicon.css @@ -108,6 +108,10 @@ border-left-color: darkred; } +.lexicon-view .lexicon-entry[data-type="offensivespeech"]{ + border-left-color: silver; +} + .lexicon-view .filter-container{ width: 100%; position: sticky; From dc02c79d179ad02c04214e0987a94c5e98800af1 Mon Sep 17 00:00:00 2001 From: mirelamuza Date: Fri, 25 Jul 2025 18:17:54 +0200 Subject: [PATCH 090/114] Add Offensive Speech annotation page and view model --- .../pages/offensiveSpeechAnnotationPage.ftl | 55 +++++++++++++++++++ .../offensiveSpeech/OffensiveSpeech.java | 7 +++ ...siveSpeechAnnotationWikiPageViewModel.java | 7 +++ .../PostgresqlDataInterface_Impl.java | 13 +++++ .../services/WikiService.java | 15 +++++ .../org/texttechnologylab/routes/WikiApi.java | 3 + 6 files changed, 100 insertions(+) create mode 100644 uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java diff --git a/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl new file mode 100644 index 00000000..75c21d59 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl @@ -0,0 +1,55 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ +
+ + +
+ <#assign offensiveSpeech = vm.getWikiModel()> +
+ <#list offensiveSpeech.collectOffensiveSpeechValues() as propertyPair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 29c45045..55637393 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -8,6 +8,7 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.utils.Pair; import javax.persistence.*; import java.util.List; @@ -53,4 +54,10 @@ public String getWikiId() { return "OS" + "-" + this.getId(); } + public List> collectOffensiveSpeechValues() { + return this.offensiveSpeechValues.stream() + .map(value -> new Pair<>(value.getOffensiveSpeechType().getName(), value.getValue())) + .toList(); + } + } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java new file mode 100644 index 00000000..ea5b8c01 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java @@ -0,0 +1,7 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class OffensiveSpeechAnnotationWikiPageViewModel extends AnnotationWikiPageViewModel{ + + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 03ef50ed..93d84b0c 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -30,7 +30,9 @@ import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.CompleteNegation; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.search.*; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; @@ -1377,6 +1379,17 @@ public synchronized OffensiveSpeechType getOrCreateOffensiveSpeechType(String na }); } + public OffensiveSpeech getInitializedOffensiveSpeechById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var offensiveSpeech = session.get(OffensiveSpeech.class, id); + Hibernate.initialize(offensiveSpeech.getOffensiveSpeechValues()); + for (OffensiveSpeechValue value : offensiveSpeech.getOffensiveSpeechValues()) { + Hibernate.initialize(value.getOffensiveSpeechType()); + } + return offensiveSpeech; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java index 8fdb832c..6a002a3a 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java @@ -109,6 +109,21 @@ public UnifiedTopicWikiPageViewModel buildUnifiedTopicWikiPageViewModel(long id, return viewModel; } + /** + * Builds a view model to render a offensive speech annotation wiki page + */ + public OffensiveSpeechAnnotationWikiPageViewModel buildOffensiveSpeechAnnotationWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new OffensiveSpeechAnnotationWikiPageViewModel(); + var offensiveSpeech = db.getInitializedOffensiveSpeechById(id); + viewModel.setWikiModel(offensiveSpeech); + viewModel.setDocument(db.getDocumentById(offensiveSpeech.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("OffensiveSpeech"); + + return viewModel; + } + /** * Gets a DocumentTopicDistributionWikiPageViewModel to render a Wikipage for that topic distribution */ diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java index 38f1b077..b2557a31 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java @@ -165,6 +165,9 @@ public WikiApi(ApplicationContext serviceContext, Configuration freemarkerConfig } else if (type.startsWith("DTR")) { model.put("vm", wikiService.buildTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/topicPage.ftl"; + } else if (type.startsWith("OS")) { + model.put("vm", wikiService.buildOffensiveSpeechAnnotationWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/offensiveSpeechAnnotationPage.ftl"; } else { // The type part of the wikiId was unknown. Throw an error. logger.warn("Someone tried to query a wiki page of a type that does not exist in UCE. This shouldn't happen."); From ea4e33f52671f719da425f7d3e9dfcc4c708e612 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Fri, 25 Jul 2025 18:56:37 +0200 Subject: [PATCH 091/114] Add ToxicValue and ToxicType classes with relationships to Toxic --- .../texttechnologylab/models/toxic/Toxic.java | 30 +++++++---------- .../models/toxic/ToxicType.java | 26 +++++++++++++++ .../models/toxic/ToxicValue.java | 33 +++++++++++++++++++ 3 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index 46aa6556..3cc355e7 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -1,20 +1,27 @@ package org.texttechnologylab.models.toxic; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; import javax.persistence.*; +import java.util.List; @Entity @Table(name = "toxic") @Typesystem(types = {org.texttechnologylab.annotation.Toxic.class}) public class Toxic extends UIMAAnnotation implements WikiModel { - @Column(name = "toxic", nullable = false) - private double toxic; - @Column(name = "non_toxic", nullable = false) - private double nonToxic; + + @Getter + @Setter + @OneToMany(mappedBy = "toxic", fetch = FetchType.LAZY, cascade = CascadeType.ALL) + @Fetch(value = FetchMode.SUBSELECT) + private List toxicValues; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) @@ -33,21 +40,6 @@ public Toxic(int begin, int end, String coveredText) { setCoveredText(coveredText); } - public double getToxic() { - return toxic; - } - public void setToxic(double newToxic) { - this.toxic = newToxic; - } - - public double getNonToxic() { - return nonToxic; - } - - public void setNonToxic(double nonToxic) { - this.nonToxic = nonToxic; - } - public Document getDocument() { return document; } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java new file mode 100644 index 00000000..8db08c50 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java @@ -0,0 +1,26 @@ +package org.texttechnologylab.models.toxic; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "toxic_type") +public class ToxicType extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public ToxicType() {} + + public ToxicType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java new file mode 100644 index 00000000..756d2037 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.toxic; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "toxic_value") +public class ToxicValue extends ModelBase { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "toxic_id", nullable = false) + private Toxic toxic; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "toxic_type_id", nullable = false) + private ToxicType toxicType; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public ToxicValue() { + } + +} From 2ee136faefc6fdcb369c842fdda855d4d4839ba1 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Fri, 25 Jul 2025 18:58:58 +0200 Subject: [PATCH 092/114] Add ToxicValue and ToxicType classes to Hibernate metadata sources --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 905962ea..f9504a3f 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -22,6 +22,8 @@ import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.toxic.Toxic; +import org.texttechnologylab.models.toxic.ToxicType; +import org.texttechnologylab.models.toxic.ToxicValue; import java.util.HashMap; @@ -83,6 +85,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); // toxic metadataSources.addAnnotatedClass(Toxic.class); + metadataSources.addAnnotatedClass(ToxicValue.class); + metadataSources.addAnnotatedClass(ToxicType.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From e47c9e70660c17a20ca9c3af47eac62466db8e0a Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Fri, 25 Jul 2025 19:40:01 +0200 Subject: [PATCH 093/114] Add ToxicValue and ToxicType handling in Importer and Data Interface --- .../PostgresqlDataInterface_Impl.java | 19 +++++++++++++ .../java/org/texttechnologylab/Importer.java | 27 +++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index dc9b4541..6fb88345 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -34,6 +34,7 @@ import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.ToxicType; import org.texttechnologylab.models.util.HealthStatus; import org.texttechnologylab.utils.ReflectionUtils; import org.texttechnologylab.utils.StringUtils; @@ -1356,6 +1357,24 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public synchronized ToxicType getOrCreateToxicType(String type) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ToxicType.class); + var root = criteriaQuery.from(ToxicType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), type)); + + ToxicType toxicType; + try { + toxicType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + toxicType = new ToxicType(type); + session.save(toxicType); + } + return toxicType; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index f6eb8ce5..b47aa863 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -50,6 +50,8 @@ import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.toxic.Toxic; +import org.texttechnologylab.models.toxic.ToxicType; +import org.texttechnologylab.models.toxic.ToxicValue; import org.texttechnologylab.services.*; import org.texttechnologylab.utils.*; import org.texttechnologylab.models.negation.CompleteNegation; @@ -1426,8 +1428,29 @@ private void setToxic(Document document, JCas jCas) { Toxic toxic = new Toxic(t.getBegin(), t.getEnd()); toxic.setDocument(document); toxic.setCoveredText(t.getCoveredText()); - toxic.setToxic(t.getToxic()); - toxic.setNonToxic(t.getNonToxic()); + + List toxicValues = new ArrayList<>(); + ToxicType toxicType, nonToxicType; + try { + toxicType = db.getOrCreateToxicType("toxic"); + nonToxicType = db.getOrCreateToxicType("non-toxic"); + } catch (DatabaseOperationException e) { + logger.error("Error while fetching ToxicType or NonToxicType from database.", e); + return; + } + + ToxicValue toxicValue = new ToxicValue(); + toxicValue.setToxicType(toxicType); + toxicValue.setValue(t.getToxic()); + toxicValue.setToxic(toxic); + toxicValues.add(toxicValue); + ToxicValue nonToxicValue = new ToxicValue(); + nonToxicValue.setToxicType(nonToxicType); + nonToxicValue.setValue(t.getNonToxic()); + nonToxicValue.setToxic(toxic); + toxicValues.add(nonToxicValue); + + toxic.setToxicValues(toxicValues); toxics.add(toxic); }); From e7ac6392929155a25594a893181071602dab36ab Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Fri, 25 Jul 2025 19:41:09 +0200 Subject: [PATCH 094/114] Handle toxic annotations in Importer to set page context --- .../src/main/java/org/texttechnologylab/Importer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index b47aa863..1c6f576e 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -999,6 +999,12 @@ private void updateAnnotationsWithPageId(Document document, Page page, boolean i anno.setPage(page); } } + if (document.getToxics() != null) { + for (var anno : document.getToxics().stream().filter(t -> + (t.getBegin() >= page.getBegin() && t.getEnd() <= page.getEnd()) || (t.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } } /** From ccdd87bee8c0453b50f6e205b787d1cd8f9b3721 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Fri, 25 Jul 2025 19:50:41 +0200 Subject: [PATCH 095/114] Update condition check in document reader to use 'rendered' class instead of 'dirty' --- uce.portal/resources/templates/js/documentReader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index 754acb88..379ade22 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -1485,7 +1485,7 @@ function renderEmotionDevelopment(containerId) { return tooltipContent; }; - if (container.classList.contains('dirty')) { + if (container.classList.contains('rendered')) { const chartId = container.getAttribute('chart-id'); if (!chartId) { console.error('No chart ID found for container:', containerId); From 29a663a144e71ddd17905a06c4cd43bb7ea5c223 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Thu, 7 Aug 2025 12:18:54 +0200 Subject: [PATCH 096/114] Add toxic annotation checkbox to corpus annotations --- .../templates/corpus/components/corpusAnnotations.ftl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl index cfd27136..ffaa726e 100644 --- a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl +++ b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl @@ -186,5 +186,16 @@ + +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isToxic() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
\ No newline at end of file From 887d1a3a54926624e0b3db498bbea9c759cbb30e Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Thu, 7 Aug 2025 12:21:17 +0200 Subject: [PATCH 097/114] Add styling for toxic entries in lexicon and include Toxic model in LexiconService --- uce.portal/resources/templates/css/lexicon.css | 4 ++++ .../java/org/texttechnologylab/services/LexiconService.java | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/css/lexicon.css b/uce.portal/resources/templates/css/lexicon.css index a025a286..2575e52c 100644 --- a/uce.portal/resources/templates/css/lexicon.css +++ b/uce.portal/resources/templates/css/lexicon.css @@ -108,6 +108,10 @@ border-left-color: darkred; } +.lexicon-view .lexicon-entry[data-type="toxic"]{ + border-left-color: darkorange; +} + .lexicon-view .filter-container{ width: 100%; position: sticky; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java index b6cf739e..c2e866ec 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java @@ -14,6 +14,7 @@ import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.models.viewModels.lexicon.LexiconOccurrenceViewModel; import org.texttechnologylab.utils.SystemStatus; @@ -47,7 +48,9 @@ public class LexiconService { Cue.class, Scope.class, XScope.class, - UnifiedTopic.class)); + UnifiedTopic.class, + Toxic.class + )); public LexiconService(PostgresqlDataInterface_Impl db) { this.db = db; From 85df96df4f8c9714fcdc86ce7c94eaaec7ab7e27 Mon Sep 17 00:00:00 2001 From: Xinran Wang Date: Thu, 7 Aug 2025 14:11:23 +0200 Subject: [PATCH 098/114] Add Toxic annotation wiki page view model and rendering logic --- .../wiki/pages/toxicAnnotationPage.ftl | 55 +++++++++++++++++++ .../ToxicAnnotationWikiPageViewModel.java | 7 +++ .../PostgresqlDataInterface_Impl.java | 12 ++++ .../services/WikiService.java | 15 +++++ .../org/texttechnologylab/routes/WikiApi.java | 3 + 5 files changed, 92 insertions(+) create mode 100644 uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java diff --git a/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl new file mode 100644 index 00000000..84b4bfe7 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl @@ -0,0 +1,55 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ +
+ + +
+ <#assign toxic = vm.getWikiModel()> +
+ <#list toxic.getToxicValues() as tv> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java new file mode 100644 index 00000000..808bc432 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java @@ -0,0 +1,7 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class ToxicAnnotationWikiPageViewModel extends AnnotationWikiPageViewModel{ + + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 6fb88345..f03d0355 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -34,6 +34,7 @@ import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.models.toxic.ToxicType; import org.texttechnologylab.models.util.HealthStatus; import org.texttechnologylab.utils.ReflectionUtils; @@ -1357,6 +1358,17 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public Toxic getInitializedToxicById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var toxic = session.get(Toxic.class, id); + Hibernate.initialize(toxic.getToxicValues()); + for(var t:toxic.getToxicValues()){ + Hibernate.initialize(t.getToxicType()); + } + return toxic; + }); + } + public synchronized ToxicType getOrCreateToxicType(String type) throws DatabaseOperationException { return executeOperationSafely((session) -> { var criteriaBuilder = session.getCriteriaBuilder(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java index 8fdb832c..faf45c78 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java @@ -109,6 +109,21 @@ public UnifiedTopicWikiPageViewModel buildUnifiedTopicWikiPageViewModel(long id, return viewModel; } + /** + * Builds a view model to render a toxic annotation wiki page + */ + public ToxicAnnotationWikiPageViewModel buildToxicAnnotationWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new ToxicAnnotationWikiPageViewModel(); + var toxic = db.getInitializedToxicById(id); + viewModel.setWikiModel(toxic); + viewModel.setDocument(db.getDocumentById(toxic.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Toxic"); + + return viewModel; + } + /** * Gets a DocumentTopicDistributionWikiPageViewModel to render a Wikipage for that topic distribution */ diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java index 38f1b077..353c6604 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java @@ -162,6 +162,9 @@ public WikiApi(ApplicationContext serviceContext, Configuration freemarkerConfig } else if (type.startsWith("UT")) { model.put("vm", wikiService.buildUnifiedTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/unifiedTopicAnnotationPage.ftl"; + } else if (type.startsWith("T")) { + model.put("vm", wikiService.buildToxicAnnotationWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/toxicAnnotationPage.ftl"; } else if (type.startsWith("DTR")) { model.put("vm", wikiService.buildTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/topicPage.ftl"; From fd4ee9bd6aaf82f44b1b74374b48188da5bc6410 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 21:54:56 +0200 Subject: [PATCH 099/114] Add NamedModel annotation to OffensiveSpeech and Toxic classes --- .../models/offensiveSpeech/OffensiveSpeech.java | 2 ++ .../src/main/java/org/texttechnologylab/models/toxic/Toxic.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 55637393..550d0634 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -8,6 +8,7 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; import org.texttechnologylab.utils.Pair; import javax.persistence.*; @@ -16,6 +17,7 @@ @Entity @Table(name = "offensivespeech") @Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) +@NamedModel(name = "offensivespeech") public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { @Getter diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index 3cc355e7..bdadaac7 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -8,6 +8,7 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; import javax.persistence.*; import java.util.List; @@ -15,6 +16,7 @@ @Entity @Table(name = "toxic") @Typesystem(types = {org.texttechnologylab.annotation.Toxic.class}) +@NamedModel(name = "toxic") public class Toxic extends UIMAAnnotation implements WikiModel { @Getter From 3cbf0e472b964ce890b3ed7da77e1a2b1939af32 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 21:58:25 +0200 Subject: [PATCH 100/114] Handle model category and model version creation for OffensiveSpeech in Importer --- .../java/org/texttechnologylab/Importer.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 615b8fb3..a62714b3 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1457,12 +1457,30 @@ private void setUnifiedTopic(Document document, JCas jCas) { } private void setOffensiveSpeech(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(OffensiveSpeech.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for OffensiveSpeech", e); + return; + } + List offensiveSpeeches = new ArrayList<>(); JCasUtil.select(jCas, org.texttechnologylab.annotation.OffensiveSpeech.class).forEach(os -> { OffensiveSpeech offensiveSpeech = new OffensiveSpeech(os.getBegin(), os.getEnd()); offensiveSpeech.setDocument(document); offensiveSpeech.setCoveredText(os.getCoveredText()); + try { + MetaData modelMetaData = os.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + offensiveSpeech.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for OffensiveSpeech", e); + return; + } FSArray offensives = os.getOffensives(); if (offensives == null) { From ba50516651513ff83e698d220b2a5226e7e98c18 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 21:59:04 +0200 Subject: [PATCH 101/114] Add model category and version handling for Toxic in setToxic method --- .../java/org/texttechnologylab/Importer.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index a62714b3..261048e4 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1523,6 +1523,14 @@ private void setOffensiveSpeech(Document document, JCas jCas) { * Selects and sets the toxicities to a document. */ private void setToxic(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Toxic.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Toxic", e); + return; + } + List toxics = new ArrayList<>(); JCasUtil.select(jCas, org.texttechnologylab.annotation.Toxic.class).forEach(t -> { @@ -1530,6 +1538,17 @@ private void setToxic(Document document, JCas jCas) { toxic.setDocument(document); toxic.setCoveredText(t.getCoveredText()); + try { + MetaData modelMetaData = t.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + toxic.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for Toxic", e); + return; + } + List toxicValues = new ArrayList<>(); ToxicType toxicType, nonToxicType; try { From 98348947b10efa012991469fa1aa4b587b112674 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 22:26:51 +0200 Subject: [PATCH 102/114] Add offensive speech handling and UI markers in document processing --- .../templates/css/document-reader.css | 28 +++++++++++++++++++ .../models/UIMAAnnotation.java | 24 ++++++++++++++++ .../models/corpus/Document.java | 5 ++-- .../offensiveSpeech/OffensiveSpeech.java | 25 +++++++++++++++-- .../PostgresqlDataInterface_Impl.java | 12 ++++++++ 5 files changed, 89 insertions(+), 5 deletions(-) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index e62a6793..bbbbc17a 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -343,6 +343,11 @@ body { position: relative; } +.document-content .colorable-offensive { + transition: background-color 0.3s ease; + position: relative; +} + .document-content .animated-topic-scroll { transition: box-shadow 0.3s ease-in-out; box-shadow: 0 0 10px 5px rgba(255, 255, 0, 0.5); @@ -388,6 +393,29 @@ body { padding: 2px 4px; } +.document-content .offensive-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .offensive-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + .side-bar { width: 500px; transition: 0.5s; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index 5e301c56..3246e49a 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -9,6 +9,7 @@ import org.texttechnologylab.models.emotion.Emotion; import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.utils.StringUtils; @@ -135,6 +136,9 @@ public String buildHTMLString(List annotations, String coveredTe Map emotionMarkers = new TreeMap<>(); Map emotionCoverWrappersStart = new TreeMap<>(); Map emotionCoverWrappersEnd = new TreeMap<>(); + Map offensiveSpeechMarkers = new TreeMap<>(); + Map offensiveSpeechCoverWrappersStart = new TreeMap<>(); + Map offensiveSpeechCoverWrappersEnd = new TreeMap<>(); for (var annotation : annotations) { @@ -186,6 +190,17 @@ public String buildHTMLString(List annotations, String coveredTe continue; } + if (annotation instanceof OffensiveSpeech offensiveSpeech) { + var start = offensiveSpeech.getBegin() - offset - errorOffset; + var end = offensiveSpeech.getEnd() - offset - errorOffset; + + offensiveSpeechCoverWrappersStart.put(start, offensiveSpeech.generateOffensiveSpeechCoveredStartSpan()); + offensiveSpeechCoverWrappersEnd.put(end, ""); + + offensiveSpeechMarkers.put(end, offensiveSpeech.generateOffensiveSpeechMarker()); + continue; + } + var start = annotation.getBegin() - offset - errorOffset; var end = annotation.getEnd() - offset - errorOffset; @@ -208,6 +223,9 @@ public String buildHTMLString(List annotations, String coveredTe if (emotionCoverWrappersEnd.containsKey(i)) { finalText.append(emotionCoverWrappersEnd.get(i)); } + if (offensiveSpeechCoverWrappersEnd.containsKey(i)) { + finalText.append(offensiveSpeechCoverWrappersEnd.get(i)); + } if (endTags.containsKey(i)) { //finalText.append(endTags.get(i).getFirst()); for (var tag : endTags.get(i)) { @@ -216,6 +234,9 @@ public String buildHTMLString(List annotations, String coveredTe } // Insert start spans + if (offensiveSpeechCoverWrappersStart.containsKey(i)) { + finalText.append(offensiveSpeechCoverWrappersStart.get(i)); + } if (emotionCoverWrappersStart.containsKey(i)) { finalText.append(emotionCoverWrappersStart.get(i)); } @@ -230,6 +251,9 @@ public String buildHTMLString(List annotations, String coveredTe if (emotionMarkers.containsKey(i)) { finalText.append(emotionMarkers.get(i)); } + if (offensiveSpeechMarkers.containsKey(i)) { + finalText.append(offensiveSpeechMarkers.get(i)); + } if (startTags.containsKey(i)) { finalText.append(generateMultiHTMLTag(startTags.get(i))); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9a9553c..6ae37a25 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -23,8 +23,6 @@ import org.texttechnologylab.models.modelInfo.ModelNameHelper; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; -import org.texttechnologylab.models.search.AnnotationSearchResult; -import org.texttechnologylab.models.search.PageSnippet; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.UnifiedTopic; @@ -388,6 +386,9 @@ public List getAllAnnotations(int pagesSkip, int pagesTake, Map< // emotions int emotionModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Emotion.class), -1); annotations.addAll(emotions.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> emotionModelId == -1 || e.getModelVersion().getModel().getId() == emotionModelId).toList()); + // offensiveSpeech + int offensiveSpeechModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(OffensiveSpeech.class), -1); + annotations.addAll(offensiveSpeeches.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> offensiveSpeechModelId == -1 || e.getModelVersion().getModel().getId() == offensiveSpeechModelId).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java index 550d0634..7b9d2054 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -12,7 +12,9 @@ import org.texttechnologylab.utils.Pair; import javax.persistence.*; +import java.util.Comparator; import java.util.List; +import java.util.UUID; @Entity @Table(name = "offensivespeech") @@ -51,15 +53,32 @@ public void setDocument(Document document) { this.document = document; } + private OffensiveSpeechValue getRepresentativeOffensiveSpeechValue() { + if (this.offensiveSpeechValues != null && !this.offensiveSpeechValues.isEmpty()) { + return this.offensiveSpeechValues.stream().max(Comparator.comparingDouble(OffensiveSpeechValue::getValue)).orElse(null); + } + return null; + } + + public String generateOffensiveSpeechMarker() { + OffensiveSpeechValue rep = getRepresentativeOffensiveSpeechValue(); + String repValue = rep != null ? rep.getOffensiveSpeechType().getName() : ""; + return String.format("os", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateOffensiveSpeechCoveredStartSpan() { + OffensiveSpeechValue rep = getRepresentativeOffensiveSpeechValue(); + String repValue = rep != null ? rep.getOffensiveSpeechType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + @Override public String getWikiId() { return "OS" + "-" + this.getId(); } public List> collectOffensiveSpeechValues() { - return this.offensiveSpeechValues.stream() - .map(value -> new Pair<>(value.getOffensiveSpeechType().getName(), value.getValue())) - .toList(); + return this.offensiveSpeechValues.stream().map(value -> new Pair<>(value.getOffensiveSpeechType().getName(), value.getValue())).toList(); } } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 630af4f2..c769eae1 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -2432,6 +2432,18 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag } } + // offensive speech + Hibernate.initialize(doc.getOffensiveSpeeches()); + + for (var offensiveSpeech : doc.getOffensiveSpeeches()) { + Hibernate.initialize(offensiveSpeech.getOffensiveSpeechValues()); + Hibernate.initialize(offensiveSpeech.getModelVersion()); + Hibernate.initialize(offensiveSpeech.getModelVersion().getModel()); + for (var osv : offensiveSpeech.getOffensiveSpeechValues()) { + Hibernate.initialize(osv.getOffensiveSpeechType()); + } + } + for (var link : doc.getWikipediaLinks()) { Hibernate.initialize(link.getWikiDataHyponyms()); } From e6bfd9d03b842c1d8337cc704085ccb0ae3c5bc2 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 22:41:13 +0200 Subject: [PATCH 103/114] Add handling and UI elements for toxic annotations in document processing --- .../templates/css/document-reader.css | 28 +++++++++++++++++++ .../models/UIMAAnnotation.java | 24 ++++++++++++++++ .../models/corpus/Document.java | 3 ++ .../texttechnologylab/models/toxic/Toxic.java | 21 ++++++++++++++ .../PostgresqlDataInterface_Impl.java | 12 ++++++++ 5 files changed, 88 insertions(+) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index bbbbc17a..7b4b337a 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -348,6 +348,11 @@ body { position: relative; } +.document-content .colorable-toxic:hover { + transition: background-color 0.3s ease; + position: relative; +} + .document-content .animated-topic-scroll { transition: box-shadow 0.3s ease-in-out; box-shadow: 0 0 10px 5px rgba(255, 255, 0, 0.5); @@ -416,6 +421,29 @@ body { padding: 2px 4px; } +.document-content .toxic-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .toxic-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + .side-bar { width: 500px; transition: 0.5s; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index 3246e49a..da970104 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -11,6 +11,7 @@ import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.utils.StringUtils; import javax.persistence.*; @@ -139,6 +140,9 @@ public String buildHTMLString(List annotations, String coveredTe Map offensiveSpeechMarkers = new TreeMap<>(); Map offensiveSpeechCoverWrappersStart = new TreeMap<>(); Map offensiveSpeechCoverWrappersEnd = new TreeMap<>(); + Map toxicMarkers = new TreeMap<>(); + Map toxicCoverWrappersStart = new TreeMap<>(); + Map toxicCoverWrappersEnd = new TreeMap<>(); for (var annotation : annotations) { @@ -201,6 +205,17 @@ public String buildHTMLString(List annotations, String coveredTe continue; } + if (annotation instanceof Toxic toxic) { + var start = toxic.getBegin() - offset - errorOffset; + var end = toxic.getEnd() - offset - errorOffset; + + toxicCoverWrappersStart.put(start, toxic.generateToxicCoveredStartSpan()); + toxicCoverWrappersEnd.put(end, ""); + + toxicMarkers.put(end, toxic.generateToxicMarker()); + continue; + } + var start = annotation.getBegin() - offset - errorOffset; var end = annotation.getEnd() - offset - errorOffset; @@ -226,6 +241,9 @@ public String buildHTMLString(List annotations, String coveredTe if (offensiveSpeechCoverWrappersEnd.containsKey(i)) { finalText.append(offensiveSpeechCoverWrappersEnd.get(i)); } + if (toxicCoverWrappersEnd.containsKey(i)) { + finalText.append(toxicCoverWrappersEnd.get(i)); + } if (endTags.containsKey(i)) { //finalText.append(endTags.get(i).getFirst()); for (var tag : endTags.get(i)) { @@ -234,6 +252,9 @@ public String buildHTMLString(List annotations, String coveredTe } // Insert start spans + if (toxicCoverWrappersStart.containsKey(i)) { + finalText.append(toxicCoverWrappersStart.get(i)); + } if (offensiveSpeechCoverWrappersStart.containsKey(i)) { finalText.append(offensiveSpeechCoverWrappersStart.get(i)); } @@ -254,6 +275,9 @@ public String buildHTMLString(List annotations, String coveredTe if (offensiveSpeechMarkers.containsKey(i)) { finalText.append(offensiveSpeechMarkers.get(i)); } + if (toxicMarkers.containsKey(i)) { + finalText.append(toxicMarkers.get(i)); + } if (startTags.containsKey(i)) { finalText.append(generateMultiHTMLTag(startTags.get(i))); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index 6ae37a25..f7811560 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -389,6 +389,9 @@ public List getAllAnnotations(int pagesSkip, int pagesTake, Map< // offensiveSpeech int offensiveSpeechModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(OffensiveSpeech.class), -1); annotations.addAll(offensiveSpeeches.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> offensiveSpeechModelId == -1 || e.getModelVersion().getModel().getId() == offensiveSpeechModelId).toList()); + // toxic + int toxicModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Toxic.class), -1); + annotations.addAll(toxics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> toxicModelId == -1 || e.getModelVersion().getModel().getId() == toxicModelId).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java index bdadaac7..e9a406a0 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -11,7 +11,9 @@ import org.texttechnologylab.models.modelInfo.NamedModel; import javax.persistence.*; +import java.util.Comparator; import java.util.List; +import java.util.UUID; @Entity @Table(name = "toxic") @@ -50,6 +52,25 @@ public void setDocument(Document document) { this.document = document; } + private ToxicValue getRepresentativeToxicValue() { + if (this.toxicValues != null && !this.toxicValues.isEmpty()) { + return this.toxicValues.stream().max(Comparator.comparingDouble(ToxicValue::getValue)).orElse(null); + } + return null; + } + + public String generateToxicMarker() { + ToxicValue rep = getRepresentativeToxicValue(); + String repValue = rep != null ? rep.getToxicType().getName() : ""; + return String.format("t", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateToxicCoveredStartSpan() { + ToxicValue rep = getRepresentativeToxicValue(); + String repValue = rep != null ? rep.getToxicType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + @Override public String getWikiId() { return "T" + "-" + this.getId(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index c769eae1..fa54b996 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -2444,6 +2444,18 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag } } + // toxic + Hibernate.initialize(doc.getToxics()); + + for (var toxic : doc.getToxics()) { + Hibernate.initialize(toxic.getToxicValues()); + Hibernate.initialize(toxic.getModelVersion()); + Hibernate.initialize(toxic.getModelVersion().getModel()); + for (var tv : toxic.getToxicValues()) { + Hibernate.initialize(tv.getToxicType()); + } + } + for (var link : doc.getWikipediaLinks()) { Hibernate.initialize(link.getWikiDataHyponyms()); } From 912e272f76e35b2612c686a1734f8aad813d8c3f Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 22:44:29 +0200 Subject: [PATCH 104/114] Add category handling for OffensiveSpeech and Toxic in document processing --- .../java/org/texttechnologylab/models/corpus/Document.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index f7811560..f7d3159e 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -461,6 +461,12 @@ public List getModelCategories() { if (!emotions.isEmpty()) { categories.add(ModelNameHelper.getModelName(Emotion.class)); } + if (!offensiveSpeeches.isEmpty()) { + categories.add(ModelNameHelper.getModelName(OffensiveSpeech.class)); + } + if (!toxics.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Toxic.class)); + } return categories; } From 184933797f4821ea0673ba8018b5bb62faef7b82 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Tue, 12 Aug 2025 22:49:53 +0200 Subject: [PATCH 105/114] Add flexed-column class for improved layout in document reader view --- uce.portal/resources/templates/css/site.css | 5 +++++ uce.portal/resources/templates/reader/documentReaderView.ftl | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/css/site.css b/uce.portal/resources/templates/css/site.css index 09df94bd..259c1d68 100644 --- a/uce.portal/resources/templates/css/site.css +++ b/uce.portal/resources/templates/css/site.css @@ -232,6 +232,11 @@ nav .selected-nav-btn.text::before { display: flex; } +.flexed-column { + display: flex; + flex-direction: column; +} + .display-none { display: none; } diff --git a/uce.portal/resources/templates/reader/documentReaderView.ftl b/uce.portal/resources/templates/reader/documentReaderView.ftl index 77788436..6ca96e8e 100644 --- a/uce.portal/resources/templates/reader/documentReaderView.ftl +++ b/uce.portal/resources/templates/reader/documentReaderView.ftl @@ -264,7 +264,7 @@ <#if modelCategories?has_content && modelCategories?size gt 0>

${languageResource.get("modelSelection")}

-
+
<#list modelCategories as category>
From 8e4afbf13d62e05ec4376ea1ae2abdcf02210ff8 Mon Sep 17 00:00:00 2001 From: imer Date: Wed, 13 Aug 2025 16:20:33 +0200 Subject: [PATCH 106/114] Dynamic Sentiment Types --- .../models/sentiment/Sentiment.java | 19 ++++-------- .../models/sentiment/SentimentType.java | 25 ++++++++++++++++ .../models/sentiment/SentimentValue.java | 29 +++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java create mode 100644 uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index c2ae6ecd..b8e48df5 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -2,12 +2,15 @@ import lombok.Getter; import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; import org.texttechnologylab.annotations.Typesystem; import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; import javax.persistence.*; +import java.util.List; @Entity @Table(name = "sentiment") @@ -21,19 +24,9 @@ public class Sentiment extends UIMAAnnotation implements WikiModel { @Getter @Setter - @Column(name = "probability_positive", nullable = false) - private double probabilityPositive; - - @Getter - @Setter - @Column(name = "probability_neutral", nullable = false) - private double probabilityNeutral; - - @Getter - @Setter - @Column(name = "probability_negative", nullable = false) - private double probabilityNegative; - + @OneToMany(mappedBy = "sentiment", fetch = FetchType.LAZY, cascade = CascadeType.ALL) + @Fetch(value = FetchMode.SUBSELECT) + private List sentimentValues; @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) @JoinColumn(name = "document_id", nullable = false) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java new file mode 100644 index 00000000..d1289b83 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java @@ -0,0 +1,25 @@ +package org.texttechnologylab.models.sentiment; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "sentiment_type") +public class SentimentType extends ModelBase { + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public SentimentType() { + } + public SentimentType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java new file mode 100644 index 00000000..f782e1f4 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java @@ -0,0 +1,29 @@ +package org.texttechnologylab.models.sentiment; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "sentiment_value") +public class SentimentValue extends ModelBase { + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "sentiment_id", nullable = false) + private Sentiment sentiment; + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "sentiment_type_id", nullable = false) + private SentimentType sentimentType; + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public SentimentValue() {} + +} From e8db73da1a4e99e0444f44668a9476806c17d5da Mon Sep 17 00:00:00 2001 From: imer Date: Wed, 13 Aug 2025 16:32:20 +0200 Subject: [PATCH 107/114] add datainterface methods for sentiment --- .../PostgresqlDataInterface_Impl.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index dc9b4541..48d66658 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -31,6 +31,8 @@ import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.CompleteNegation; import org.texttechnologylab.models.search.*; +import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; @@ -1356,6 +1358,35 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera }); } + public Sentiment getInitializedSentimentById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var sentiment = session.get(Sentiment.class, id); + Hibernate.initialize(sentiment.getSentimentValues()); + for (var s : sentiment.getSentimentValues()) { + Hibernate.initialize(s.getSentimentType()); + } + return sentiment; + }); + } + + public synchronized SentimentType getOrCreateSentimentType(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(SentimentType.class); + var root = criteriaQuery.from(SentimentType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + + SentimentType sentimentType; + try { + sentimentType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + sentimentType = new SentimentType(name); + session.save(sentimentType); + } + return sentimentType; + }); + } + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); From 641e4bc71e8decf5224e548a95d53b7589b5a37e Mon Sep 17 00:00:00 2001 From: imer Date: Wed, 13 Aug 2025 16:45:49 +0200 Subject: [PATCH 108/114] added sentiment values to Importer --- .../java/org/texttechnologylab/Importer.java | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 66097393..11476809 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -46,6 +46,8 @@ import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; +import org.texttechnologylab.models.sentiment.SentimentValue; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -1431,9 +1433,37 @@ private void setSentiment(Document document, JCas jCas){ sentiment.setDocument(document); sentiment.setSentiment(s.getSentiment()); - sentiment.setProbabilityPositive(s.getProbabilityPositive()); - sentiment.setProbabilityNegative(s.getProbabilityNegative()); - sentiment.setProbabilityNeutral(s.getProbabilityNeutral()); + + List sentimentValues = new ArrayList<>(); + SentimentType positiveType, negativeType, neutralType; + try { + positiveType = db.getOrCreateSentimentType("positive"); + negativeType = db.getOrCreateSentimentType("negative"); + neutralType = db.getOrCreateSentimentType("neutral"); + } catch (DatabaseOperationException e) { + logger.error("Error while fetching Sentiment Types", e); + return; + } + + SentimentValue positiveValue = new SentimentValue(); + positiveValue.setSentimentType(positiveType); + positiveValue.setValue(s.getProbabilityPositive()); + positiveValue.setSentiment(sentiment); + sentimentValues.add(positiveValue); + + SentimentValue negativeValue = new SentimentValue(); + negativeValue.setSentimentType(negativeType); + negativeValue.setValue(s.getProbabilityNegative()); + negativeValue.setSentiment(sentiment); + sentimentValues.add(negativeValue); + + SentimentValue neutralValue = new SentimentValue(); + neutralValue.setSentimentType(neutralType); + neutralValue.setValue(s.getProbabilityNeutral()); + neutralValue.setSentiment(sentiment); + sentimentValues.add(neutralValue); + + sentiment.setSentimentValues(sentimentValues); sentiment.setCoveredText(s.getCoveredText()); sentiments.add(sentiment); From 2c7b9f4bb2f97fdecd3b268f198c1a7bb26fdb6c Mon Sep 17 00:00:00 2001 From: imer Date: Wed, 13 Aug 2025 16:52:46 +0200 Subject: [PATCH 109/114] add classes to HibernateConf --- .../main/java/org/texttechnologylab/config/HibernateConf.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index d7d31b82..e4c97a69 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -18,6 +18,8 @@ import org.texttechnologylab.models.imp.UCEImport; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; +import org.texttechnologylab.models.sentiment.SentimentValue; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; @@ -83,6 +85,8 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); // sentiment metadataSources.addAnnotatedClass(Sentiment.class); + metadataSources.addAnnotatedClass(SentimentType.class); + metadataSources.addAnnotatedClass(SentimentValue.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); From 1524475f2cd9a842434b0db7c5d0f1f992f7a6f8 Mon Sep 17 00:00:00 2001 From: imer Date: Wed, 13 Aug 2025 16:58:34 +0200 Subject: [PATCH 110/114] fix merge related issues --- .../org/texttechnologylab/models/sentiment/Sentiment.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index d4679935..b45c0229 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -61,10 +61,8 @@ public String getWikiId() { } public List> loopThroughProperties() { - return List.of( - new Pair<>("probability positive", Double.toString(probabilityPositive)), - new Pair<>("probability negative", Double.toString(probabilityNegative)), - new Pair<>("probability neutral", Double.toString(probabilityNeutral)) - ); + return sentimentValues.stream() + .map(v -> new Pair<>(v.getSentimentType().getName(), String.format("%.6f", v.getValue()))) + .toList(); } } From edebe1cdf9012614b2e72a04185f8bbde619fe55 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 13 Aug 2025 17:24:06 +0200 Subject: [PATCH 111/114] Add sentiment checkbox to corpus annotations for improved UI --- .../templates/corpus/components/corpusAnnotations.ftl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl index 40fa849e..bbb0df4d 100644 --- a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl +++ b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl @@ -219,5 +219,16 @@
+ +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isSentiment() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
\ No newline at end of file From 43f5a0e43fb6cf221276b446a10fcf592bece37c Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 13 Aug 2025 17:44:24 +0200 Subject: [PATCH 112/114] Add model information display to annotation pages and update data interface --- .../templates/wiki/components/modelInfo.ftl | 21 +++++++++++++++++++ .../wiki/pages/emotionAnnotationPage.ftl | 6 ++++++ .../pages/offensiveSpeechAnnotationPage.ftl | 6 ++++++ .../wiki/pages/sentimentAnnotationPage.ftl | 6 ++++++ .../wiki/pages/toxicAnnotationPage.ftl | 6 ++++++ .../PostgresqlDataInterface_Impl.java | 16 ++++++++++++++ 6 files changed, 61 insertions(+) create mode 100644 uce.portal/resources/templates/wiki/components/modelInfo.ftl diff --git a/uce.portal/resources/templates/wiki/components/modelInfo.ftl b/uce.portal/resources/templates/wiki/components/modelInfo.ftl new file mode 100644 index 00000000..3f71413e --- /dev/null +++ b/uce.portal/resources/templates/wiki/components/modelInfo.ftl @@ -0,0 +1,21 @@ +<#if model.getModelVersion()??> + <#assign modelVersionObject = model.getModelVersion()> + <#assign modelObject = modelVersionObject.getModel()> + <#assign modelVersion = modelVersionObject.getVersion()!> + <#assign modelName = modelObject.getName()!> +<#else> + <#assign modelVersion = "N/A"> + <#assign modelName = "N/A"> + +
+
+
+ + +
+
+
\ No newline at end of file diff --git a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl index 1b012c8c..574d15e2 100644 --- a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl +++ b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl @@ -10,6 +10,12 @@ <#include "*/wiki/components/metadata.ftl"> + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+
diff --git a/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl index 75c21d59..d960757e 100644 --- a/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl +++ b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl @@ -10,6 +10,12 @@ <#include "*/wiki/components/metadata.ftl"> + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+
diff --git a/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl index 15baabcb..db0e6fa5 100644 --- a/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl +++ b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl @@ -10,6 +10,12 @@ <#include "*/wiki/components/metadata.ftl"> + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+
diff --git a/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl index 84b4bfe7..63b8c638 100644 --- a/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl +++ b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl @@ -10,6 +10,12 @@ <#include "*/wiki/components/metadata.ftl"> + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 39f337e8..9a77cdc9 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -1424,6 +1424,10 @@ public Sentiment getInitializedSentimentById(long id) throws DatabaseOperationEx for (var s : sentiment.getSentimentValues()) { Hibernate.initialize(s.getSentimentType()); } + Hibernate.initialize(sentiment.getModelVersion()); + if (sentiment.getModelVersion() != null) { + Hibernate.initialize(sentiment.getModelVersion().getModel()); + } return sentiment; }); } @@ -1473,6 +1477,10 @@ public OffensiveSpeech getInitializedOffensiveSpeechById(long id) throws Databas for (OffensiveSpeechValue value : offensiveSpeech.getOffensiveSpeechValues()) { Hibernate.initialize(value.getOffensiveSpeechType()); } + Hibernate.initialize(offensiveSpeech.getModelVersion()); + if (offensiveSpeech.getModelVersion() != null) { + Hibernate.initialize(offensiveSpeech.getModelVersion().getModel()); + } return offensiveSpeech; }); } @@ -1484,6 +1492,10 @@ public Toxic getInitializedToxicById(long id) throws DatabaseOperationException for(var t:toxic.getToxicValues()){ Hibernate.initialize(t.getToxicType()); } + Hibernate.initialize(toxic.getModelVersion()); + if (toxic.getModelVersion() != null) { + Hibernate.initialize(toxic.getModelVersion().getModel()); + } return toxic; }); } @@ -1513,6 +1525,10 @@ public Emotion getInitializedEmotionById(long id) throws DatabaseOperationExcept for (var ev : emotion.getEmotionValues()) { Hibernate.initialize(ev.getEmotionType()); } + Hibernate.initialize(emotion.getModelVersion()); + if (emotion.getModelVersion() != null) { + Hibernate.initialize(emotion.getModelVersion().getModel()); + } return emotion; }); } From 82b7393d39f32d05b7032b2c051ae89b86ebe223 Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 13 Aug 2025 17:50:26 +0200 Subject: [PATCH 113/114] Add model handling in sentiment processing --- .../models/sentiment/Sentiment.java | 3 ++- .../java/org/texttechnologylab/Importer.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index b45c0229..1f1f2dab 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -8,15 +8,16 @@ import org.texttechnologylab.models.UIMAAnnotation; import org.texttechnologylab.models.WikiModel; import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; import org.texttechnologylab.utils.Pair; import javax.persistence.*; import java.util.List; -import java.util.ArrayList; @Entity @Table(name = "sentiment") @Typesystem(types = {org.texttechnologylab.annotation.SentimentModel.class}) +@NamedModel(name = "sentiment") public class Sentiment extends UIMAAnnotation implements WikiModel { @Getter diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index cae89340..dd23bf8c 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -1472,12 +1472,30 @@ private void setUnifiedTopic(Document document, JCas jCas) { } private void setSentiment(Document document, JCas jCas){ + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Sentiment.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Sentiment", e); + return; + } List sentiments = new ArrayList<>(); JCasUtil.select(jCas, org.texttechnologylab.annotation.SentimentModel.class).forEach(s -> { Sentiment sentiment = new Sentiment(s.getBegin(), s.getEnd()); sentiment.setDocument(document); + try { + MetaData modelMetaData = s.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + sentiment.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for Toxic", e); + return; + } + sentiment.setSentiment(s.getSentiment()); List sentimentValues = new ArrayList<>(); From a2913732d8803a0778678f341f7f5642eb805a3c Mon Sep 17 00:00:00 2001 From: UnleqitQ Date: Wed, 13 Aug 2025 18:03:11 +0200 Subject: [PATCH 114/114] Add sentiment handling and UI enhancements for sentiment markers --- .../templates/css/document-reader.css | 30 ++++++++++++++++++- .../models/UIMAAnnotation.java | 24 +++++++++++++++ .../models/corpus/Document.java | 6 ++++ .../models/sentiment/Sentiment.java | 23 ++++++++++++++ .../PostgresqlDataInterface_Impl.java | 12 ++++++++ 5 files changed, 94 insertions(+), 1 deletion(-) diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 7b4b337a..5d4d9879 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -348,7 +348,12 @@ body { position: relative; } -.document-content .colorable-toxic:hover { +.document-content .colorable-toxic { + transition: background-color 0.3s ease; + position: relative; +} + +.document-content .colorable-sentiment { transition: background-color 0.3s ease; position: relative; } @@ -444,6 +449,29 @@ body { padding: 2px 4px; } +.document-content .sentiment-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .sentiment-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + .side-bar { width: 500px; transition: 0.5s; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index da970104..467fddef 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -10,6 +10,7 @@ import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.UnifiedTopic; import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.utils.StringUtils; @@ -143,6 +144,9 @@ public String buildHTMLString(List annotations, String coveredTe Map toxicMarkers = new TreeMap<>(); Map toxicCoverWrappersStart = new TreeMap<>(); Map toxicCoverWrappersEnd = new TreeMap<>(); + Map sentimentMarkers = new TreeMap<>(); + Map sentimentCoverWrappersStart = new TreeMap<>(); + Map sentimentCoverWrappersEnd = new TreeMap<>(); for (var annotation : annotations) { @@ -216,6 +220,17 @@ public String buildHTMLString(List annotations, String coveredTe continue; } + if (annotation instanceof Sentiment sentiment) { + var start = sentiment.getBegin() - offset - errorOffset; + var end = sentiment.getEnd() - offset - errorOffset; + + sentimentCoverWrappersStart.put(start, sentiment.generateSentimentCoveredStartSpan()); + sentimentCoverWrappersEnd.put(end, ""); + + sentimentMarkers.put(end, sentiment.generateSentimentMarker()); + continue; + } + var start = annotation.getBegin() - offset - errorOffset; var end = annotation.getEnd() - offset - errorOffset; @@ -244,6 +259,9 @@ public String buildHTMLString(List annotations, String coveredTe if (toxicCoverWrappersEnd.containsKey(i)) { finalText.append(toxicCoverWrappersEnd.get(i)); } + if (sentimentCoverWrappersEnd.containsKey(i)) { + finalText.append(sentimentCoverWrappersEnd.get(i)); + } if (endTags.containsKey(i)) { //finalText.append(endTags.get(i).getFirst()); for (var tag : endTags.get(i)) { @@ -252,6 +270,9 @@ public String buildHTMLString(List annotations, String coveredTe } // Insert start spans + if (sentimentCoverWrappersStart.containsKey(i)) { + finalText.append(sentimentCoverWrappersStart.get(i)); + } if (toxicCoverWrappersStart.containsKey(i)) { finalText.append(toxicCoverWrappersStart.get(i)); } @@ -278,6 +299,9 @@ public String buildHTMLString(List annotations, String coveredTe if (toxicMarkers.containsKey(i)) { finalText.append(toxicMarkers.get(i)); } + if (sentimentMarkers.containsKey(i)) { + finalText.append(sentimentMarkers.get(i)); + } if (startTags.containsKey(i)) { finalText.append(generateMultiHTMLTag(startTags.get(i))); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index 11ecc7ae..f6b69dfa 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -399,6 +399,9 @@ public List getAllAnnotations(int pagesSkip, int pagesTake, Map< // toxic int toxicModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Toxic.class), -1); annotations.addAll(toxics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> toxicModelId == -1 || e.getModelVersion().getModel().getId() == toxicModelId).toList()); + // sentiment + int sentimentModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Sentiment.class), -1); + annotations.addAll(sentiments.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> sentimentModelId == -1 || e.getModelVersion().getModel().getId() == sentimentModelId).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; @@ -474,6 +477,9 @@ public List getModelCategories() { if (!toxics.isEmpty()) { categories.add(ModelNameHelper.getModelName(Toxic.class)); } + if (!sentiments.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Sentiment.class)); + } return categories; } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java index 1f1f2dab..acde4be4 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -12,7 +12,9 @@ import org.texttechnologylab.utils.Pair; import javax.persistence.*; +import java.util.Comparator; import java.util.List; +import java.util.UUID; @Entity @Table(name = "sentiment") @@ -56,6 +58,27 @@ public void setDocument(Document document) { this.document = document; } + private SentimentValue getRepresentativeSentimentValue() { + if (this.sentimentValues != null && !this.sentimentValues.isEmpty()) { + return this.sentimentValues.stream() + .max(Comparator.comparingDouble(SentimentValue::getValue)) + .orElse(null); + } + return null; + } + + public String generateSentimentMarker() { + SentimentValue rep = getRepresentativeSentimentValue(); + String repValue = rep != null ? rep.getSentimentType().getName() : ""; + return String.format("s", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateSentimentCoveredStartSpan() { + SentimentValue rep = getRepresentativeSentimentValue(); + String repValue = rep != null ? rep.getSentimentType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + @Override public String getWikiId() { return "S" + "-" + this.getId(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index 9a77cdc9..a6511c73 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -2503,6 +2503,18 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag } } + // sentiment + Hibernate.initialize(doc.getSentiments()); + + for (var sentiment : doc.getSentiments()) { + Hibernate.initialize(sentiment.getSentimentValues()); + Hibernate.initialize(sentiment.getModelVersion()); + Hibernate.initialize(sentiment.getModelVersion().getModel()); + for (var sv : sentiment.getSentimentValues()) { + Hibernate.initialize(sv.getSentimentType()); + } + } + for (var link : doc.getWikipediaLinks()) { Hibernate.initialize(link.getWikiDataHyponyms()); }