diff --git a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl index cfd27136..bbb0df4d 100644 --- a/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl +++ b/uce.portal/resources/templates/corpus/components/corpusAnnotations.ftl @@ -186,5 +186,49 @@ + +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isOffensiveSpeech() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
+ +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isEmotion() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
+ +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isToxic() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
+ +
+ <#assign isChecked = "" /> + <#if corpusConfig.getAnnotations().isSentiment() == true> + <#assign isChecked = "checked"/> + +
+ + +
+
\ No newline at end of file diff --git a/uce.portal/resources/templates/css/document-reader.css b/uce.portal/resources/templates/css/document-reader.css index 8c88a990..5d4d9879 100644 --- a/uce.portal/resources/templates/css/document-reader.css +++ b/uce.portal/resources/templates/css/document-reader.css @@ -141,21 +141,21 @@ body { .header-btn:hover { background-color: lightgray; transition: 0.15s; - color:white; + color: white; } .document-content { padding: 2rem 4.5rem; } -.document-content *{ - font-family: Courier !important; +.document-content * { + font-family: Courier !important; word-break: break-word; } .document-content p, .document-content label, -.document-content span{ +.document-content span { font-size: 16px; } @@ -163,8 +163,8 @@ body { .document-content h4, .document-content h3, .document-content h2, -.document-content h1{ - color:rgba(0, 0, 0, 0.6); +.document-content h1 { + color: rgba(0, 0, 0, 0.6); font-weight: bold; } @@ -338,6 +338,26 @@ body { position: relative; } +.document-content .colorable-emotion { + transition: background-color 0.3s ease; + position: relative; +} + +.document-content .colorable-offensive { + transition: background-color 0.3s ease; + position: relative; +} + +.document-content .colorable-toxic { + transition: background-color 0.3s ease; + position: relative; +} + +.document-content .colorable-sentiment { + transition: background-color 0.3s ease; + position: relative; +} + .document-content .animated-topic-scroll { transition: box-shadow 0.3s ease-in-out; box-shadow: 0 0 10px 5px rgba(255, 255, 0, 0.5); @@ -347,9 +367,9 @@ body { display: inline-flex; align-items: center; justify-content: center; - min-width: 20px; /* minimum size */ + min-width: 20px; /* minimum size */ height: 20px; - padding: 0 4px; /* horizontal padding */ + padding: 0 4px; /* horizontal padding */ border-radius: 50%; border: 1px solid #333; font-size: 14px; @@ -360,6 +380,98 @@ body { margin-left: 4px; } +.document-content .emotion-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .emotion-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + +.document-content .offensive-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .offensive-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + +.document-content .toxic-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .toxic-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + +.document-content .sentiment-marker { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; /* minimum size */ + height: 20px; + padding: 0 4px; /* horizontal padding */ + border-radius: 20%; + border: 1px solid #333; + font-size: 14px; + font-weight: bold; + background-color: white; + color: #333; + cursor: pointer; + margin-left: 4px; +} + +.document-content .sentiment-covered.highlight { + background-color: rgba(255, 0, 0, 0.2); + border-radius: 3px; + padding: 2px 4px; +} + .side-bar { width: 500px; transition: 0.5s; @@ -468,31 +580,72 @@ body { font-size: 0.7rem; color: #333; font-weight: 600; - box-shadow: 0 2px 4px rgba(0,0,0,0.1); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); transition: transform 0.2s ease, box-shadow 0.2s ease; text-align: center; width: 100%; white-space: normal; word-wrap: break-word; hyphens: auto; - border: 1px solid rgba(0,0,0,0.1); + border: 1px solid rgba(0, 0, 0, 0.1); min-height: 32px; } .side-bar-content .document-topics-list .topic-tag:hover { transform: translateY(-2px); - box-shadow: 0 4px 8px rgba(0,0,0,0.15); + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.15); cursor: pointer; } .side-bar-content .document-topics-list .topic-tag.active-topic { transform: translateY(-2px); - box-shadow: 0 4px 8px rgba(0,0,0,0.2); - border: 2px solid rgba(0,0,0,0.3); + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + border: 2px solid rgba(0, 0, 0, 0.3); font-weight: 700; opacity: 1.2; } +/* Sidebar Model Selection */ +.side-bar-content .model-category { + display: flex; + flex-direction: row; + align-items: center; + justify-content: space-between; +} +.side-bar-content .model-category label { + font-weight: bold; + font-size: 1.1rem; + color: #333; +} +.side-bar-content .model-category select { + padding: 6px 10px; + font-size: 14px; + border-radius: 5px; + border: 1px solid #ccc; + background-color: #f9f9f9; + transition: border-color 0.2s ease-in-out; +} +.side-bar-content .model-category select:focus { + border-color: #007bff; + outline: none; +} +.side-bar-content .model-category select option { + padding: 6px 10px; + font-size: 14px; + background-color: #fff; + color: #333; +} +.side-bar-content .model-category select option:hover { + background-color: #f0f0f0; +} +.side-bar-content .model-category select option:checked { + background-color: #007bff; + color: white; +} +.side-bar-content .model-category select option:checked:hover { + background-color: #0056b3; +} + /* Scrollbar Minimap Styles */ .scrollbar-minimap { position: fixed; @@ -566,14 +719,14 @@ body { --c: #ffffff; padding: 1em; - border-radius: var(--r)/var(--r) min(var(--r),var(--p) - var(--b)/2) min(var(--r),100% - var(--p) - var(--b)/2) var(--r); - clip-path: polygon(100% 100%,0 100%,0 0,100% 0, - 100% max(0% ,var(--p) - var(--b)/2), + border-radius: var(--r)/var(--r) min(var(--r), var(--p) - var(--b) / 2) min(var(--r), 100% - var(--p) - var(--b) / 2) var(--r); + clip-path: polygon(100% 100%, 0 100%, 0 0, 100% 0, + 100% max(0%, var(--p) - var(--b) / 2), calc(100% + var(--h)) var(--p), - 100% min(100%,var(--p) + var(--b)/2)); + 100% min(100%, var(--p) + var(--b) / 2)); background: var(--c); border-image: conic-gradient(var(--c) 0 0) fill 0/ - calc(var(--p) - var(--b)/2) 0 calc(100% - var(--p) - var(--b)/2) var(--r)/ + calc(var(--p) - var(--b) / 2) 0 calc(100% - var(--p) - var(--b) / 2) var(--r)/ 0 var(--h) 0 0; } @@ -622,6 +775,7 @@ body { .tab-content .tab-pane.active { display: block; } + .side-bar.visualization-expanded { width: 150vw !important; transition: width 0.3s ease; @@ -633,6 +787,7 @@ body { height: 100%; position: relative; } + .visualization-wrapper .visualization-content { flex: 1; overflow-y: auto; @@ -651,13 +806,16 @@ body { width: 100%; } + .visualization-wrapper .visualization-spinner__icon { margin-bottom: 16px; } + .visualization-wrapper .visualization-spinner__icon i { font-size: 2.5rem; color: #0078d4; } + .visualization-wrapper .visualization-spinner__text { font-size: 1.1rem; color: #444; @@ -669,6 +827,7 @@ body { .visualization-content .viz-panel { display: none; } + .viz-panel.active { display: block; } @@ -685,7 +844,7 @@ body { display: flex; justify-content: space-around; border-radius: 24px; - box-shadow: 0 4px 24px rgba(0,0,0,0.12); + box-shadow: 0 4px 24px rgba(0, 0, 0, 0.12); background: #fff; border: 1px solid #e0e0e0; padding: 0.5rem 1.5rem; @@ -704,6 +863,7 @@ body { color: #555; transition: color 0.2s, background 0.2s; } + .viz-nav-btn.active { color: #007bff; background: none; @@ -741,7 +901,7 @@ body { outline: none; } -#vp-3, #vp-4, #vp-5, #vp-2, #vp-1 { +#vp-3, #vp-4, #vp-5, #vp-2, #vp-1, #vp-6 { display: flex; align-items: center; justify-content: center; diff --git a/uce.portal/resources/templates/css/lexicon.css b/uce.portal/resources/templates/css/lexicon.css index a025a286..58685b0e 100644 --- a/uce.portal/resources/templates/css/lexicon.css +++ b/uce.portal/resources/templates/css/lexicon.css @@ -87,6 +87,14 @@ .lexicon-view .lexicon-entry[data-type="unifiedtopic"] { border-left-color: lightpink; } +.lexicon-view .lexicon-entry[data-type="sentiment"] { + /* off-white */ + border-left-color: #f8f0e3; +} + +.lexicon-view .lexicon-entry[data-type="emotion"] { + border-left-color: darkorange; +} .lexicon-view .lexicon-entry[data-type="geoname"] { border-left-color: dodgerblue; @@ -108,6 +116,14 @@ border-left-color: darkred; } +.lexicon-view .lexicon-entry[data-type="offensivespeech"]{ + border-left-color: silver; +} + +.lexicon-view .lexicon-entry[data-type="toxic"]{ + border-left-color: darkorange; +} + .lexicon-view .filter-container{ width: 100%; position: sticky; diff --git a/uce.portal/resources/templates/css/site.css b/uce.portal/resources/templates/css/site.css index 09df94bd..259c1d68 100644 --- a/uce.portal/resources/templates/css/site.css +++ b/uce.portal/resources/templates/css/site.css @@ -232,6 +232,11 @@ nav .selected-nav-btn.text::before { display: flex; } +.flexed-column { + display: flex; + flex-direction: column; +} + .display-none { display: none; } diff --git a/uce.portal/resources/templates/js/documentReader.js b/uce.portal/resources/templates/js/documentReader.js index ac7fb3c7..379ade22 100644 --- a/uce.portal/resources/templates/js/documentReader.js +++ b/uce.portal/resources/templates/js/documentReader.js @@ -272,6 +272,35 @@ function attachTopicClickHandlers() { dot.style.top = event.clientY - 9 + "px"; });*/ +/** + * Retrieve the model selection from the sidebar. + * @return Record + */ +function retrieveModelSelection() { + const modelSelects = $('.side-bar select.model-select'); + const modelSelection = {}; + for (let i = 0; i < modelSelects.length; i++) { + const modelSelect = $(modelSelects[i]); + const category = modelSelect.attr('model-category-name'); + const model = parseInt(modelSelect.val()); + if (category !== undefined && model !== undefined) { + modelSelection[category] = model; + } + } + return modelSelection; +} + +function modelCategoriesChanged(element) { + lazyLoadPages() + + const $element = $(element); + if ($element.attr('model-category-name') === 'emotion') { + const containerId = 'vp-6'; + const $container = $('#' + containerId); + $container.addClass('dirty'); + } +} + /** * Handle the lazy loading of more pages */ @@ -280,6 +309,12 @@ async function lazyLoadPages() { const id = $readerContainer.data('id'); const pagesCount = $readerContainer.data('pagescount'); + // clear out the existing pages + $('.reader-container .document-content').empty(); + + const modelSelection = retrieveModelSelection(); + const modelSelectionString = JSON.stringify(modelSelection); + for (let i = 0; i <= pagesCount; i += 10) { const $loadedPagesCount = $('.site-container .loaded-pages-count'); $loadedPagesCount.html(i); @@ -288,7 +323,7 @@ async function lazyLoadPages() { $loadedPagesCount.html(i); } else { await $.ajax({ - url: "/api/document/reader/pagesList?id=" + id + "&skip=" + i, + url: "/api/document/reader/pagesList?id=" + id + "&skip=" + i + "&modelSelection=" + encodeURIComponent(modelSelectionString), type: "GET", success: function (response) { // Render the new pages @@ -370,8 +405,7 @@ function colorUnifiedTopics(selectedTopic) { if ($selectedTopicTag.length === 0) { color = topicColorMap[selectedTopic]; - } - else{ + } else { color = $selectedTopicTag.css('background-color'); } @@ -477,16 +511,16 @@ function updateTopicNavButtonStates() { function initScrollbarMinimap() { setTimeout(updateMinimapMarkers, 500); - $(window).on('scroll', function() { + $(window).on('scroll', function () { updateMinimapScroll(); }); - $(window).on('resize', function() { + $(window).on('resize', function () { updateMinimapMarkers(); updateMinimapScroll(); }); - $('.scrollbar-minimap').on('click', function(e) { + $('.scrollbar-minimap').on('click', function (e) { const clickPosition = (e.pageY - $(this).offset().top) / $(this).height(); const dimensions = getMinimapDimensions(); const documentPosition = minimapToDocumentPosition(clickPosition * dimensions.minimapHeight, dimensions); @@ -497,7 +531,7 @@ function initScrollbarMinimap() { }, 300); }); - $(document).on('mouseenter', '.minimap-marker', function(e) { + $(document).on('mouseenter', '.minimap-marker', function (e) { const $marker = $(this); const $preview = $('.minimap-preview'); const $previewContent = $('.preview-content'); @@ -513,14 +547,14 @@ function initScrollbarMinimap() { const markerTop = parseFloat($marker.css('top')); const approximateDocumentPosition = minimapToDocumentPosition(markerTop, dimensions); - const $topicElements = $('.colorable-topic').filter(function() { + const $topicElements = $('.colorable-topic').filter(function () { return $(this).data('topic-value') === topicValue; }); let closestElement = null; let minDistance = Number.MAX_VALUE; - $topicElements.each(function() { + $topicElements.each(function () { const $element = $(this); const elementOffset = $element.offset(); @@ -547,7 +581,7 @@ function initScrollbarMinimap() { const coveredText = closestElement.data('wcovered'); if (coveredText) { - previewText = coveredText; + previewText = coveredText; } } if (previewTitle) { @@ -580,11 +614,11 @@ function initScrollbarMinimap() { } }); - $(document).on('mouseleave', '.minimap-marker', function() { + $(document).on('mouseleave', '.minimap-marker', function () { $('.minimap-preview').hide(); }); - $('.scrollbar-minimap').on('mouseleave', function() { + $('.scrollbar-minimap').on('mouseleave', function () { $('.minimap-preview').hide(); }); } @@ -592,10 +626,10 @@ function initScrollbarMinimap() { function updateMinimapMarkers() { const $minimap = $('.minimap-markers'); const dimensions = getMinimapDimensions(); - + $minimap.empty(); - $('.document-content .page').each(function(index) { + $('.document-content .page').each(function (index) { const $page = $(this); if (!$page.attr('id')) { @@ -628,7 +662,7 @@ function addAllTopicMarkersToMinimap() { const dimensions = getMinimapDimensions(); const topicPositions = {}; - $('.colorable-topic').each(function() { + $('.colorable-topic').each(function () { const $topic = $(this); const topicValue = $topic.data('topic-value'); @@ -651,7 +685,7 @@ function addAllTopicMarkersToMinimap() { } }); - Object.keys(topicPositions).forEach(function(position) { + Object.keys(topicPositions).forEach(function (position) { const pos = parseInt(position); const topicData = topicPositions[pos]; const topicValues = Object.keys(topicData.topics); @@ -673,7 +707,7 @@ function addAllTopicMarkersToMinimap() { }); } -function updateTopicMarkersOnMinimap(selectedTopic=null) { +function updateTopicMarkersOnMinimap(selectedTopic = null) { const $minimap = $('.minimap-markers'); const dimensions = getMinimapDimensions(); @@ -685,7 +719,7 @@ function updateTopicMarkersOnMinimap(selectedTopic=null) { const activeTopic = selectedTopic ? selectedTopic : $activeTopic.data('topic'); const topicColor = topicColorMap[activeTopic]; - $('.colorable-topic').each(function() { + $('.colorable-topic').each(function () { const $topic = $(this); const topicValue = $topic.data('topic-value'); @@ -718,18 +752,18 @@ function getMinimapDimensions() { } function documentToMinimapPosition(documentPos, dimensions) { - const { documentHeight, minimapHeight } = dimensions || getMinimapDimensions(); + const {documentHeight, minimapHeight} = dimensions || getMinimapDimensions(); return (documentPos / documentHeight) * minimapHeight; } function minimapToDocumentPosition(minimapPos, dimensions) { - const { documentHeight, minimapHeight } = dimensions || getMinimapDimensions(); + const {documentHeight, minimapHeight} = dimensions || getMinimapDimensions(); return (minimapPos / minimapHeight) * documentHeight; } function createMinimapMarker(options) { - const { top, height, color, elementId, topic, className } = options; - + const {top, height, color, elementId, topic, className} = options; + const $marker = $('
') .addClass('minimap-marker') .addClass(className || '') @@ -738,10 +772,10 @@ function createMinimapMarker(options) { 'height': Math.max(2, height) + 'px', 'background-color': color || '#ccc' }); - + if (elementId) $marker.attr('data-element-id', elementId); if (topic) $marker.attr('data-topic', topic); - + return $marker; } @@ -814,7 +848,7 @@ document.querySelectorAll('.tab-btn').forEach(btn => { $('.scrollbar-minimap').hide(); sideBar.classList.add('visualization-expanded'); } else { - setTimeout(updateFloatingUIPositions,500) ; + setTimeout(updateFloatingUIPositions, 500); currentSelectedTopic = null; sideBar.classList.remove('visualization-expanded'); $('.scrollbar-minimap').show(); @@ -863,10 +897,12 @@ $(document).on('click', '.viz-nav-btn', function () { setTimeout(() => renderSentenceTopicSankey('vp-5'), 500); } + if (target === '#viz-panel-6') { + setTimeout(() => renderEmotionDevelopment('vp-6'), 500); + } }); - function renderSentenceTopicNetwork(containerId) { const container = document.getElementById(containerId); if (!container || container.classList.contains('rendered')) return; @@ -878,13 +914,13 @@ function renderSentenceTopicNetwork(containerId) { $.ajax({ url: `/api/document/unifiedTopicSentenceMap`, method: 'GET', - data: { documentId }, + data: {documentId}, dataType: 'json', success: function (utToSentenceMapList) { const utToSentenceMap = new Map(); const topicToSentences = {}; - utToSentenceMapList.forEach(({ unifiedtopicId, sentenceId }) => { + utToSentenceMapList.forEach(({unifiedtopicId, sentenceId}) => { const utId = unifiedtopicId.toString(); const sId = sentenceId.toString(); utToSentenceMap.set(utId, sId); @@ -895,7 +931,7 @@ function renderSentenceTopicNetwork(containerId) { $.ajax({ url: `/api/rag/sentenceEmbeddings`, method: 'GET', - data: { documentId }, + data: {documentId}, dataType: 'json', success: function (embeddings) { $('.visualization-spinner').hide() @@ -908,7 +944,7 @@ function renderSentenceTopicNetwork(containerId) { return; } const sentenceEmbeddingMap = new Map(); - embeddings.forEach(({ sentenceId, tsne2d }) => { + embeddings.forEach(({sentenceId, tsne2d}) => { sentenceEmbeddingMap.set(sentenceId.toString(), tsne2d); }); @@ -931,11 +967,11 @@ function renderSentenceTopicNetwork(containerId) { nodes.push({ id: sentenceId, - name: `Sentence `+utId, + name: `Sentence ` + utId, symbolSize: 20, x, y, - itemStyle: { color }, - label: { show: false }, + itemStyle: {color}, + label: {show: false}, tooltip: { confine: true, // prevent overflow @@ -970,15 +1006,15 @@ function renderSentenceTopicNetwork(containerId) { const neighbors = embeddingArray .filter(([id2]) => id1 !== id2 && nodeSet.has(id2)) - .map(([id2, vec2]) => ({ id2, dist: euclidean(vec1, vec2) })) + .map(([id2, vec2]) => ({id2, dist: euclidean(vec1, vec2)})) .sort((a, b) => a.dist - b.dist) .slice(0, k); - neighbors.forEach(({ id2 }) => { + neighbors.forEach(({id2}) => { links.push({ source: id1, target: id2, - lineStyle: { color: '#bbb', opacity: 0.2 } + lineStyle: {color: '#bbb', opacity: 0.2} }); }); }); @@ -999,27 +1035,27 @@ function renderSentenceTopicNetwork(containerId) { '', nodes, links, - null, - function (params) { - if (params.dataType === 'node') { - const name = params.name.split('Sentence ')[1]; - $('.scrollbar-minimap').hide(); - hideTopicNavButtons(); - clearTopicColoring(); - $('.colorable-topic').each(function () { - const topicValue = $(this).data('topic-value'); - const utId = this.id.replace('utopic-UT-', ''); - if (utId === name) { - $(this).css({ - 'background-color': topicColorMap[topicValue], - 'border-radius': '3px', - 'padding': '0 2px' - }); - this.scrollIntoView({ behavior: 'smooth', block: 'center' }); - } - }); - } - }); + null, + function (params) { + if (params.dataType === 'node') { + const name = params.name.split('Sentence ')[1]; + $('.scrollbar-minimap').hide(); + hideTopicNavButtons(); + clearTopicColoring(); + $('.colorable-topic').each(function () { + const topicValue = $(this).data('topic-value'); + const utId = this.id.replace('utopic-UT-', ''); + if (utId === name) { + $(this).css({ + 'background-color': topicColorMap[topicValue], + 'border-radius': '3px', + 'padding': '0 2px' + }); + this.scrollIntoView({behavior: 'smooth', block: 'center'}); + } + }); + } + }); container.classList.add('rendered'); }, @@ -1074,17 +1110,16 @@ function computeTopicSimilarityMatrix(data, type = "cosine") { } - function renderTopicSimilarityMatrix(containerId) { const container = document.getElementById(containerId); - if (!container || container.classList.contains('rendered')){ + if (!container || container.classList.contains('rendered')) { $('.selector-container').show(); return; } $('.visualization-spinner').show() const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - $.get('/api/document/page/topicWords', { documentId: docId }) + $.get('/api/document/page/topicWords', {documentId: docId}) .then(data => { $('.visualization-spinner').hide() if (!data || !Array.isArray(data) || data.length === 0) { @@ -1101,13 +1136,13 @@ function renderTopicSimilarityMatrix(containerId) { function updateChart() { const type = similarityTypeSelector.value; - const { labels, matrix } = computeTopicSimilarityMatrix(data, type); + const {labels, matrix} = computeTopicSimilarityMatrix(data, type); const tooltipFormatter = function (params) { const xLabel = labels[params.data[0]]; const yLabel = labels[params.data[1]]; const value = type === "count" ? params.data[2] : params.data[2].toFixed(3); - return xLabel + " & " + yLabel + "
" + (type.charAt(0).toUpperCase() + type.slice(1)) + ": " + value; + return xLabel + " & " + yLabel + "
" + (type.charAt(0).toUpperCase() + type.slice(1)) + ": " + value; }; window.graphVizHandler.createHeatMap( @@ -1136,7 +1171,7 @@ function renderTopicEntityChordDiagram(containerId) { const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - $.get('/api/document/page/topicEntityRelation', { documentId: docId }) + $.get('/api/document/page/topicEntityRelation', {documentId: docId}) .then(data => { $('.visualization-spinner').hide() if (!data || !Array.isArray(data) || data.length === 0) { @@ -1154,8 +1189,8 @@ function renderTopicEntityChordDiagram(containerId) { let nodeIndex = 0; const categories = [ - { name: 'Topic', itemStyle: { color: '#5470C6' } }, - { name: 'Entity', itemStyle: { color: '#91CC75' } } + {name: 'Topic', itemStyle: {color: '#5470C6'}}, + {name: 'Entity', itemStyle: {color: '#91CC75'}} ]; function getCategory(name, isEntity) { @@ -1169,11 +1204,11 @@ function renderTopicEntityChordDiagram(containerId) { if (topic && !nodeMap.has(topic)) { nodeMap.set(topic, nodeIndex++); - nodes.push({ name: topic, value: 0, category: getCategory(topic, false) }); + nodes.push({name: topic, value: 0, category: getCategory(topic, false)}); } if (entityType && !nodeMap.has(entityType)) { nodeMap.set(entityType, nodeIndex++); - nodes.push({ name: entityType, value: 0, category: getCategory(entityType, true) }); + nodes.push({name: entityType, value: 0, category: getCategory(entityType, true)}); } // Step 2: count link frequency as weight @@ -1262,7 +1297,7 @@ function renderTopicEntityChordDiagram(containerId) { const pageNumber = params.name; const pageElement = document.querySelector('.page[data-id="' + pageNumber + '"]'); if (pageElement) { - pageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }); + pageElement.scrollIntoView({behavior: 'smooth', block: 'start'}); } else { console.error(`Page ` + pageNumber + ` not found.`); } @@ -1364,7 +1399,7 @@ function renderSentenceTopicSankey(containerId) { 'border-radius': '3px', 'padding': '0 2px' }); - this.scrollIntoView({ behavior: 'smooth', block: 'center' }); + this.scrollIntoView({behavior: 'smooth', block: 'center'}); } }); } @@ -1377,6 +1412,181 @@ function renderSentenceTopicSankey(containerId) { }); } +function renderEmotionDevelopment(containerId) { + const container = document.getElementById(containerId); + if (!container || (container.classList.contains('rendered') && !container.classList.contains('dirty'))) return; + + $('.visualization-spinner').show(); + const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); + const modelSelection = retrieveModelSelection(); + const emotionReq = $.get('/api/document/page/emotionDev', {documentId: docId, modelId: modelSelection['emotion']}); + emotionReq.then(emotionData => { + $('.visualization-spinner').hide(); + if (!emotionData || typeof emotionData !== 'object' || !("emotionTypes" in emotionData) || !Array.isArray(emotionData.emotionTypes) || emotionData.emotionTypes.length === 0 || !("emotionData" in emotionData) || !Array.isArray(emotionData.emotionData) || emotionData.emotionData.length === 0) { + if (container.classList.contains('rendered') && container.classList.contains('dirty')) { + return; + } + const container = document.getElementById(containerId); + if (container) { + container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; + } + container.classList.add('rendered'); + return; + } + + const emotionColors = {}; + [...emotionData.emotionTypes].sort((a, b) => a.name.localeCompare(b.name)).forEach((emotionType, index) => { + const hue = (index * 360 / emotionData.emotionTypes.length) % 360; + emotionColors[emotionType.id] = 'hsl(' + hue + ', 70%, 50%)'; + }); + + const seriesData = emotionData.emotionTypes.map(emotionType => { + const emotionId = emotionType.id; + const emotionName = emotionType.name; + const emotionValues = emotionData.emotionData.map(pageData => { + const emotionValue = pageData.find(e => e.emotionType === emotionId); + return emotionValue ? emotionValue.emotionValue : 0; + }); + return { + name: emotionName, + type: 'bar', + data: emotionValues, + color: emotionColors[emotionId] || '#888', + smooth: true, + lineStyle: { + width: 2 + }, + itemStyle: { + borderWidth: 1 + } + }; + }); + const xData = emotionData.emotionData.map((_, index) => index + 1); + const chartConfig = { + xData: xData, + seriesData: seriesData, + yLabel: 'Emotion Value', + }; + const tooltipFormatter = function (params) { + const index = params[0].dataIndex; + const data = [...emotionData.emotionData[index]].sort((a, b) => b.emotionValue - a.emotionValue); + let tooltipContent = 'Emotion Segment ' + (index + 1) + '
'; + for (const emotion of data) { + const emotionType = emotionData.emotionTypes.find(e => e.id === emotion.emotionType); + tooltipContent += '' + emotionType.name + ': ' + emotion.emotionValue.toFixed(2) + '
'; + } + const elementId = 'emot-E-' + data[0].emotionId; + const element = document.getElementById(elementId); + if (element) { + tooltipContent += 'Content:
'; + const content = $(element).attr('data-wcovered'); + tooltipContent += '' + content + ''; + } + return tooltipContent; + }; + + if (container.classList.contains('rendered')) { + const chartId = container.getAttribute('chart-id'); + if (!chartId) { + console.error('No chart ID found for container:', containerId); + return; + } + window.graphVizHandler.updateLineChart( + chartId, + '', + chartConfig, + tooltipFormatter + ); + const chart = window.graphVizHandler.getChartById(chartId); + if (!chart) { + console.error('Chart not found for ID:', chartId); + return; + } + // remove old event listeners + chart.getInstance().off('mouseover'); + chart.getInstance().off('mouseout'); + chart.getInstance().off('click'); + + // add new event listeners + chart.getInstance().on('mouseover', function (params) { + // remove previous highlights to be safe + $('.emotion-covered.highlight').removeClass('highlight'); + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.addClass('highlight'); + }); + chart.getInstance().on('mouseout', function (params) { + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.removeClass('highlight'); + }); + chart.getInstance().on('click', function (params) { + const index = params.dataIndex; + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } + }); + container.classList.remove('dirty'); + } + else { + window.graphVizHandler.createLineChart( + containerId, + '', + chartConfig, + tooltipFormatter + ).then(chart => { + chart.getInstance().on('mouseover', function (params) { + // remove previous highlights to be safe + $('.emotion-covered.highlight').removeClass('highlight'); + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.addClass('highlight'); + }); + chart.getInstance().on('mouseout', function (params) { + const index = params.dataIndex; + const elementId = 'emot-E-' + emotionData.emotionData[index][0].emotionId; + const element = $('#' + elementId); + element.removeClass('highlight'); + }); + chart.getInstance().on('click', function (params) { + const index = params.dataIndex; + const emotionId = emotionData.emotionData[index][0].emotionId; + const elementId = 'emot-E-' + emotionId; + const element = document.getElementById(elementId); + if (element) { + element.scrollIntoView({behavior: 'smooth', block: 'center'}); + clearTopicColoring(); + hideTopicNavButtons(); + $('.scrollbar-minimap').hide(); + } + }); + $(container).on('mouseout', function () { + $('.emotion-covered.highlight').removeClass('highlight'); + }); + $(container).attr('chart-id', chart.getChartId()); + }); + + container.classList.add('rendered'); + } + }).catch(() => { + $('.visualization-spinner').hide(); + const container = document.getElementById(containerId); + if (container) { + container.innerHTML = '
' + document.getElementById('viz-content').getAttribute('data-message') + '
'; + } + container.classList.add('rendered'); + }); +} + function renderTemporalExplorer(containerId) { const container = document.getElementById(containerId); @@ -1384,11 +1594,11 @@ function renderTemporalExplorer(containerId) { $('.visualization-spinner').show() const docId = document.getElementsByClassName('reader-container')[0].getAttribute('data-id'); - const taxonReq = $.get('/api/document/page/taxon', { documentId: docId }); - const topicReq = $.get('/api/document/page/topics', { documentId: docId }); - const entityReq = $.get('/api/document/page/namedEntities', { documentId: docId }); - const lemmaReq = $.get('/api/document/page/lemma', { documentId: docId }); - const geonameReq = $.get('/api/document/page/geoname', { documentId: docId }); + const taxonReq = $.get('/api/document/page/taxon', {documentId: docId}); + const topicReq = $.get('/api/document/page/topics', {documentId: docId}); + const entityReq = $.get('/api/document/page/namedEntities', {documentId: docId}); + const lemmaReq = $.get('/api/document/page/lemma', {documentId: docId}); + const geonameReq = $.get('/api/document/page/geoname', {documentId: docId}); Promise.all([taxonReq, topicReq, entityReq, lemmaReq, geonameReq]).then(([taxon, topics, entities, lemma, geoname]) => { $('.visualization-spinner').hide() @@ -1447,7 +1657,7 @@ function renderTemporalExplorer(containerId) { // Collect unique sorted page IDs const rawPageIds = []; - annotationSources.forEach(({ data, pageField }) => { + annotationSources.forEach(({data, pageField}) => { data.forEach(d => { const pid = parseInt(d[pageField]); if (!isNaN(pid)) rawPageIds.push(pid); @@ -1462,7 +1672,7 @@ function renderTemporalExplorer(containerId) { const dataMap = new Map(); - annotationSources.forEach(({ key, data, pageField, valueField, transformValue }) => { + annotationSources.forEach(({key, data, pageField, valueField, transformValue}) => { data.forEach(item => { const pid = parseInt(item[pageField]); const page = pageIdToPageNumber.get(pid); @@ -1490,10 +1700,10 @@ function renderTemporalExplorer(containerId) { const pages = sorted.map(row => row.page); const seriesData = annotationSources - .map(({ key, label, color }) => { + .map(({key, label, color}) => { const data = sorted.map(row => row[key]?.length || 0); const hasNonZero = data.some(count => count > 0); - return hasNonZero ? { name: label, data, color } : null; + return hasNonZero ? {name: label, data, color} : null; }) .filter(d => d !== null); @@ -1514,7 +1724,7 @@ function renderTemporalExplorer(containerId) { let tooltipHtml = '
Page ' + page + '
'; - annotationSources.forEach(({ key, label, color }) => { + annotationSources.forEach(({key, label, color}) => { if (!seriesNames.has(label)) return; const items = record[key]; if (!items || items.length === 0) return; @@ -1555,7 +1765,7 @@ function renderTemporalExplorer(containerId) { const pageNumber = params.name; const pageElement = document.querySelector('.page[data-id="' + pageNumber + '"]'); if (pageElement) { - pageElement.scrollIntoView({ behavior: 'smooth', block: 'start' }); + pageElement.scrollIntoView({behavior: 'smooth', block: 'start'}); } else { console.error(`Page ` + pageNumber + ` not found.`); } diff --git a/uce.portal/resources/templates/js/graphViz.js b/uce.portal/resources/templates/js/graphViz.js index 366de940..f7401364 100644 --- a/uce.portal/resources/templates/js/graphViz.js +++ b/uce.portal/resources/templates/js/graphViz.js @@ -132,7 +132,7 @@ var GraphVizHandler = (function () { return getColorForWeight(weight); } - GraphVizHandler.prototype.createSankeyChart = async function (target, title, linksData, nodesData,onClick = null) { + GraphVizHandler.prototype.createSankeyChart = async function (target, title, linksData, nodesData, onClick = null) { const chartId = generateUUID(); const option = { @@ -223,11 +223,10 @@ var GraphVizHandler = (function () { }] }); - } - else { + } else { const resetNodes = nodesData.map(node => ({ ...node, - label: { show: false }, + label: {show: false}, // itemStyle: { // ...(node.itemStyle || {}), // opacity: 0.5 @@ -296,8 +295,7 @@ var GraphVizHandler = (function () { target, title, config, - tooltipFormatter, - onClick = null + tooltipFormatter ) { const chartId = generateUUID(); @@ -364,8 +362,8 @@ var GraphVizHandler = (function () { data: s.data, symbol: 'circle', symbolSize: 10, - lineStyle: { width: 3, color: s.color }, - itemStyle: { color: s.color }, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, z: 2 }); }); @@ -379,7 +377,141 @@ var GraphVizHandler = (function () { return echart; }; + GraphVizHandler.prototype.createLineChart = async function ( + target, + title, + config, + tooltipFormatter, + onClick = null + ) { + const chartId = generateUUID(); + const { + xData, + seriesData, + yLabel = 'Count' + } = config; + + const option = { + tooltip: { + trigger: 'axis', + enterable: true, + backgroundColor: '#fff', + borderColor: '#ccc', + borderWidth: 1, + textStyle: { + color: '#000', + fontSize: 12 + }, + formatter: tooltipFormatter + }, + + title: { + text: title, + left: 'center' + }, + + legend: { + data: seriesData.map(s => s.name), + top: 'auto' + }, + + xAxis: { + type: 'category', + name: 'X', + data: xData + }, + + yAxis: { + type: 'value', + name: yLabel + }, + + series: [] + }; + + seriesData.forEach(s => { + option.series.push({ + name: s.name, + type: 'line', + data: s.data, + symbol: 'circle', + symbolSize: 10, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, + z: 2 + }); + }); + + const echart = new ECharts(target, option); + this.activeCharts[chartId] = echart; + echart.setChartId(chartId); + + return echart; + }; + + GraphVizHandler.prototype.updateLineChart = function ( + chartId, + title, + config, + tooltipFormatter + ) { + const echart = this.getChartById(chartId); + if (!echart) { + console.warn('No chart found with ID:', chartId); + return; + } + const { + xData, + seriesData, + yLabel = 'Count' + } = config; + const option = { + title: { + text: title, + left: 'center' + }, + tooltip: { + trigger: 'axis', + enterable: true, + backgroundColor: '#fff', + borderColor: '#ccc', + borderWidth: 1, + textStyle: { + color: '#000', + fontSize: 12 + }, + formatter: tooltipFormatter + }, + legend: { + data: seriesData.map(s => s.name), + top: 'auto' + }, + xAxis: { + type: 'category', + name: 'X', + data: xData + }, + yAxis: { + type: 'value', + name: yLabel + }, + series: [] + }; + seriesData.forEach(s => { + option.series.push({ + name: s.name, + type: 'line', + data: s.data, + symbol: 'circle', + symbolSize: 10, + lineStyle: {width: 3, color: s.color}, + itemStyle: {color: s.color}, + z: 2 + }); + }); + echart.updateOption(option); + } GraphVizHandler.prototype.createChordChart = async function ( target, @@ -392,7 +524,7 @@ var GraphVizHandler = (function () { const hasGraphData = data.nodes && data.links && data.categories; const option = { - title: { text: title, left: 'center' }, + title: {text: title, left: 'center'}, tooltip: { trigger: 'item', enterable: hasGraphData, @@ -414,15 +546,15 @@ var GraphVizHandler = (function () { series: hasGraphData ? [{ type: 'graph', layout: 'circular', - circular: { rotateLabel: true }, + circular: {rotateLabel: true}, data: data.nodes, links: data.links, categories: data.categories, roam: true, - label: { rotate: 90, show: true }, - itemStyle: { borderWidth: 1, borderColor: '#aaa' }, - lineStyle: { opacity: 0.5, width: 2, curveness: 0.3 }, - emphasis: { focus: 'adjacency', label: { show: true } } + label: {rotate: 90, show: true}, + itemStyle: {borderWidth: 1, borderColor: '#aaa'}, + lineStyle: {opacity: 0.5, width: 2, curveness: 0.3}, + emphasis: {focus: 'adjacency', label: {show: true}} }] : { type: 'chord', data: data.nodes, @@ -458,7 +590,7 @@ var GraphVizHandler = (function () { title, matrix, labels, - series_name= null, + series_name = null, tooltipFormatter = null, onClick = null ) { @@ -541,10 +673,10 @@ var GraphVizHandler = (function () { ) { const chartId = generateUUID(); const option = { - title: { text: '', left: 'center' }, + title: {text: '', left: 'center'}, tooltip: {}, - xAxis: { show: false, min: 'dataMin', max: 'dataMax' }, - yAxis: { show: false, min: 'dataMin', max: 'dataMax' }, + xAxis: {show: false, min: 'dataMin', max: 'dataMax'}, + yAxis: {show: false, min: 'dataMin', max: 'dataMax'}, animationDuration: 1500, animationEasingUpdate: 'quinticInOut', series: [{ @@ -560,7 +692,7 @@ var GraphVizHandler = (function () { edges: links, roam: true, symbolSize: 10, - label: { show: false }, + label: {show: false}, emphasis: { focus: 'adjacency', lineStyle: { @@ -568,7 +700,7 @@ var GraphVizHandler = (function () { } }, - itemStyle: { borderColor: '#fff', borderWidth: 1 } + itemStyle: {borderColor: '#fff', borderWidth: 1} }] }; @@ -585,6 +717,19 @@ var GraphVizHandler = (function () { return echart; }; + GraphVizHandler.prototype.getActiveCharts = function () { + return this.activeCharts; + } + + GraphVizHandler.prototype.getChartById = function (chartId) { + if (chartId in this.activeCharts) { + return this.activeCharts[chartId]; + } else { + console.warn('No chart found with ID:', chartId); + return null; + } + } + return GraphVizHandler; }()); diff --git a/uce.portal/resources/templates/reader/documentReaderView.ftl b/uce.portal/resources/templates/reader/documentReaderView.ftl index ee05a3f5..6ca96e8e 100644 --- a/uce.portal/resources/templates/reader/documentReaderView.ftl +++ b/uce.portal/resources/templates/reader/documentReaderView.ftl @@ -259,6 +259,32 @@

+ + + <#if modelCategories?has_content && modelCategories?size gt 0> +
+

${languageResource.get("modelSelection")}

+
+ <#list modelCategories as category> +
+ + +
+ +
+
+ @@ -300,6 +326,9 @@
+
+
+
@@ -308,6 +337,7 @@ +
diff --git a/uce.portal/resources/templates/wiki/components/modelInfo.ftl b/uce.portal/resources/templates/wiki/components/modelInfo.ftl new file mode 100644 index 00000000..3f71413e --- /dev/null +++ b/uce.portal/resources/templates/wiki/components/modelInfo.ftl @@ -0,0 +1,21 @@ +<#if model.getModelVersion()??> + <#assign modelVersionObject = model.getModelVersion()> + <#assign modelObject = modelVersionObject.getModel()> + <#assign modelVersion = modelVersionObject.getVersion()!> + <#assign modelName = modelObject.getName()!> +<#else> + <#assign modelVersion = "N/A"> + <#assign modelName = "N/A"> + +
+
+
+ + +
+
+
\ No newline at end of file diff --git a/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl new file mode 100644 index 00000000..574d15e2 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/emotionAnnotationPage.ftl @@ -0,0 +1,61 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+ +
+ + +
+ <#assign emotion = vm.getWikiModel()> +
+ <#list emotion.collectEmotionValues() as emotionValuePair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ +
diff --git a/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl new file mode 100644 index 00000000..d960757e --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/offensiveSpeechAnnotationPage.ftl @@ -0,0 +1,61 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+ +
+ + +
+ <#assign offensiveSpeech = vm.getWikiModel()> +
+ <#list offensiveSpeech.collectOffensiveSpeechValues() as propertyPair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ +
diff --git a/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl new file mode 100644 index 00000000..db0e6fa5 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/sentimentAnnotationPage.ftl @@ -0,0 +1,62 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+ +
+ + +
+ <#assign sentiment = vm.getWikiModel()> +
+ <#list sentiment.loopThroughProperties() as propertyPair> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ + +
diff --git a/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl new file mode 100644 index 00000000..63b8c638 --- /dev/null +++ b/uce.portal/resources/templates/wiki/pages/toxicAnnotationPage.ftl @@ -0,0 +1,61 @@ +
+ + +
+ <#include "*/wiki/components/breadcrumbs.ftl"> +
+ + +
+ <#include "*/wiki/components/metadata.ftl"> +
+ + +
+ <#assign model = vm.getWikiModel()> + <#include "*/wiki/components/modelInfo.ftl"> +
+ +
+ + +
+ <#assign toxic = vm.getWikiModel()> +
+ <#list toxic.getToxicValues() as tv> +
+
+ + +
+
+ +
+
+ + +
+
+ <#assign document = vm.getDocument()> + <#assign searchId = ""> + <#assign reduced = true> + <#include '*/search/components/documentCardContent.ftl' > +
+
+ + +
+ <#assign unique = (vm.getWikiModel().getUnique())!"none"> + <#assign height = 500> + <#if unique != "none"> +
+ <#include "*/wiki/components/linkableSpace.ftl"> +
+ +
+ +
diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java index 1f1da164..a8cade36 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/HibernateConf.java @@ -13,14 +13,29 @@ import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; +import org.texttechnologylab.models.emotion.EmotionValue; import org.texttechnologylab.models.gbif.GbifOccurrence; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; +import org.texttechnologylab.models.sentiment.SentimentValue; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; +import org.texttechnologylab.models.toxic.ToxicType; +import org.texttechnologylab.models.toxic.ToxicValue; import java.util.HashMap; @@ -80,6 +95,26 @@ public static SessionFactory buildSessionFactory() { metadataSources.addAnnotatedClass(TopicWord.class); metadataSources.addAnnotatedClass(TopicValueBase.class); metadataSources.addAnnotatedClass(TopicValueBaseWithScore.class); + // sentiment + metadataSources.addAnnotatedClass(Sentiment.class); + metadataSources.addAnnotatedClass(SentimentType.class); + metadataSources.addAnnotatedClass(SentimentValue.class); + // offensive speech + metadataSources.addAnnotatedClass(OffensiveSpeech.class); + metadataSources.addAnnotatedClass(OffensiveSpeechValue.class); + metadataSources.addAnnotatedClass(OffensiveSpeechType.class); + // toxic + metadataSources.addAnnotatedClass(Toxic.class); + metadataSources.addAnnotatedClass(ToxicValue.class); + metadataSources.addAnnotatedClass(ToxicType.class); + // emotion + metadataSources.addAnnotatedClass(Emotion.class); + metadataSources.addAnnotatedClass(EmotionValue.class); + metadataSources.addAnnotatedClass(EmotionType.class); + // model info + metadataSources.addAnnotatedClass(Model.class); + metadataSources.addAnnotatedClass(ModelVersion.class); + metadataSources.addAnnotatedClass(ModelCategory.class); metadataSources.addAnnotatedClass(DocumentTopThreeTopics.class); var metadata = metadataSources.buildMetadata(); diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java index ec588bd3..33c133f4 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/config/corpusConfig/CorpusAnnotationConfig.java @@ -24,6 +24,10 @@ public class CorpusAnnotationConfig { private boolean scope; private boolean xscope; private boolean unifiedTopic; + private boolean sentiment; + private boolean offensiveSpeech; + private boolean emotion; + private boolean toxic; public boolean isGeoNames() { return geoNames; @@ -202,5 +206,19 @@ public void setXscope(boolean xscope) { public boolean isUnifiedTopic() { return unifiedTopic; } + public boolean isSentiment() { + return sentiment; + } + + public boolean isOffensiveSpeech() { + return offensiveSpeech; + } + public boolean isEmotion() { + return emotion; + } + + public boolean isToxic(){ + return toxic; + } } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java index 96f53ca3..467fddef 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/UIMAAnnotation.java @@ -1,21 +1,22 @@ package org.texttechnologylab.models; -import io.micrometer.common.lang.Nullable; import lombok.Getter; import lombok.Setter; -import org.texttechnologylab.models.biofid.BiofidTaxon; import org.texttechnologylab.models.corpus.*; import org.texttechnologylab.models.corpus.links.AnnotationLink; import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; -import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.utils.StringUtils; import javax.persistence.*; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; @MappedSuperclass public class UIMAAnnotation extends ModelBase implements Linkable { @@ -51,6 +52,12 @@ public long getPrimaryDbIdentifier() { @Column(name = "page_id", insertable = false, updatable = false) private Long pageId; + @Getter + @Setter + @ManyToOne(fetch = FetchType.LAZY, optional = true) + @JoinColumn(name = "model_version_id", nullable = true) + private ModelVersion modelVersion; + public String getCoveredText() { if (coveredText == null) { return ""; // Or return an empty string "" if that's preferred @@ -128,6 +135,18 @@ public String buildHTMLString(List annotations, String coveredTe Map topicMarkers = new TreeMap<>(); Map topicCoverWrappersStart = new TreeMap<>(); Map topicCoverWrappersEnd = new TreeMap<>(); + Map emotionMarkers = new TreeMap<>(); + Map emotionCoverWrappersStart = new TreeMap<>(); + Map emotionCoverWrappersEnd = new TreeMap<>(); + Map offensiveSpeechMarkers = new TreeMap<>(); + Map offensiveSpeechCoverWrappersStart = new TreeMap<>(); + Map offensiveSpeechCoverWrappersEnd = new TreeMap<>(); + Map toxicMarkers = new TreeMap<>(); + Map toxicCoverWrappersStart = new TreeMap<>(); + Map toxicCoverWrappersEnd = new TreeMap<>(); + Map sentimentMarkers = new TreeMap<>(); + Map sentimentCoverWrappersStart = new TreeMap<>(); + Map sentimentCoverWrappersEnd = new TreeMap<>(); for (var annotation : annotations) { @@ -168,6 +187,50 @@ public String buildHTMLString(List annotations, String coveredTe continue; } + if (annotation instanceof Emotion emotion) { + var start = emotion.getBegin() - offset - errorOffset; + var end = emotion.getEnd() - offset - errorOffset; + + emotionCoverWrappersStart.put(start, emotion.generateEmotionCoveredStartSpan()); + emotionCoverWrappersEnd.put(end, ""); + + emotionMarkers.put(end, emotion.generateEmotionMarker()); + continue; + } + + if (annotation instanceof OffensiveSpeech offensiveSpeech) { + var start = offensiveSpeech.getBegin() - offset - errorOffset; + var end = offensiveSpeech.getEnd() - offset - errorOffset; + + offensiveSpeechCoverWrappersStart.put(start, offensiveSpeech.generateOffensiveSpeechCoveredStartSpan()); + offensiveSpeechCoverWrappersEnd.put(end, ""); + + offensiveSpeechMarkers.put(end, offensiveSpeech.generateOffensiveSpeechMarker()); + continue; + } + + if (annotation instanceof Toxic toxic) { + var start = toxic.getBegin() - offset - errorOffset; + var end = toxic.getEnd() - offset - errorOffset; + + toxicCoverWrappersStart.put(start, toxic.generateToxicCoveredStartSpan()); + toxicCoverWrappersEnd.put(end, ""); + + toxicMarkers.put(end, toxic.generateToxicMarker()); + continue; + } + + if (annotation instanceof Sentiment sentiment) { + var start = sentiment.getBegin() - offset - errorOffset; + var end = sentiment.getEnd() - offset - errorOffset; + + sentimentCoverWrappersStart.put(start, sentiment.generateSentimentCoveredStartSpan()); + sentimentCoverWrappersEnd.put(end, ""); + + sentimentMarkers.put(end, sentiment.generateSentimentMarker()); + continue; + } + var start = annotation.getBegin() - offset - errorOffset; var end = annotation.getEnd() - offset - errorOffset; @@ -187,6 +250,18 @@ public String buildHTMLString(List annotations, String coveredTe if (topicCoverWrappersEnd.containsKey(i)) { finalText.append(topicCoverWrappersEnd.get(i)); } + if (emotionCoverWrappersEnd.containsKey(i)) { + finalText.append(emotionCoverWrappersEnd.get(i)); + } + if (offensiveSpeechCoverWrappersEnd.containsKey(i)) { + finalText.append(offensiveSpeechCoverWrappersEnd.get(i)); + } + if (toxicCoverWrappersEnd.containsKey(i)) { + finalText.append(toxicCoverWrappersEnd.get(i)); + } + if (sentimentCoverWrappersEnd.containsKey(i)) { + finalText.append(sentimentCoverWrappersEnd.get(i)); + } if (endTags.containsKey(i)) { //finalText.append(endTags.get(i).getFirst()); for (var tag : endTags.get(i)) { @@ -195,6 +270,18 @@ public String buildHTMLString(List annotations, String coveredTe } // Insert start spans + if (sentimentCoverWrappersStart.containsKey(i)) { + finalText.append(sentimentCoverWrappersStart.get(i)); + } + if (toxicCoverWrappersStart.containsKey(i)) { + finalText.append(toxicCoverWrappersStart.get(i)); + } + if (offensiveSpeechCoverWrappersStart.containsKey(i)) { + finalText.append(offensiveSpeechCoverWrappersStart.get(i)); + } + if (emotionCoverWrappersStart.containsKey(i)) { + finalText.append(emotionCoverWrappersStart.get(i)); + } if (topicCoverWrappersStart.containsKey(i)) { finalText.append(topicCoverWrappersStart.get(i)); } @@ -203,6 +290,18 @@ public String buildHTMLString(List annotations, String coveredTe if (topicMarkers.containsKey(i)) { finalText.append(topicMarkers.get(i)); } + if (emotionMarkers.containsKey(i)) { + finalText.append(emotionMarkers.get(i)); + } + if (offensiveSpeechMarkers.containsKey(i)) { + finalText.append(offensiveSpeechMarkers.get(i)); + } + if (toxicMarkers.containsKey(i)) { + finalText.append(toxicMarkers.get(i)); + } + if (sentimentMarkers.containsKey(i)) { + finalText.append(sentimentMarkers.get(i)); + } if (startTags.containsKey(i)) { finalText.append(generateMultiHTMLTag(startTags.get(i))); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java index d9d898b4..f6b69dfa 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/corpus/Document.java @@ -19,13 +19,15 @@ import org.texttechnologylab.models.corpus.links.AnnotationToDocumentLink; import org.texttechnologylab.models.corpus.links.DocumentLink; import org.texttechnologylab.models.corpus.links.DocumentToAnnotationLink; -import org.texttechnologylab.models.corpus.links.Link; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.modelInfo.ModelNameHelper; import org.texttechnologylab.models.negation.*; -import org.texttechnologylab.models.search.AnnotationSearchResult; -import org.texttechnologylab.models.search.PageSnippet; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.sentiment.Sentiment; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.utils.FreemarkerUtils; import org.texttechnologylab.utils.StringUtils; @@ -218,6 +220,30 @@ public long getPrimaryDbIdentifier() { @Fetch(value = FetchMode.SUBSELECT) private List unifiedTopics; + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List offensiveSpeeches; + + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List toxics; + + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List emotions; + + @Setter + @Getter + @OneToMany(mappedBy = "document", cascade = CascadeType.ALL, fetch = FetchType.EAGER) + @Fetch(value = FetchMode.SUBSELECT) + private List sentiments; + public Document() { metadataTitleInfo = new MetadataTitleInfo(); } @@ -343,7 +369,7 @@ public List getAllTaxa(){ * * @return */ - public List getAllAnnotations(int pagesSkip, int pagesTake) { + public List getAllAnnotations(int pagesSkip, int pagesTake, Map modelSelection) { var pagesBegin = getPages().stream().skip(pagesSkip).limit(1).findFirst().get().getBegin(); var pagesEnd = getPages().stream().skip(Math.min(pagesSkip + pagesTake, getPages().size() - 1)).limit(1).findFirst().get().getEnd(); @@ -364,6 +390,18 @@ public List getAllAnnotations(int pagesSkip, int pagesTake) { annotations.addAll(events.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); // unifiedTopics annotations.addAll(unifiedTopics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).toList()); + // emotions + int emotionModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Emotion.class), -1); + annotations.addAll(emotions.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> emotionModelId == -1 || e.getModelVersion().getModel().getId() == emotionModelId).toList()); + // offensiveSpeech + int offensiveSpeechModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(OffensiveSpeech.class), -1); + annotations.addAll(offensiveSpeeches.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> offensiveSpeechModelId == -1 || e.getModelVersion().getModel().getId() == offensiveSpeechModelId).toList()); + // toxic + int toxicModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Toxic.class), -1); + annotations.addAll(toxics.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> toxicModelId == -1 || e.getModelVersion().getModel().getId() == toxicModelId).toList()); + // sentiment + int sentimentModelId = modelSelection.getOrDefault(ModelNameHelper.getModelName(Sentiment.class), -1); + annotations.addAll(sentiments.stream().filter(a -> a.getBegin() >= pagesBegin && a.getEnd() <= pagesEnd).filter(e -> sentimentModelId == -1 || e.getModelVersion().getModel().getId() == sentimentModelId).toList()); annotations.sort(Comparator.comparingInt(UIMAAnnotation::getBegin)); return annotations; @@ -428,4 +466,21 @@ public List getDocumentUnifiedTopicDistribution(Integer topN) { .collect(Collectors.toList()); } + public List getModelCategories() { + List categories = new ArrayList<>(); + if (!emotions.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Emotion.class)); + } + if (!offensiveSpeeches.isEmpty()) { + categories.add(ModelNameHelper.getModelName(OffensiveSpeech.class)); + } + if (!toxics.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Toxic.class)); + } + if (!sentiments.isEmpty()) { + categories.add(ModelNameHelper.getModelName(Sentiment.class)); + } + return categories; + } + } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java new file mode 100644 index 00000000..f0649d26 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/Emotion.java @@ -0,0 +1,94 @@ +package org.texttechnologylab.models.emotion; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.annotations.Typesystem; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; +import org.texttechnologylab.utils.Pair; + +import javax.persistence.*; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +@Entity +@Table(name = "emotion") +@Typesystem(types = {org.texttechnologylab.annotation.Emotion.class}) +@NamedModel(name = "emotion") +public class Emotion extends UIMAAnnotation implements WikiModel { + + @Getter + @Setter + @OneToMany(mappedBy = "emotion", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + @Fetch(value = FetchMode.SUBSELECT) + private List emotionValues; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Emotion() { + super(-1, -1); + } + + public Emotion(int begin, int end) { + super(begin, end); + } + + public Emotion(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + private EmotionValue getRepresentativeEmotionValue() { + if (this.emotionValues != null && !this.emotionValues.isEmpty()) { + return this.emotionValues.stream().max(Comparator.comparingDouble(EmotionValue::getValue)).orElse(null); + } + return null; + } + + public String generateEmotionMarker() { + String repEmotionValue = ""; + if (this.getEmotionValues() != null && !this.getEmotionValues().isEmpty()) { + var repEmotion = this.getRepresentativeEmotionValue(); + if (repEmotion != null) { + repEmotionValue = repEmotion.getEmotionType().getName(); + } + } + return String.format("e", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repEmotionValue); + } + + public String generateEmotionCoveredStartSpan() { + String repEmotionValue = ""; + if (this.getEmotionValues() != null && !this.getEmotionValues().isEmpty()) { + var repEmotion = this.getRepresentativeEmotionValue(); + if (repEmotion != null) { + repEmotionValue = repEmotion.getEmotionType().getName(); + } + } + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repEmotionValue); + } + + public List> collectEmotionValues() { + return this.emotionValues.stream().map(ev -> new Pair<>(ev.getEmotionType().getName(), ev.getValue())).toList(); + } + + @Override + public String getWikiId() { + return "E" + "-" + this.getId(); + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java new file mode 100644 index 00000000..f3cc3061 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionType.java @@ -0,0 +1,28 @@ +package org.texttechnologylab.models.emotion; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "emotion_type") +public class EmotionType extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public EmotionType() { + // Default constructor for JPA + } + + public EmotionType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java new file mode 100644 index 00000000..5ed6d93b --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/emotion/EmotionValue.java @@ -0,0 +1,62 @@ +package org.texttechnologylab.models.emotion; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; + +import javax.persistence.*; + +@Entity +@Table(name = "emotion_value") +public class EmotionValue extends UIMAAnnotation implements WikiModel { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "emotion_type_id", nullable = false) + private EmotionType emotionType; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "emotion_id", nullable = false) + private Emotion emotion; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public EmotionValue() { + super(-1, -1); + } + + public EmotionValue(int begin, int end) { + super(begin, end); + } + + public EmotionValue(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + @Override + public String getWikiId() { + return "EV" + "-" + this.getId(); + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java new file mode 100644 index 00000000..38ffc831 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/Model.java @@ -0,0 +1,38 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.ManyToMany; +import javax.persistence.Table; +import java.util.Set; + +@Entity +@Table(name = "model") +public class Model extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + @Getter + @Setter + @ManyToMany + @Fetch(FetchMode.JOIN) + private Set categories; + + public Model() { + // Default constructor for JPA + } + + public Model(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java new file mode 100644 index 00000000..37c6f68d --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelCategory.java @@ -0,0 +1,38 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.ManyToMany; +import javax.persistence.Table; +import java.util.Set; + +@Entity +@Table(name = "model_category") +public class ModelCategory extends ModelBase { + + @Getter + @Setter + @Column(name = "category_name", nullable = false, unique = true) + private String categoryName; + + @Getter + @Setter + @ManyToMany(mappedBy = "categories") + @Fetch(FetchMode.JOIN) + private Set models; + + public ModelCategory() { + // Default constructor for JPA + } + + public ModelCategory(String categoryName) { + this.categoryName = categoryName; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java new file mode 100644 index 00000000..10d47da6 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelNameHelper.java @@ -0,0 +1,23 @@ +package org.texttechnologylab.models.modelInfo; + +import org.jetbrains.annotations.NotNull; + +public class ModelNameHelper { + + private ModelNameHelper() { + // Utility class, no instantiation allowed + } + + @NotNull + public static String getModelName(@NotNull Class modelClass) { + NamedModel namedModel = modelClass.getAnnotation(NamedModel.class); + if (namedModel == null) { + throw new IllegalArgumentException("Class " + modelClass.getName() + " is not annotated with @NamedModel"); + } + String modelName = namedModel.name(); + if (modelName.isEmpty()) { + throw new IllegalArgumentException("Model name cannot be empty for class " + modelClass.getName()); + } + return modelName; + } +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java new file mode 100644 index 00000000..e189c680 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/ModelVersion.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.modelInfo; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "model_version", uniqueConstraints = @UniqueConstraint(columnNames = {"model_id", "version"})) +public class ModelVersion extends ModelBase { + + @Getter + @Setter + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "model_id", nullable = false) + private Model model; + + @Getter + @Setter + @Column(name = "version", nullable = false) + private String version; + + public ModelVersion() { + // Default constructor for JPA + } + + public ModelVersion(Model model, String version) { + this.model = model; + this.version = version; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java new file mode 100644 index 00000000..f90344f2 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/modelInfo/NamedModel.java @@ -0,0 +1,25 @@ +package org.texttechnologylab.models.modelInfo; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to specify the name of a model. + * This can be used to annotate classes that represent models + * in the Text Technology Lab framework. + * Use this to ensure consistency with the name of the model + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface NamedModel { + + /** + * The name of the model. + * + * @return the name of the model + */ + String name(); + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java new file mode 100644 index 00000000..7b9d2054 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeech.java @@ -0,0 +1,84 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.annotations.Typesystem; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; +import org.texttechnologylab.utils.Pair; + +import javax.persistence.*; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +@Entity +@Table(name = "offensivespeech") +@Typesystem(types = {org.texttechnologylab.annotation.OffensiveSpeech.class}) +@NamedModel(name = "offensivespeech") +public class OffensiveSpeech extends UIMAAnnotation implements WikiModel { + + @Getter + @Setter + @OneToMany(mappedBy = "offensiveSpeech", cascade = CascadeType.ALL, fetch = FetchType.LAZY) + @Fetch(FetchMode.SUBSELECT) + private List offensiveSpeechValues; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public OffensiveSpeech() { + super(-1, -1); + } + + public OffensiveSpeech(int begin, int end) { + super(begin, end); + } + + public OffensiveSpeech(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + private OffensiveSpeechValue getRepresentativeOffensiveSpeechValue() { + if (this.offensiveSpeechValues != null && !this.offensiveSpeechValues.isEmpty()) { + return this.offensiveSpeechValues.stream().max(Comparator.comparingDouble(OffensiveSpeechValue::getValue)).orElse(null); + } + return null; + } + + public String generateOffensiveSpeechMarker() { + OffensiveSpeechValue rep = getRepresentativeOffensiveSpeechValue(); + String repValue = rep != null ? rep.getOffensiveSpeechType().getName() : ""; + return String.format("os", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateOffensiveSpeechCoveredStartSpan() { + OffensiveSpeechValue rep = getRepresentativeOffensiveSpeechValue(); + String repValue = rep != null ? rep.getOffensiveSpeechType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + + @Override + public String getWikiId() { + return "OS" + "-" + this.getId(); + } + + public List> collectOffensiveSpeechValues() { + return this.offensiveSpeechValues.stream().map(value -> new Pair<>(value.getOffensiveSpeechType().getName(), value.getValue())).toList(); + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java new file mode 100644 index 00000000..d1bbad48 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechType.java @@ -0,0 +1,27 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "offensivespeech_type") +public class OffensiveSpeechType extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public OffensiveSpeechType() { + } + + public OffensiveSpeechType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java new file mode 100644 index 00000000..15bc1f1a --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/offensiveSpeech/OffensiveSpeechValue.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.offensiveSpeech; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "offensivespeech_value") +public class OffensiveSpeechValue extends ModelBase { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "offensivespeech_type_id", nullable = false) + private OffensiveSpeechType offensiveSpeechType; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "offensivespeech_id", nullable = false) + private OffensiveSpeech offensiveSpeech; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public OffensiveSpeechValue() { + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java new file mode 100644 index 00000000..acde4be4 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/Sentiment.java @@ -0,0 +1,92 @@ +package org.texttechnologylab.models.sentiment; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.annotations.Typesystem; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; +import org.texttechnologylab.utils.Pair; + +import javax.persistence.*; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +@Entity +@Table(name = "sentiment") +@Typesystem(types = {org.texttechnologylab.annotation.SentimentModel.class}) +@NamedModel(name = "sentiment") +public class Sentiment extends UIMAAnnotation implements WikiModel { + + @Getter + @Setter + @Column(name = "sentiment", nullable = false) + private int sentiment; + + @Getter + @Setter + @OneToMany(mappedBy = "sentiment", fetch = FetchType.LAZY, cascade = CascadeType.ALL) + @Fetch(value = FetchMode.SUBSELECT) + private List sentimentValues; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Sentiment() { + super(-1, -1); + } + + public Sentiment(int begin, int end) { + super(begin, end); + } + + public Sentiment(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + private SentimentValue getRepresentativeSentimentValue() { + if (this.sentimentValues != null && !this.sentimentValues.isEmpty()) { + return this.sentimentValues.stream() + .max(Comparator.comparingDouble(SentimentValue::getValue)) + .orElse(null); + } + return null; + } + + public String generateSentimentMarker() { + SentimentValue rep = getRepresentativeSentimentValue(); + String repValue = rep != null ? rep.getSentimentType().getName() : ""; + return String.format("s", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateSentimentCoveredStartSpan() { + SentimentValue rep = getRepresentativeSentimentValue(); + String repValue = rep != null ? rep.getSentimentType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + + @Override + public String getWikiId() { + return "S" + "-" + this.getId(); + } + + public List> loopThroughProperties() { + return sentimentValues.stream() + .map(v -> new Pair<>(v.getSentimentType().getName(), String.format("%.6f", v.getValue()))) + .toList(); + } +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java new file mode 100644 index 00000000..d1289b83 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentType.java @@ -0,0 +1,25 @@ +package org.texttechnologylab.models.sentiment; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "sentiment_type") +public class SentimentType extends ModelBase { + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public SentimentType() { + } + public SentimentType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java new file mode 100644 index 00000000..f782e1f4 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/sentiment/SentimentValue.java @@ -0,0 +1,29 @@ +package org.texttechnologylab.models.sentiment; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "sentiment_value") +public class SentimentValue extends ModelBase { + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "sentiment_id", nullable = false) + private Sentiment sentiment; + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "sentiment_type_id", nullable = false) + private SentimentType sentimentType; + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public SentimentValue() {} + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java new file mode 100644 index 00000000..e9a406a0 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/Toxic.java @@ -0,0 +1,78 @@ +package org.texttechnologylab.models.toxic; + +import lombok.Getter; +import lombok.Setter; +import org.hibernate.annotations.Fetch; +import org.hibernate.annotations.FetchMode; +import org.texttechnologylab.annotations.Typesystem; +import org.texttechnologylab.models.UIMAAnnotation; +import org.texttechnologylab.models.WikiModel; +import org.texttechnologylab.models.corpus.Document; +import org.texttechnologylab.models.modelInfo.NamedModel; + +import javax.persistence.*; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; + +@Entity +@Table(name = "toxic") +@Typesystem(types = {org.texttechnologylab.annotation.Toxic.class}) +@NamedModel(name = "toxic") +public class Toxic extends UIMAAnnotation implements WikiModel { + + @Getter + @Setter + @OneToMany(mappedBy = "toxic", fetch = FetchType.LAZY, cascade = CascadeType.ALL) + @Fetch(value = FetchMode.SUBSELECT) + private List toxicValues; + + @ManyToOne(fetch = FetchType.LAZY, optional = false, cascade = {CascadeType.PERSIST, CascadeType.MERGE}) + @JoinColumn(name = "document_id", nullable = false) + private Document document; + + public Toxic() { + super(-1, -1); + } + + public Toxic(int begin, int end) { + super(begin, end); + } + + public Toxic(int begin, int end, String coveredText) { + super(begin, end); + setCoveredText(coveredText); + } + + public Document getDocument() { + return document; + } + + public void setDocument(Document document) { + this.document = document; + } + + private ToxicValue getRepresentativeToxicValue() { + if (this.toxicValues != null && !this.toxicValues.isEmpty()) { + return this.toxicValues.stream().max(Comparator.comparingDouble(ToxicValue::getValue)).orElse(null); + } + return null; + } + + public String generateToxicMarker() { + ToxicValue rep = getRepresentativeToxicValue(); + String repValue = rep != null ? rep.getToxicType().getName() : ""; + return String.format("t", this.getWikiId(), this.getWikiId(), this.getCoveredText(), repValue); + } + + public String generateToxicCoveredStartSpan() { + ToxicValue rep = getRepresentativeToxicValue(); + String repValue = rep != null ? rep.getToxicType().getName() : ""; + return String.format("", UUID.randomUUID(), this.getWikiId(), this.getCoveredText(), repValue); + } + + @Override + public String getWikiId() { + return "T" + "-" + this.getId(); + } +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java new file mode 100644 index 00000000..8db08c50 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicType.java @@ -0,0 +1,26 @@ +package org.texttechnologylab.models.toxic; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Table; + +@Entity +@Table(name = "toxic_type") +public class ToxicType extends ModelBase { + + @Getter + @Setter + @Column(name = "name", nullable = false, unique = true) + private String name; + + public ToxicType() {} + + public ToxicType(String name) { + this.name = name; + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java new file mode 100644 index 00000000..756d2037 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/toxic/ToxicValue.java @@ -0,0 +1,33 @@ +package org.texttechnologylab.models.toxic; + +import lombok.Getter; +import lombok.Setter; +import org.texttechnologylab.models.ModelBase; + +import javax.persistence.*; + +@Entity +@Table(name = "toxic_value") +public class ToxicValue extends ModelBase { + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "toxic_id", nullable = false) + private Toxic toxic; + + @Getter + @Setter + @ManyToOne + @JoinColumn(name = "toxic_type_id", nullable = false) + private ToxicType toxicType; + + @Getter + @Setter + @Column(name = "value", nullable = false) + private double value; + + public ToxicValue() { + } + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java new file mode 100644 index 00000000..17514f20 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/EmotionWikiPageViewModel.java @@ -0,0 +1,6 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class EmotionWikiPageViewModel extends AnnotationWikiPageViewModel { + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java new file mode 100644 index 00000000..ea5b8c01 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/OffensiveSpeechAnnotationWikiPageViewModel.java @@ -0,0 +1,7 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class OffensiveSpeechAnnotationWikiPageViewModel extends AnnotationWikiPageViewModel{ + + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java new file mode 100644 index 00000000..fbe01749 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/SentimentWikiPageViewModel.java @@ -0,0 +1,9 @@ +package org.texttechnologylab.models.viewModels.wiki; + +import org.texttechnologylab.models.topic.TopicValueBase; + +import java.util.List; + +public class SentimentWikiPageViewModel extends AnnotationWikiPageViewModel { + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java new file mode 100644 index 00000000..808bc432 --- /dev/null +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/models/viewModels/wiki/ToxicAnnotationWikiPageViewModel.java @@ -0,0 +1,7 @@ +package org.texttechnologylab.models.viewModels.wiki; + +public class ToxicAnnotationWikiPageViewModel extends AnnotationWikiPageViewModel{ + + + +} diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java index b6cf739e..f0d814ea 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/LexiconService.java @@ -1,6 +1,5 @@ package org.texttechnologylab.services; -import io.micrometer.common.lang.Nullable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Service; @@ -8,19 +7,19 @@ import org.texttechnologylab.exceptions.DatabaseOperationException; import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.models.UIMAAnnotation; -import org.texttechnologylab.models.biofid.BiofidTaxon; import org.texttechnologylab.models.biofid.GazetteerTaxon; import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; +import org.texttechnologylab.models.emotion.Emotion; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; import org.texttechnologylab.models.viewModels.lexicon.LexiconOccurrenceViewModel; -import org.texttechnologylab.utils.SystemStatus; import javax.persistence.Table; -import java.sql.Array; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; @Service @@ -47,7 +46,11 @@ public class LexiconService { Cue.class, Scope.class, XScope.class, - UnifiedTopic.class)); + UnifiedTopic.class, + Sentiment.class, + OffensiveSpeech.class, + Toxic.class, + Emotion.class)); public LexiconService(PostgresqlDataInterface_Impl db) { this.db = db; diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java index a7482171..a6511c73 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/PostgresqlDataInterface_Impl.java @@ -8,6 +8,7 @@ import org.hibernate.criterion.Restrictions; import org.hibernate.type.LongType; import org.hibernate.type.StandardBasicTypes; +import org.jetbrains.annotations.NotNull; import org.springframework.stereotype.Service; import org.texttechnologylab.annotations.Searchable; import org.texttechnologylab.config.HibernateConf; @@ -20,20 +21,32 @@ import org.texttechnologylab.models.biofid.GazetteerTaxon; import org.texttechnologylab.models.biofid.GnFinderTaxon; import org.texttechnologylab.models.corpus.*; -import org.texttechnologylab.models.corpus.Time; import org.texttechnologylab.models.corpus.links.*; import org.texttechnologylab.models.dto.UCEMetadataFilterDto; import org.texttechnologylab.models.dto.map.MapClusterDto; import org.texttechnologylab.models.dto.map.PointDto; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; import org.texttechnologylab.models.gbif.GbifOccurrence; import org.texttechnologylab.models.globe.GlobeTaxon; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.UCEImport; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; +import org.texttechnologylab.models.modelInfo.ModelVersion; +import org.texttechnologylab.models.modelInfo.NamedModel; import org.texttechnologylab.models.negation.CompleteNegation; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.search.*; +import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; +import org.texttechnologylab.models.toxic.ToxicType; import org.texttechnologylab.models.util.HealthStatus; import org.texttechnologylab.utils.ReflectionUtils; import org.texttechnologylab.utils.StringUtils; @@ -43,7 +56,10 @@ import javax.persistence.criteria.Order; import javax.persistence.criteria.Path; import javax.persistence.criteria.Predicate; -import java.sql.*; +import java.sql.Array; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.*; import java.util.function.Function; @@ -434,12 +450,13 @@ public List getGlobeDataForDocument(long documentId) throws Database () -> getAllLinksOfLinkable(taxon.getId(), taxon.getClass(), List.of(AnnotationLink.class)) .stream() .filter(l -> l.getLinkId().equals("context") && l.getToAnnotationType().equals(GeoName.class.getName())).toList(), - (ex) -> { }); + (ex) -> { + }); // Foreach taxa, fetch a possible geoname link. if (links != null) for (var link : links) { var geoname = doc.getGeoNames().stream().filter(g -> g.getId() == link.getToId()).findFirst(); - if(geoname.isEmpty()) continue; + if (geoname.isEmpty()) continue; var globeTaxon = new GlobeTaxon(); globeTaxon.setLongitude(geoname.get().getLongitude()); globeTaxon.setLatitude(geoname.get().getLatitude()); @@ -981,7 +998,7 @@ public DocumentSearchResult defaultSearchForDocuments(int skip, return executeOperationSafely((session) -> session.doReturningWork((connection) -> { DocumentSearchResult search = null; try (var storedProcedure = connection.prepareCall("{call uce_search_layer_" + layer.name().toLowerCase() + - "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { + "(?::bigint, ?::text[], ?::text, ?::integer, ?::integer, ?::boolean, ?::text, ?::text, ?::jsonb, ?::boolean, ?::text, ?::text)}")) { storedProcedure.setInt(1, (int) corpusId); storedProcedure.setArray(2, connection.createArrayOf("text", searchTokens.stream().map(this::escapeSql).toArray())); storedProcedure.setString(3, ogSearchQuery); @@ -1200,9 +1217,9 @@ public List getDistinctTimesByCondition(String condition, long corpusId, return executeOperationSafely((session) -> { // Construct HQL dynamically (THIS IS UNSAFE BECAUSE OF THE CONDITION INSERTION) String hql = "SELECT DISTINCT t.coveredText " + - "FROM Time t " + - "JOIN Document d ON t.documentId = d.id " + - "WHERE " + condition + " AND d.corpusId = :corpusId"; + "FROM Time t " + + "JOIN Document d ON t.documentId = d.id " + + "WHERE " + condition + " AND d.corpusId = :corpusId"; var query = session.createQuery(hql, String.class); query.setParameter("corpusId", corpusId); @@ -1393,13 +1410,253 @@ public UnifiedTopic getInitializedUnifiedTopicById(long id) throws DatabaseOpera return executeOperationSafely((session) -> { var topic = session.get(UnifiedTopic.class, id); Hibernate.initialize(topic.getTopics()); - for(var t:topic.getTopics()){ + for (var t : topic.getTopics()) { Hibernate.initialize(t.getWords()); } return topic; }); } + public Sentiment getInitializedSentimentById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var sentiment = session.get(Sentiment.class, id); + Hibernate.initialize(sentiment.getSentimentValues()); + for (var s : sentiment.getSentimentValues()) { + Hibernate.initialize(s.getSentimentType()); + } + Hibernate.initialize(sentiment.getModelVersion()); + if (sentiment.getModelVersion() != null) { + Hibernate.initialize(sentiment.getModelVersion().getModel()); + } + return sentiment; + }); + } + + public synchronized SentimentType getOrCreateSentimentType(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(SentimentType.class); + var root = criteriaQuery.from(SentimentType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + + SentimentType sentimentType; + try { + sentimentType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + sentimentType = new SentimentType(name); + session.save(sentimentType); + } + return sentimentType; + }); + } + + public synchronized OffensiveSpeechType getOrCreateOffensiveSpeechType(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(OffensiveSpeechType.class); + var root = criteriaQuery.from(OffensiveSpeechType.class); + criteriaQuery.select(root) + .where(criteriaBuilder.equal(root.get("name"), name)); + + OffensiveSpeechType offensiveSpeechType; + try { + offensiveSpeechType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + // If not found, create a new one + offensiveSpeechType = new OffensiveSpeechType(name); + session.save(offensiveSpeechType); + } + return offensiveSpeechType; + }); + } + + public OffensiveSpeech getInitializedOffensiveSpeechById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var offensiveSpeech = session.get(OffensiveSpeech.class, id); + Hibernate.initialize(offensiveSpeech.getOffensiveSpeechValues()); + for (OffensiveSpeechValue value : offensiveSpeech.getOffensiveSpeechValues()) { + Hibernate.initialize(value.getOffensiveSpeechType()); + } + Hibernate.initialize(offensiveSpeech.getModelVersion()); + if (offensiveSpeech.getModelVersion() != null) { + Hibernate.initialize(offensiveSpeech.getModelVersion().getModel()); + } + return offensiveSpeech; + }); + } + + public Toxic getInitializedToxicById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var toxic = session.get(Toxic.class, id); + Hibernate.initialize(toxic.getToxicValues()); + for(var t:toxic.getToxicValues()){ + Hibernate.initialize(t.getToxicType()); + } + Hibernate.initialize(toxic.getModelVersion()); + if (toxic.getModelVersion() != null) { + Hibernate.initialize(toxic.getModelVersion().getModel()); + } + return toxic; + }); + } + + public synchronized ToxicType getOrCreateToxicType(String type) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ToxicType.class); + var root = criteriaQuery.from(ToxicType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), type)); + + ToxicType toxicType; + try { + toxicType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + toxicType = new ToxicType(type); + session.save(toxicType); + } + return toxicType; + }); + } + + public Emotion getInitializedEmotionById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var emotion = session.get(Emotion.class, id); + Hibernate.initialize(emotion.getEmotionValues()); + for (var ev : emotion.getEmotionValues()) { + Hibernate.initialize(ev.getEmotionType()); + } + Hibernate.initialize(emotion.getModelVersion()); + if (emotion.getModelVersion() != null) { + Hibernate.initialize(emotion.getModelVersion().getModel()); + } + return emotion; + }); + } + + public EmotionType getEmotionTypeById(long id) throws DatabaseOperationException { + return executeOperationSafely((session) -> session.get(EmotionType.class, id)); + } + + public EmotionType getEmotionTypeByName(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(EmotionType.class); + var root = criteriaQuery.from(EmotionType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + return session.createQuery(criteriaQuery).uniqueResult(); + }); + } + + public synchronized EmotionType getOrCreateEmotionTypeByName(String name) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(EmotionType.class); + var root = criteriaQuery.from(EmotionType.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), name)); + + EmotionType emotionType; + try { + emotionType = session.createQuery(criteriaQuery).getSingleResult(); + } catch (NoResultException e) { + // If not found, create a new one + emotionType = new EmotionType(name); + session.save(emotionType); + } + return emotionType; + }); + } + + public synchronized Model getOrCreateModel(String modelName) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(Model.class); + var root = criteriaQuery.from(Model.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("name"), modelName)); + + List models = session.createQuery(criteriaQuery).getResultList(); + if (models.isEmpty()) { + Model newModel = new Model(modelName); + session.save(newModel); + return newModel; + } else { + return models.getFirst(); + } + }); + } + + public synchronized ModelVersion getOrCreateModelVersion(Model model, String version) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ModelVersion.class); + var root = criteriaQuery.from(ModelVersion.class); + criteriaQuery.select(root).where( + criteriaBuilder.and( + criteriaBuilder.equal(root.get("model"), model), + criteriaBuilder.equal(root.get("version"), version) + ) + ); + + List versions = session.createQuery(criteriaQuery).getResultList(); + if (versions.isEmpty()) { + ModelVersion newVersion = new ModelVersion(model, version); + session.save(newVersion); + return newVersion; + } else { + return versions.getFirst(); + } + }); + } + + public ModelVersion getOrCreateModelVersion(String modelName, String version) throws DatabaseOperationException { + Model model = getOrCreateModel(modelName); + return getOrCreateModelVersion(model, version); + } + + public synchronized ModelCategory getOrCreateModelCategory(String categoryName) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + var criteriaBuilder = session.getCriteriaBuilder(); + var criteriaQuery = criteriaBuilder.createQuery(ModelCategory.class); + var root = criteriaQuery.from(ModelCategory.class); + criteriaQuery.select(root).where(criteriaBuilder.equal(root.get("categoryName"), categoryName)); + + List categories = session.createQuery(criteriaQuery).getResultList(); + if (categories.isEmpty()) { + ModelCategory newCategory = new ModelCategory(categoryName); + session.save(newCategory); + return newCategory; + } else { + return categories.getFirst(); + } + }); + } + + public ModelCategory getOrCreateModelCategory(@NotNull Class modelClass) throws DatabaseOperationException { + NamedModel namedModel = modelClass.getAnnotation(NamedModel.class); + if (namedModel == null) { + throw new IllegalArgumentException("The class " + modelClass.getName() + " is not annotated with @NamedModel."); + } + return getOrCreateModelCategory(namedModel.name()); + } + + public synchronized void registerModelToCategoryAssociation(Model model, ModelCategory category) throws DatabaseOperationException { + executeOperationSafely((session) -> { + // First, ensure the model and category are managed entities + Model managedModel = session.get(Model.class, model.getId()); + ModelCategory managedCategory = session.get(ModelCategory.class, category.getId()); + if (managedModel == null || managedCategory == null) { + throw new IllegalArgumentException("Model or Category not found in the database."); + } + // Add the category to the model's categories + managedModel.getCategories().add(managedCategory); + // Add the model to the category's models + managedCategory.getModels().add(managedModel); + // Save the changes + session.saveOrUpdate(managedModel); + return null; + }); + } + + public List getKeywordDistributionsByString(Class clazz, String topic, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { var builder = session.getCriteriaBuilder(); @@ -1641,9 +1898,9 @@ public List getTopTopicsByDocument(long documentId, int limit) throws return executeOperationSafely((session) -> { // Use native SQL to query the document_topics_raw table String sql = "SELECT topiclabel, thetadt FROM documenttopicsraw " + - "WHERE document_id = :documentId " + - "ORDER BY thetadt DESC " + - "LIMIT :limit"; + "WHERE document_id = :documentId " + + "ORDER BY thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId) @@ -1657,9 +1914,9 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws return executeOperationSafely((session) -> { // Direct query using sentence_id String sql = "SELECT topiclabel, thetast FROM sentencetopics " + - "WHERE sentence_id = :sentenceId " + - "ORDER BY thetast DESC " + - "LIMIT :limit"; + "WHERE sentence_id = :sentenceId " + + "ORDER BY thetast DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("sentenceId", sentenceId) @@ -1672,12 +1929,12 @@ public List getTopTopicsBySentence(long sentenceId, int limit) throws public List getTopDocumentsByTopicLabel(String topicValue, long corpusId, int limit) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT d.id, d.documentid, dtr.thetadt " + - "FROM document d " + - "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + - "WHERE dtr.topiclabel = :topicValue " + - "AND d.corpusid = :corpusId " + - "ORDER BY dtr.thetadt DESC " + - "LIMIT :limit"; + "FROM document d " + + "JOIN documenttopicsraw dtr ON d.id = dtr.document_id " + + "WHERE dtr.topiclabel = :topicValue " + + "AND d.corpusid = :corpusId " + + "ORDER BY dtr.thetadt DESC " + + "LIMIT :limit"; var query = session.createNativeQuery(sql) .setParameter("topicValue", topicValue) @@ -1691,10 +1948,10 @@ public List getTopDocumentsByTopicLabel(String topicValue, long corpus public List getTopicWordsByTopicLabel(String topicValue, long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, probability " + - "FROM corpustopicwords " + - "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + - "ORDER BY probability DESC " + - "LIMIT 20"; + "FROM corpustopicwords " + + "WHERE topiclabel = :topicValue AND corpus_id = :corpusId " + + "ORDER BY probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("topicValue", topicValue); @@ -1731,12 +1988,12 @@ public List getSimilarTopicsbyTopicLabel(String topicValue, long corpu public List getNormalizedTopicWordsForCorpus(long corpusId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, " + - "AVG(probability) AS avg_probability, " + - "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + - "FROM corpustopicwords " + - "WHERE corpus_id = :corpusId " + - "GROUP BY word " + - "ORDER BY normalized_probability DESC"; + "AVG(probability) AS avg_probability, " + + "AVG(probability) / SUM(AVG(probability)) OVER () AS normalized_probability " + + "FROM corpustopicwords " + + "WHERE corpus_id = :corpusId " + + "GROUP BY word " + + "ORDER BY normalized_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("corpusId", corpusId); @@ -1783,11 +2040,11 @@ FROM get_normalized_topic_scores(:corpusId) public List getDocumentWordDistribution(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY word " + - "ORDER BY avg_probability DESC " + - "LIMIT 20"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY word " + + "ORDER BY avg_probability DESC " + + "LIMIT 20"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); @@ -1818,25 +2075,25 @@ public List getDocumentWordDistribution(long documentId) throws Datab public List getSimilarDocumentbyDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "WITH sourcewords AS (" + - " SELECT word " + - " FROM documenttopicwords " + - " WHERE document_id = :documentId " + - " GROUP BY word" + - "), " + - "similardocs AS (" + - " SELECT " + - " dtw.document_id, " + - " COUNT(DISTINCT dtw.word) AS sharedwords " + - " FROM documenttopicwords dtw " + - " JOIN sourcewords sw ON sw.word = dtw.word " + - " WHERE dtw.document_id != :documentId " + - " GROUP BY dtw.document_id " + - " ORDER BY sharedwords DESC" + - ") " + - "SELECT d.documentid, s.sharedwords " + - "FROM similardocs s " + - "JOIN document d ON s.document_id = d.id " + - "LIMIT 20"; + " SELECT word " + + " FROM documenttopicwords " + + " WHERE document_id = :documentId " + + " GROUP BY word" + + "), " + + "similardocs AS (" + + " SELECT " + + " dtw.document_id, " + + " COUNT(DISTINCT dtw.word) AS sharedwords " + + " FROM documenttopicwords dtw " + + " JOIN sourcewords sw ON sw.word = dtw.word " + + " WHERE dtw.document_id != :documentId " + + " GROUP BY dtw.document_id " + + " ORDER BY sharedwords DESC" + + ") " + + "SELECT d.documentid, s.sharedwords " + + "FROM similardocs s " + + "JOIN document d ON s.document_id = d.id " + + "LIMIT 20"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -1862,7 +2119,7 @@ public List getTaxonValuesAndCountByPageId(long documentId) throws Dat // Outer query: group by page_id, aggregate values and count String finalSql = "SELECT page_id, valuee AS taxon_value " + - "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; + "FROM (" + sqlBuilder.toString() + ") AS combined_taxon "; var query = session.createNativeQuery(finalSql) .setParameter("documentId", documentId) @@ -1878,8 +2135,8 @@ public List getNamedEntityValuesAndCountByPage(long documentId) throws return executeOperationSafely((session) -> { // Construct the SQL query to select page_id and coveredtext from the namedentity table String sql = "SELECT ne.page_id, ne.coveredtext AS named_entity_value, ne.typee AS named_entity_type " + - "FROM namedentity ne " + - "WHERE ne.document_id = :documentId"; + "FROM namedentity ne " + + "WHERE ne.document_id = :documentId"; var query = session.createNativeQuery(sql) .setParameter("documentId", documentId); @@ -1902,6 +2159,103 @@ public List getLemmaByPage(long documentId) throws DatabaseOperationEx }); } + public List getDocumentEmotionsOrdered(long documentId, int modelId) throws DatabaseOperationException { + class EmotionEntry { + long emotionId; + long emotionType; + double emotionValue; + } + return executeOperationSafely((session) -> { + String sql = """ + SELECT e.id AS emotionId, + ev.emotion_type_id AS emotionType, + ev.value AS emotionValue + FROM emotion e + JOIN emotion_value ev ON e.id = ev.emotion_id + JOIN model_version mv ON e.model_version_id = mv.id + WHERE e.document_id = :documentId + AND mv.model_id = :modelId + ORDER BY e.beginn, e.endd + """; + var query = session.createNativeQuery(sql) + .setParameter("documentId", documentId) + .setParameter("modelId", modelId); + + var result = query.getResultList(); + List emotionIds = new ArrayList<>(); + Map> emotionMap = new HashMap<>(); + for (Object raw : result) { + Object[] row = (Object[]) raw; + long emotionId = ((Number) row[0]).longValue(); + long emotionType = ((Number) row[1]).longValue(); + double emotionValue = ((Number) row[2]).doubleValue(); + + if (!emotionIds.contains(emotionId)) { + emotionIds.add(emotionId); + } + + EmotionEntry entry = new EmotionEntry(); + entry.emotionId = emotionId; + entry.emotionType = emotionType; + entry.emotionValue = emotionValue; + + emotionMap.computeIfAbsent(emotionId, k -> new ArrayList<>()).add(entry); + } + return emotionIds.stream() + .map(emotionMap::get) + .map(e -> e.stream().map(entry -> Map.of( + "emotionType", entry.emotionType, + "emotionValue", entry.emotionValue, + "emotionId", entry.emotionId + )).toArray()) + .toList(); + }); + } + + public List getEmotionTypes() throws DatabaseOperationException { + return executeOperationSafely((session) -> { + String sql = "SELECT id, name FROM emotion_type ORDER BY name"; + var query = session.createNativeQuery(sql); + //noinspection unchecked + return query.getResultList().stream() + .map(result -> { + Object[] row = (Object[]) result; + return Map.of( + "id", ((Number) row[0]).longValue(), + "name", (String) row[1] + ); + }) + .toList(); + }); + } + + public List getEmotionTypesForDocument(long documentId, long modelId) throws DatabaseOperationException { + return executeOperationSafely((session) -> { + String sql = """ + SELECT DISTINCT et.id AS emotionTypeId, et.name AS emotionTypeName + FROM emotion e + JOIN emotion_value ev ON e.id = ev.emotion_id + JOIN emotion_type et ON ev.emotion_type_id = et.id + JOIN model_version mv ON e.model_version_id = mv.id + WHERE e.document_id = :documentId + AND mv.model_id = :modelId + """; + var query = session.createNativeQuery(sql) + .setParameter("documentId", documentId) + .setParameter("modelId", modelId); + //noinspection unchecked + return query.getResultList().stream() + .map(result -> { + Object[] row = (Object[]) result; + return Map.of( + "id", ((Number) row[0]).longValue(), + "name", (String) row[1] + ); + }) + .toList(); + }); + } + public List getGeonameByPage(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT gn.page_id, gn.coveredtext AS geoname_value " + @@ -2000,10 +2354,10 @@ entities_in_sentences AS ( public List getTopicWordsByDocumentId(long documentId) throws DatabaseOperationException { return executeOperationSafely((session) -> { String sql = "SELECT topiclabel, word, AVG(probability) AS avg_probability " + - "FROM documenttopicwords " + - "WHERE document_id = :documentId " + - "GROUP BY topiclabel, word " + - "ORDER BY avg_probability DESC"; + "FROM documenttopicwords " + + "WHERE document_id = :documentId " + + "GROUP BY topiclabel, word " + + "ORDER BY avg_probability DESC"; var query = session.createNativeQuery(sql); query.setParameter("documentId", documentId); @@ -2113,6 +2467,54 @@ private Document initializeCompleteDocument(Document doc, int skipPages, int pag Hibernate.initialize(topic.getTopics()); } + // emotion + Hibernate.initialize(doc.getEmotions()); + + for (var emotion : doc.getEmotions()) { + Hibernate.initialize(emotion.getEmotionValues()); + Hibernate.initialize(emotion.getModelVersion()); + Hibernate.initialize(emotion.getModelVersion().getModel()); + for (var ev : emotion.getEmotionValues()) { + Hibernate.initialize(ev.getEmotionType()); + } + } + + // offensive speech + Hibernate.initialize(doc.getOffensiveSpeeches()); + + for (var offensiveSpeech : doc.getOffensiveSpeeches()) { + Hibernate.initialize(offensiveSpeech.getOffensiveSpeechValues()); + Hibernate.initialize(offensiveSpeech.getModelVersion()); + Hibernate.initialize(offensiveSpeech.getModelVersion().getModel()); + for (var osv : offensiveSpeech.getOffensiveSpeechValues()) { + Hibernate.initialize(osv.getOffensiveSpeechType()); + } + } + + // toxic + Hibernate.initialize(doc.getToxics()); + + for (var toxic : doc.getToxics()) { + Hibernate.initialize(toxic.getToxicValues()); + Hibernate.initialize(toxic.getModelVersion()); + Hibernate.initialize(toxic.getModelVersion().getModel()); + for (var tv : toxic.getToxicValues()) { + Hibernate.initialize(tv.getToxicType()); + } + } + + // sentiment + Hibernate.initialize(doc.getSentiments()); + + for (var sentiment : doc.getSentiments()) { + Hibernate.initialize(sentiment.getSentimentValues()); + Hibernate.initialize(sentiment.getModelVersion()); + Hibernate.initialize(sentiment.getModelVersion().getModel()); + for (var sv : sentiment.getSentimentValues()) { + Hibernate.initialize(sv.getSentimentType()); + } + } + for (var link : doc.getWikipediaLinks()) { Hibernate.initialize(link.getWikiDataHyponyms()); } diff --git a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java index 51b93f9f..d934b60e 100644 --- a/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java +++ b/uce.portal/uce.common/src/main/java/org/texttechnologylab/services/WikiService.java @@ -113,6 +113,60 @@ public UnifiedTopicWikiPageViewModel buildUnifiedTopicWikiPageViewModel(long id, return viewModel; } + public SentimentWikiPageViewModel buildSentimentWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new SentimentWikiPageViewModel(); + var sentiment = db.getInitializedSentimentById(id); + viewModel.setWikiModel(sentiment); + viewModel.setDocument(db.getDocumentById(sentiment.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Sentiment"); + + return viewModel; + } + + public EmotionWikiPageViewModel buildEmotionWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new EmotionWikiPageViewModel(); + var emotion = db.getInitializedEmotionById(id); + viewModel.setWikiModel(emotion); + viewModel.setDocument(db.getDocumentById(emotion.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Emotion"); + + return viewModel; + } + + /** + * Builds a view model to render a toxic annotation wiki page + */ + public ToxicAnnotationWikiPageViewModel buildToxicAnnotationWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new ToxicAnnotationWikiPageViewModel(); + var toxic = db.getInitializedToxicById(id); + viewModel.setWikiModel(toxic); + viewModel.setDocument(db.getDocumentById(toxic.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("Toxic"); + + return viewModel; + } + + /** + * Builds a view model to render a offensive speech annotation wiki page + */ + public OffensiveSpeechAnnotationWikiPageViewModel buildOffensiveSpeechAnnotationWikiPageViewModel(long id, String coveredText) throws DatabaseOperationException { + var viewModel = new OffensiveSpeechAnnotationWikiPageViewModel(); + var offensiveSpeech = db.getInitializedOffensiveSpeechById(id); + viewModel.setWikiModel(offensiveSpeech); + viewModel.setDocument(db.getDocumentById(offensiveSpeech.getDocumentId())); + viewModel.setCorpus(db.getCorpusById(viewModel.getDocument().getCorpusId()).getViewModel()); + viewModel.setCoveredText(coveredText); + viewModel.setAnnotationType("OffensiveSpeech"); + + return viewModel; + } + /** * Gets a DocumentTopicDistributionWikiPageViewModel to render a Wikipage for that topic distribution */ diff --git a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java index 9efd4281..dd23bf8c 100644 --- a/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java +++ b/uce.portal/uce.corpus-importer/src/main/java/org/texttechnologylab/Importer.java @@ -19,10 +19,17 @@ import org.apache.uima.util.CasIOUtils; import org.apache.uima.util.CasLoadMode; import org.springframework.context.ApplicationContext; +import org.texttechnologylab.annotation.AnnotationComment; import org.texttechnologylab.annotation.DocumentAnnotation; import org.texttechnologylab.annotation.geonames.GeoNamesEntity; -import org.texttechnologylab.annotation.link.*; -import org.texttechnologylab.annotation.ocr.*; +import org.texttechnologylab.annotation.link.ADLink; +import org.texttechnologylab.annotation.link.DALink; +import org.texttechnologylab.annotation.link.DLink; +import org.texttechnologylab.annotation.model.MetaData; +import org.texttechnologylab.annotation.ocr.OCRBlock; +import org.texttechnologylab.annotation.ocr.OCRLine; +import org.texttechnologylab.annotation.ocr.OCRPage; +import org.texttechnologylab.annotation.ocr.OCRToken; import org.texttechnologylab.config.CommonConfig; import org.texttechnologylab.config.CorpusConfig; import org.texttechnologylab.exceptions.DatabaseOperationException; @@ -39,19 +46,33 @@ import org.texttechnologylab.models.corpus.ocr.OCRPageAdapterImpl; import org.texttechnologylab.models.corpus.ocr.PageAdapter; import org.texttechnologylab.models.corpus.ocr.PageAdapterImpl; +import org.texttechnologylab.models.emotion.Emotion; +import org.texttechnologylab.models.emotion.EmotionType; +import org.texttechnologylab.models.emotion.EmotionValue; import org.texttechnologylab.models.imp.ImportLog; import org.texttechnologylab.models.imp.ImportStatus; import org.texttechnologylab.models.imp.LogStatus; +import org.texttechnologylab.models.modelInfo.Model; +import org.texttechnologylab.models.modelInfo.ModelCategory; +import org.texttechnologylab.models.modelInfo.ModelVersion; import org.texttechnologylab.models.negation.*; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeech; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechType; +import org.texttechnologylab.models.offensiveSpeech.OffensiveSpeechValue; import org.texttechnologylab.models.rag.DocumentChunkEmbedding; import org.texttechnologylab.models.rag.DocumentSentenceEmbedding; +import org.texttechnologylab.models.sentiment.Sentiment; +import org.texttechnologylab.models.sentiment.SentimentType; +import org.texttechnologylab.models.sentiment.SentimentValue; import org.texttechnologylab.models.topic.TopicValueBase; import org.texttechnologylab.models.topic.TopicValueBaseWithScore; import org.texttechnologylab.models.topic.TopicWord; import org.texttechnologylab.models.topic.UnifiedTopic; +import org.texttechnologylab.models.toxic.Toxic; +import org.texttechnologylab.models.toxic.ToxicType; +import org.texttechnologylab.models.toxic.ToxicValue; import org.texttechnologylab.services.*; import org.texttechnologylab.utils.*; -import org.texttechnologylab.models.negation.CompleteNegation; import java.io.*; import java.nio.file.Files; @@ -140,13 +161,13 @@ public int getXMICountInPath() { public void start(int numThreads) throws DatabaseOperationException { logger.info( "\n _ _ _____ _____ _____ _ \n" + - "| | | / __ \\| ___| |_ _| | | \n" + - "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + - "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + - "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + - " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + - " | | \n" + - " |_|" + "| | | / __ \\| ___| |_ _| | | \n" + + "| | | | / \\/| |__ | | _ __ ___ _ __ ___ _ __| |_ \n" + + "| | | | | | __| | || '_ ` _ \\| '_ \\ / _ \\| '__| __|\n" + + "| |_| | \\__/\\| |___ _| || | | | | | |_) | (_) | | | |_ \n" + + " \\___/ \\____/\\____/ \\___/_| |_| |_| .__/ \\___/|_| \\__|\n" + + " | | \n" + + " |_|" ); logger.info("===========> Global Import Id: " + importId); logger.info("===========> Importer Number: " + importerNumber); @@ -208,7 +229,7 @@ public void storeCorpusFromFolderAsync(String folderName, int numThreads) throws final var corpusConfig1 = corpusConfig; // This sucks so hard - why doesn't java just do this itself if needed? var existingCorpus = ExceptionUtils.tryCatchLog(() -> db.getCorpusByName(corpusConfig1.getName()), (ex) -> logger.error("Error getting an existing corpus by name. The corpus config should probably be changed " + - "to not add to existing corpus then.", ex)); + "to not add to existing corpus then.", ex)); if (existingCorpus != null) { // If we have the corpus, use that. Else store the new corpus. corpus = existingCorpus; @@ -468,7 +489,7 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { var exists = db.documentExists(corpus.getId(), document.getDocumentId()); if (exists) { logger.info("Document with id " + document.getDocumentId() - + " already exists in the corpus " + corpus.getId() + "."); + + " already exists in the corpus " + corpus.getId() + "."); logger.info("Checking if that document was also post-processed yet..."); var existingDoc = db.getDocumentByCorpusAndDocumentId(corpus.getId(), document.getDocumentId()); if (!existingDoc.isPostProcessed()) { @@ -567,6 +588,27 @@ public Document XMIToDocument(JCas jCas, Corpus corpus, String filePath) { () -> setUnifiedTopic(document, jCas), (ex) -> logImportWarn("This file should have contained UnifiedTopic annotations, but selecting them caused an error.", ex, filePath)); + if (corpusConfig.getAnnotations().isSentiment()) + ExceptionUtils.tryCatchLog( + () -> setSentiment(document, jCas), + (ex) -> logImportWarn("This file should have contained sentiment annotations, but selecting them caused an error.", ex, filePath)); + + + if (corpusConfig.getAnnotations().isOffensiveSpeech()) + ExceptionUtils.tryCatchLog( + () -> setOffensiveSpeech(document, jCas), + (ex) -> logImportWarn("This file should have contained OffensiveSpeech annotations, but selecting them caused an error.", ex, filePath)); + + if (corpusConfig.getAnnotations().isToxic()) + ExceptionUtils.tryCatchLog( + () -> setToxic(document, jCas), + (ex) -> logImportWarn("This file should have contained Toxic annotations, but selecting them caused an error.", ex, filePath)); + + if (corpusConfig.getAnnotations().isEmotion()) + ExceptionUtils.tryCatchLog( + () -> setEmotion(document, jCas), + (ex) -> logImportWarn("This file should have contained emotion annotations, but selecting them caused an error.", ex, filePath)); + // Keep this at the end of the annotation setting, as they might require previous annotations. Order matter here! if (corpusConfig.getAnnotations().isLogicalLinks()) ExceptionUtils.tryCatchLog( @@ -820,8 +862,8 @@ private void setMetadataTitleInfo(Document document, JCas jCas, CorpusConfig cor if (documentAnnotation != null) { try { metadataTitleInfo.setPublished(documentAnnotation.getDateDay() + "." - + documentAnnotation.getDateMonth() + "." - + documentAnnotation.getDateYear()); + + documentAnnotation.getDateMonth() + "." + + documentAnnotation.getDateYear()); metadataTitleInfo.setAuthor(documentAnnotation.getAuthor()); } catch (Exception ex) { logger.warn("Tried extracting DocumentAnnotation type, it caused an error. Import will be continued as usual."); @@ -992,6 +1034,24 @@ private void updateAnnotationsWithPageId(Document document, Page page, boolean i anno.setPage(page); } } + if (document.getSentiments() != null) { + for (var anno: document.getSentiments().stream().filter(s -> + (s.getBegin() >= page.getBegin() && s.getEnd() <= page.getEnd()) || (s.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } + if (document.getToxics() != null) { + for (var anno : document.getToxics().stream().filter(t -> + (t.getBegin() >= page.getBegin() && t.getEnd() <= page.getEnd()) || (t.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } + if (document.getEmotions() != null) { + for (var anno : document.getEmotions().stream().filter(t -> + (t.getBegin() >= page.getBegin() && t.getEnd() <= page.getEnd()) || (t.getPage() == null && isLastPage)).toList()) { + anno.setPage(page); + } + } } /** @@ -1411,6 +1471,255 @@ private void setUnifiedTopic(Document document, JCas jCas) { document.setUnifiedTopics(unifiedTopics); } + private void setSentiment(Document document, JCas jCas){ + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Sentiment.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Sentiment", e); + return; + } + List sentiments = new ArrayList<>(); + + JCasUtil.select(jCas, org.texttechnologylab.annotation.SentimentModel.class).forEach(s -> { + Sentiment sentiment = new Sentiment(s.getBegin(), s.getEnd()); + sentiment.setDocument(document); + + try { + MetaData modelMetaData = s.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + sentiment.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for Toxic", e); + return; + } + + sentiment.setSentiment(s.getSentiment()); + + List sentimentValues = new ArrayList<>(); + SentimentType positiveType, negativeType, neutralType; + try { + positiveType = db.getOrCreateSentimentType("positive"); + negativeType = db.getOrCreateSentimentType("negative"); + neutralType = db.getOrCreateSentimentType("neutral"); + } catch (DatabaseOperationException e) { + logger.error("Error while fetching Sentiment Types", e); + return; + } + + SentimentValue positiveValue = new SentimentValue(); + positiveValue.setSentimentType(positiveType); + positiveValue.setValue(s.getProbabilityPositive()); + positiveValue.setSentiment(sentiment); + sentimentValues.add(positiveValue); + + SentimentValue negativeValue = new SentimentValue(); + negativeValue.setSentimentType(negativeType); + negativeValue.setValue(s.getProbabilityNegative()); + negativeValue.setSentiment(sentiment); + sentimentValues.add(negativeValue); + + SentimentValue neutralValue = new SentimentValue(); + neutralValue.setSentimentType(neutralType); + neutralValue.setValue(s.getProbabilityNeutral()); + neutralValue.setSentiment(sentiment); + sentimentValues.add(neutralValue); + + sentiment.setSentimentValues(sentimentValues); + + sentiment.setCoveredText(s.getCoveredText()); + sentiments.add(sentiment); + }); + document.setSentiments(sentiments); + } + + private void setOffensiveSpeech(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(OffensiveSpeech.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for OffensiveSpeech", e); + return; + } + + List offensiveSpeeches = new ArrayList<>(); + + JCasUtil.select(jCas, org.texttechnologylab.annotation.OffensiveSpeech.class).forEach(os -> { + OffensiveSpeech offensiveSpeech = new OffensiveSpeech(os.getBegin(), os.getEnd()); + offensiveSpeech.setDocument(document); + offensiveSpeech.setCoveredText(os.getCoveredText()); + try { + MetaData modelMetaData = os.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + offensiveSpeech.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for OffensiveSpeech", e); + return; + } + + FSArray offensives = os.getOffensives(); + if (offensives == null) { + logger.warn("OffensiveSpeech annotation without offensives found. Skipping this annotation."); + return; + } + + List offensivesList = new ArrayList<>(); + for (AnnotationComment offensive : offensives) { + String offensiveSpeechName = offensive.getKey(); + OffensiveSpeechType offensiveSpeechType; + try { + offensiveSpeechType = db.getOrCreateOffensiveSpeechType(offensiveSpeechName); + } catch (DatabaseOperationException e) { + logger.warn("Unknown OffensiveSpeechType: " + offensiveSpeechName + ". Skipping this annotation."); + continue; + } + double value; + try { + value = Double.parseDouble(offensive.getValue()); + } + catch (NumberFormatException e) { + logger.warn("Invalid value for OffensiveSpeechType: " + offensiveSpeechName + ". Skipping this annotation."); + continue; + } + OffensiveSpeechValue offensiveSpeechValue = new OffensiveSpeechValue(); + offensiveSpeechValue.setOffensiveSpeechType(offensiveSpeechType); + offensiveSpeechValue.setValue(value); + offensiveSpeechValue.setOffensiveSpeech(offensiveSpeech); + offensivesList.add(offensiveSpeechValue); + } + offensiveSpeech.setOffensiveSpeechValues(offensivesList); + offensiveSpeeches.add(offensiveSpeech); + }); + + document.setOffensiveSpeeches(offensiveSpeeches); + } + + /** + * Selects and sets the toxicities to a document. + */ + private void setToxic(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Toxic.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Toxic", e); + return; + } + + List toxics = new ArrayList<>(); + + JCasUtil.select(jCas, org.texttechnologylab.annotation.Toxic.class).forEach(t -> { + Toxic toxic = new Toxic(t.getBegin(), t.getEnd()); + toxic.setDocument(document); + toxic.setCoveredText(t.getCoveredText()); + + try { + MetaData modelMetaData = t.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + toxic.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model for Toxic", e); + return; + } + + List toxicValues = new ArrayList<>(); + ToxicType toxicType, nonToxicType; + try { + toxicType = db.getOrCreateToxicType("toxic"); + nonToxicType = db.getOrCreateToxicType("non-toxic"); + } catch (DatabaseOperationException e) { + logger.error("Error while fetching ToxicType or NonToxicType from database.", e); + return; + } + + ToxicValue toxicValue = new ToxicValue(); + toxicValue.setToxicType(toxicType); + toxicValue.setValue(t.getToxic()); + toxicValue.setToxic(toxic); + toxicValues.add(toxicValue); + ToxicValue nonToxicValue = new ToxicValue(); + nonToxicValue.setToxicType(nonToxicType); + nonToxicValue.setValue(t.getNonToxic()); + nonToxicValue.setToxic(toxic); + toxicValues.add(nonToxicValue); + + toxic.setToxicValues(toxicValues); + toxics.add(toxic); + }); + + document.setToxics(toxics); + } + + /** + * Selects and sets the emotions to a document + */ + private void setEmotion(Document document, JCas jCas) { + ModelCategory modelCategory; + try { + modelCategory = db.getOrCreateModelCategory(Emotion.class); + } catch (DatabaseOperationException e) { + logger.error("Error while getting or creating model category for Emotion", e); + return; + } + + List emotions = new ArrayList<>(); + + JCasUtil.select(jCas, org.texttechnologylab.annotation.Emotion.class).forEach(e -> { + Emotion emotion = new Emotion(e.getBegin(), e.getEnd()); + emotion.setDocument(document); + try { + MetaData modelMetaData = e.getModel(); + Model model = db.getOrCreateModel(modelMetaData.getModelName()); + ModelVersion modelVersion = db.getOrCreateModelVersion(model, modelMetaData.getModelVersion()); + emotion.setModelVersion(modelVersion); + db.registerModelToCategoryAssociation(model, modelCategory); + } catch (DatabaseOperationException ex) { + logger.error("Error while getting or creating model for Emotion", ex); + return; + } + + FSArray emotionAnnotations = e.getEmotions(); + List emotionValues = new ArrayList<>(); + for (AnnotationComment emotionAnnotation : emotionAnnotations) { + String emotionName = emotionAnnotation.getKey(); + EmotionType emotionType; + try { + emotionType = db.getOrCreateEmotionTypeByName(emotionName); + } catch (DatabaseOperationException ex) { + logger.error("Error while getting or creating emotion type for name: " + emotionName, ex); + continue; + } + String rawValue = emotionAnnotation.getValue(); + double value; + try { + value = Double.parseDouble(rawValue); + } catch (NumberFormatException ex) { + logger.warn("Invalid emotion value for " + emotionName + ": " + rawValue + ". Skipping this emotion.", ex); + continue; + } + EmotionValue emotionValue = new EmotionValue(e.getBegin(), e.getEnd()); + emotionValue.setDocument(document); + emotionValue.setEmotion(emotion); + emotionValue.setEmotionType(emotionType); + emotionValue.setValue(value); + emotionValue.setCoveredText(e.getCoveredText()); + emotionValues.add(emotionValue); + } + emotion.setEmotionValues(emotionValues); + emotion.setCoveredText(e.getCoveredText()); + + emotions.add(emotion); + }); + + document.setEmotions(emotions); + } + /** * Set the cleaned full text. That is the sum of all tokens except of all anomalies * Update: OBSOLETE for now, as this takes forever and makes the OCR worse. diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java index f374a964..5628bfa1 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/App.java @@ -8,8 +8,6 @@ import org.apache.commons.cli.ParseException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.keycloak.authorization.client.AuthzClient; -import org.keycloak.representations.idm.authorization.AuthorizationRequest; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.texttechnologylab.auth.AuthenticationRouteRegister; @@ -19,7 +17,6 @@ import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.freeMarker.Renderer; import org.texttechnologylab.freeMarker.RequestContextHolder; -import org.texttechnologylab.models.authentication.UceUser; import org.texttechnologylab.models.corpus.Corpus; import org.texttechnologylab.models.corpus.UCELog; import org.texttechnologylab.modules.ModelGroup; @@ -39,7 +36,10 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.StandardOpenOption; -import java.util.*; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import static spark.Spark.*; @@ -420,6 +420,7 @@ private static void initSparkRoutes(ApplicationContext context) throws IOExcepti get("/page/namedEntities", (registry.get(DocumentApi.class)).getDocumentNamedEntitiesByPage); get("/page/lemma", (registry.get(DocumentApi.class)).getDocumentLemmaByPage); get("/page/geoname", (registry.get(DocumentApi.class)).getDocumentGeonameByPage); + get("/page/emotionDev", (registry.get(DocumentApi.class)).getDocumentEmotionDevelopment); }); path("/rag", () -> { diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java index 7b29d11a..9c5586b3 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/DocumentApi.java @@ -2,6 +2,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import freemarker.template.Configuration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -13,6 +15,7 @@ import org.texttechnologylab.config.CorpusConfig; import org.texttechnologylab.exceptions.ExceptionUtils; import org.texttechnologylab.models.corpus.UCEMetadataValueType; +import org.texttechnologylab.models.modelInfo.ModelCategory; import org.texttechnologylab.models.search.SearchType; import org.texttechnologylab.services.PostgresqlDataInterface_Impl; import org.texttechnologylab.services.S3StorageService; @@ -159,7 +162,7 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo var id = ExceptionUtils.tryCatchLog(() -> request.queryParams("id"), (ex) -> logger.error("Error: the url for the document reader requires an 'id' query parameter. " + - "Document reader can't be built.", ex)); + "Document reader can't be built.", ex)); if (id == null) return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); @@ -186,6 +189,14 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo model.put("searchTokens", String.join("[TOKEN]", activeSearchState.getSearchTokens())); } } + + List modelCategories = new ArrayList<>(); + List modelCategoryNames = doc.getModelCategories(); + for (String categoryName : modelCategoryNames) { + ModelCategory modelCategory = db.getOrCreateModelCategory(categoryName); + modelCategories.add(modelCategory); + } + model.put("modelCategories", modelCategories); } catch (Exception ex) { logger.error("Error creating the document reader view for document with id: " + id, ex); return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); @@ -203,10 +214,24 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo if (id == null) return new CustomFreeMarkerEngine(this.freemarkerConfig).render(new ModelAndView(null, "defaultError.ftl")); + String modelSelectionString = request.queryParams("modelSelection"); + Map modelSelection = new HashMap<>(); + if (modelSelectionString != null && !modelSelectionString.isEmpty()) { + try { + JsonObject jsonObject = JsonParser.parseString(modelSelectionString).getAsJsonObject(); + for (var entry : jsonObject.entrySet()) { + modelSelection.put(entry.getKey(), entry.getValue().getAsInt()); + } + } catch (Exception ex) { + logger.error("Error parsing model selection JSON: " + modelSelectionString, ex); + } + } + + try { var skip = Integer.parseInt(request.queryParams("skip")); var doc = db.getCompleteDocumentById(Long.parseLong(id), skip, 10); - var annotations = doc.getAllAnnotations(skip, 10); + var annotations = doc.getAllAnnotations(skip, 10, modelSelection); model.put("documentAnnotations", annotations); model.put("documentText", doc.getFullText()); model.put("documentPages", doc.getPages(10, skip)); @@ -408,7 +433,6 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo } }); - public Route getSentenceTopicsWithEntities = ((request, response) -> { var documentId = ExceptionUtils.tryCatchLog(() -> Long.parseLong(request.queryParams("documentId")), (ex) -> logger.error("Error: couldn't determine the documentId for sentence topics with entities. ", ex)); @@ -516,5 +540,32 @@ public DocumentApi(ApplicationContext serviceContext, Configuration freemarkerCo } }; + public Route getDocumentEmotionDevelopment = (request, response) -> { + var documentId = ExceptionUtils.tryCatchLog(() -> Long.parseLong(request.queryParams("documentId")), + (ex) -> logger.error("Error: couldn't determine the documentId for emotion development.", ex)); + String modelIdString = request.queryParams("modelId"); + int modelId = modelIdString == null ? -1 : Integer.parseInt(modelIdString); + + if (documentId == null) { + response.status(400); + return new CustomFreeMarkerEngine(this.freemarkerConfig) + .render(new ModelAndView(Map.of("information", "Missing documentId parameter for emotion development"), "defaultError.ftl")); + } + + try { + var emotionData = db.getDocumentEmotionsOrdered(documentId, modelId); + var emotionTypes = db.getEmotionTypesForDocument(documentId, modelId); + Map emotionDevelopment = new HashMap<>(); + emotionDevelopment.put("emotionTypes", emotionTypes); + emotionDevelopment.put("emotionData", emotionData); + response.type("application/json"); + return new Gson().toJson(emotionDevelopment); + } catch (Exception ex) { + logger.error("Error retrieving document emotion development.", ex); + response.status(500); + return new CustomFreeMarkerEngine(this.freemarkerConfig) + .render(new ModelAndView(Map.of("information", "Error retrieving document emotion development."), "defaultError.ftl")); + } + }; } diff --git a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java index 38f1b077..c96e5de3 100644 --- a/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java +++ b/uce.portal/uce.web/src/main/java/org/texttechnologylab/routes/WikiApi.java @@ -162,9 +162,21 @@ public WikiApi(ApplicationContext serviceContext, Configuration freemarkerConfig } else if (type.startsWith("UT")) { model.put("vm", wikiService.buildUnifiedTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/unifiedTopicAnnotationPage.ftl"; + } else if (type.startsWith("T")) { + model.put("vm", wikiService.buildToxicAnnotationWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/toxicAnnotationPage.ftl"; } else if (type.startsWith("DTR")) { model.put("vm", wikiService.buildTopicWikiPageViewModel(id, coveredText)); renderView = "/wiki/pages/topicPage.ftl"; + } else if (type.startsWith("S")) { + model.put("vm", wikiService.buildSentimentWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/sentimentAnnotationPage.ftl"; + } else if (type.startsWith("OS")) { + model.put("vm", wikiService.buildOffensiveSpeechAnnotationWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/offensiveSpeechAnnotationPage.ftl"; + } else if (type.startsWith("E")) { + model.put("vm", wikiService.buildEmotionWikiPageViewModel(id, coveredText)); + renderView = "/wiki/pages/emotionAnnotationPage.ftl"; } else { // The type part of the wikiId was unknown. Throw an error. logger.warn("Someone tried to query a wiki page of a type that does not exist in UCE. This shouldn't happen."); diff --git a/uce.portal/uce.web/src/main/resources/languageTranslations.json b/uce.portal/uce.web/src/main/resources/languageTranslations.json index fe109024..9549e573 100644 --- a/uce.portal/uce.web/src/main/resources/languageTranslations.json +++ b/uce.portal/uce.web/src/main/resources/languageTranslations.json @@ -451,6 +451,10 @@ "de-DE": "Wichtige Themen", "en-EN": "Key Topics" }, + "modelSelection": { + "de-DE": "Modellauswahl", + "en-EN": "Model Selection" + }, "corpusWords": { "de-DE": "Top Wörter aus dem Korpus", "en-EN": "Top words from Corpus" diff --git a/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js b/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js index 446bdd74..787a6074 100644 --- a/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js +++ b/uce.portal/uce.web/src/main/resources/public/js/visualization/echarts.js @@ -23,6 +23,14 @@ class ECharts { this.option = newOption; this.render(); } + + setChartId(id) { + this.chartId = id; + } + + getChartId() { + return this.chartId; + } } -export { ECharts }; +export {ECharts};