From 33383f966e1b356abfa0abe28a294b5f7617396a Mon Sep 17 00:00:00 2001 From: Bionic711 Date: Fri, 3 Oct 2025 21:50:02 -0500 Subject: [PATCH 01/24] fix for bug 485 (#486) Co-authored-by: Bionic711 --- RELEASE_NOTES.md | 11 +++++++++++ application/single_app/config.py | 2 +- .../single_app/static/js/admin/admin_plugins.js | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c1ba78df..542b35de 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,6 +1,17 @@ # Feature Release +### **(v0.229.063)** + +#### Bug Fixes + +* **Admin Plugins Modal Load Fix** + * Fixed issue where Admin Plugins modal would fail to load when using sidenav navigation. + * **Root Cause**: JavaScript code attempted to access DOM elements that didn't exist in sidenav navigation. + * **Solution**: Corrected DOM element checks to ensure compatibility with both top-nav and sidenav layouts. + * **User Experience**: Admins can now access the Plugins modal reglardless of navigation style. + * (Ref: `admin_plugins.js`, DOM existence checks) + ### **(v0.229.062)** #### Bug Fixes diff --git a/application/single_app/config.py b/application/single_app/config.py index 1078e6f4..d9895cd5 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -88,7 +88,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.229.062" +VERSION = "0.229.063" SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') diff --git a/application/single_app/static/js/admin/admin_plugins.js b/application/single_app/static/js/admin/admin_plugins.js index 93c7a926..682d329d 100644 --- a/application/single_app/static/js/admin/admin_plugins.js +++ b/application/single_app/static/js/admin/admin_plugins.js @@ -4,7 +4,7 @@ import { renderPluginsTable as sharedRenderPluginsTable, validatePluginManifest // Main logic document.addEventListener('DOMContentLoaded', function () { - if (!document.getElementById('agents-tab')) return; + if (!document.getElementById('actions-configuration')) return; // Load and render plugins table loadPlugins(); From b2aa14a12f66f6d446d116e0c1e8b0a9b94eef1d Mon Sep 17 00:00:00 2001 From: Steve Carroll <37545884+SteveCInVA@users.noreply.github.com> Date: Tue, 18 Nov 2025 14:30:09 -0500 Subject: [PATCH 02/24] Update RELEASE_NOTES.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 542b35de..624e40ec 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -9,7 +9,7 @@ * Fixed issue where Admin Plugins modal would fail to load when using sidenav navigation. * **Root Cause**: JavaScript code attempted to access DOM elements that didn't exist in sidenav navigation. * **Solution**: Corrected DOM element checks to ensure compatibility with both top-nav and sidenav layouts. - * **User Experience**: Admins can now access the Plugins modal reglardless of navigation style. + * **User Experience**: Admins can now access the Plugins modal regardless of navigation style. * (Ref: `admin_plugins.js`, DOM existence checks) ### **(v0.229.062)** From 5546e5a7532e0896d68faaea5ba97cb1423088f5 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 21 Nov 2025 11:33:14 -0500 Subject: [PATCH 03/24] Add 372 fix 489 (#528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix video indexer auth * fixed video indexer support * fixed video indexer support * fix video indexer support * fixed video indexer support * added support for .xlsm * fixed all scope issue * added supprot for xml, yaml, and log #### **JSON** - Uses `RecursiveJsonSplitter`: - `max_chunk_size=600` - `convert_lists=True` - Produces JSON strings that retain original structure. - See `process_json_file`. #### **XML** - Uses `RecursiveCharacterTextSplitter` with XML-aware separators. - **Structure-preserving chunking**: - Separators prioritized: `\n\n` → `\n` → `>` (end of XML tags) → space → character - Splits at logical boundaries to maintain tag integrity - **Chunked by 4000 characters** with 200-character overlap for context preservation. - **Goal**: Preserve XML structure while providing manageable chunks for LLM processing. - See `process_xml`. #### **YAML / YML** - Processed using regex word splitting (similar to TXT). - **Chunked by 400 words**. - Maintains YAML structure through simple word-based splitting. - See `process_yaml`. #### **LOG** - Processed using line-based chunking to maintain log record integrity. - **Never splits mid-line** to preserve complete log entries. - **Line-Level Chunking**: 1. Split file by lines using `splitlines(keepends=True)` to preserve line endings. 2. Accumulate complete lines until reaching target word count ≈1000 words. 3. When adding next line would exceed target AND chunk already has content: - Finalize current chunk - Start new chunk with current line 4. If single line exceeds target, it gets its own chunk to prevent infinite loops. 5. Emit chunks with complete log records. - **Goal**: Provide substantial log context (1000 words) while ensuring no log entry is split across chunks. - See `process_log`. * updated yaml to use recursivesplitter * removed chunk overlap for yaml and xml * added support for older .doc files and .docm * added keyword and abstract for each doc in citation * added multi-modal input support * added ai vision analysis --- application/single_app/config.py | 6 +- .../single_app/functions_authentication.py | 25 +- application/single_app/functions_content.py | 2 +- application/single_app/functions_documents.py | 1248 +++++++++++++++-- application/single_app/functions_search.py | 56 +- application/single_app/functions_settings.py | 7 +- application/single_app/route_backend_chats.py | 218 ++- .../single_app/route_backend_documents.py | 28 +- .../single_app/route_backend_settings.py | 83 ++ .../route_frontend_admin_settings.py | 7 +- .../single_app/route_frontend_chats.py | 203 ++- .../route_frontend_group_workspaces.py | 15 +- .../route_frontend_public_workspaces.py | 5 +- .../single_app/route_frontend_workspace.py | 15 +- .../static/js/admin/admin_settings.js | 147 +- .../static/js/chat/chat-citations.js | 70 + .../static/js/chat/chat-messages.js | 176 ++- .../templates/_video_indexer_info.html | 215 +-- .../single_app/templates/admin_settings.html | 153 +- application/single_app/templates/chats.html | 4 +- .../templates/group_workspaces.html | 2 +- .../single_app/templates/workspace.html | 2 +- docs/admin_configuration.md | 2 +- .../VIDEO_INDEXER_DUAL_AUTHENTICATION.md | 149 ++ .../v0.229.086/METADATA_ENHANCED_CITATIONS.md | 325 +++++ .../v0.229.088/MULTIMODAL_VISION_ANALYSIS.md | 554 ++++++++ .../MULTIMODAL_VISION_SETTINGS_SAVE_FIX.md | 159 +++ ...EO_INDEXER_API_KEY_TOKEN_GENERATION_FIX.md | 199 +++ .../fixes/VISION_MODEL_DETECTION_EXPANSION.md | 185 +++ docs/setup_instructions_manual.md | 12 +- ...deo_indexer_dual_authentication_support.py | 262 ++++ 31 files changed, 4238 insertions(+), 296 deletions(-) create mode 100644 docs/features/VIDEO_INDEXER_DUAL_AUTHENTICATION.md create mode 100644 docs/features/v0.229.086/METADATA_ENHANCED_CITATIONS.md create mode 100644 docs/features/v0.229.088/MULTIMODAL_VISION_ANALYSIS.md create mode 100644 docs/fixes/MULTIMODAL_VISION_SETTINGS_SAVE_FIX.md create mode 100644 docs/fixes/VIDEO_INDEXER_API_KEY_TOKEN_GENERATION_FIX.md create mode 100644 docs/fixes/VISION_MODEL_DETECTION_EXPANSION.md create mode 100644 functional_tests/test_video_indexer_dual_authentication_support.py diff --git a/application/single_app/config.py b/application/single_app/config.py index d9895cd5..59f383c3 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -88,7 +88,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.229.063" +VERSION = "0.229.098" SECRET_KEY = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') @@ -121,9 +121,9 @@ CLIENTS_LOCK = threading.Lock() ALLOWED_EXTENSIONS = { - 'txt', 'pdf', 'docx', 'xlsx', 'xls', 'csv', 'pptx', 'html', 'jpg', 'jpeg', 'png', 'bmp', 'tiff', 'tif', 'heif', 'md', 'json', + 'txt', 'pdf', 'doc', 'docm', 'docx', 'xlsx', 'xls', 'xlsm','csv', 'pptx', 'html', 'jpg', 'jpeg', 'png', 'bmp', 'tiff', 'tif', 'heif', 'md', 'json', 'mp4', 'mov', 'avi', 'mkv', 'flv', 'mxf', 'gxf', 'ts', 'ps', '3gp', '3gpp', 'mpg', 'wmv', 'asf', 'm4a', 'm4v', 'isma', 'ismv', - 'dvr-ms', 'wav' + 'dvr-ms', 'wav', 'xml', 'yaml', 'yml', 'log' } ALLOWED_EXTENSIONS_IMG = {'png', 'jpg', 'jpeg'} MAX_CONTENT_LENGTH = 5000 * 1024 * 1024 # 5000 MB AKA 5 GB diff --git a/application/single_app/functions_authentication.py b/application/single_app/functions_authentication.py index 4d068938..e3d6357d 100644 --- a/application/single_app/functions_authentication.py +++ b/application/single_app/functions_authentication.py @@ -245,15 +245,34 @@ def get_valid_access_token_for_plugins(scopes=None): def get_video_indexer_account_token(settings, video_id=None): """ - For ARM-based VideoIndexer accounts: + Get Video Indexer access token using managed identity authentication. + + This function authenticates with Azure Video Indexer using the App Service's + managed identity. The managed identity must have Contributor role on the + Video Indexer resource. + + Authentication flow: + 1. Acquire ARM access token using DefaultAzureCredential (managed identity) + 2. Call ARM generateAccessToken API to get Video Indexer access token + 3. Use Video Indexer access token for all API operations + """ + from functions_debug import debug_print + + debug_print(f"[VIDEO INDEXER AUTH] Starting token acquisition using managed identity for video_id: {video_id}") + debug_print(f"[VIDEO INDEXER AUTH] Azure environment: {AZURE_ENVIRONMENT}") + + return get_video_indexer_managed_identity_token(settings, video_id) + +def get_video_indexer_managed_identity_token(settings, video_id=None): + """ + For ARM-based VideoIndexer accounts using managed identity: 1) Acquire an ARM token with DefaultAzureCredential 2) POST to the ARM generateAccessToken endpoint 3) Return the account-level accessToken """ from functions_debug import debug_print - debug_print(f"[VIDEO INDEXER AUTH] Starting token acquisition for video_id: {video_id}") - debug_print(f"[VIDEO INDEXER AUTH] Azure environment: {AZURE_ENVIRONMENT}") + debug_print(f"[VIDEO INDEXER AUTH] Using managed identity authentication") # 1) ARM token if AZURE_ENVIRONMENT == "usgovernment": diff --git a/application/single_app/functions_content.py b/application/single_app/functions_content.py index ffa5c559..9cd2a835 100644 --- a/application/single_app/functions_content.py +++ b/application/single_app/functions_content.py @@ -172,7 +172,7 @@ def extract_table_file(file_path, file_ext): try: if file_ext == '.csv': df = pandas.read_csv(file_path) - elif file_ext in ['.xls', '.xlsx']: + elif file_ext in ['.xls', '.xlsx', '.xlsm']: df = pandas.read_excel(file_path) else: raise ValueError("Unsupported file extension for table extraction.") diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index de71cd60..3659ccdf 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -220,6 +220,8 @@ def save_video_chunk( ): """ Saves one 30-second video chunk to the search index, with separate fields for transcript and OCR. + Video Indexer insights (keywords, labels, topics, audio effects, emotions, sentiments) are + already appended to page_text_content for searchability. The chunk_id is built from document_id and the integer second offset to ensure a valid key. """ from functions_debug import debug_print @@ -373,19 +375,33 @@ def to_seconds(ts: str) -> float: debug_print(f"[VIDEO INDEXER] Configuration - Endpoint: {vi_ep}, Location: {vi_loc}, Account ID: {vi_acc}") - # Validate required settings + # Validate required settings based on authentication type + auth_type = settings.get("video_indexer_authentication_type", "managed_identity") + debug_print(f"[VIDEO INDEXER] Using authentication type: {auth_type}") + + # Common required settings for both authentication types required_settings = { "video_indexer_endpoint": vi_ep, "video_indexer_location": vi_loc, - "video_indexer_account_id": vi_acc, - "video_indexer_resource_group": settings.get("video_indexer_resource_group"), - "video_indexer_subscription_id": settings.get("video_indexer_subscription_id"), - "video_indexer_account_name": settings.get("video_indexer_account_name") + "video_indexer_account_id": vi_acc } + if auth_type == "key": + # For API key authentication, only need API key in addition to common settings + required_settings["video_indexer_api_key"] = settings.get("video_indexer_api_key") + debug_print(f"[VIDEO INDEXER] API key authentication requires: endpoint, location, account_id, api_key") + else: + # For managed identity authentication, need ARM-related settings + required_settings.update({ + "video_indexer_resource_group": settings.get("video_indexer_resource_group"), + "video_indexer_subscription_id": settings.get("video_indexer_subscription_id"), + "video_indexer_account_name": settings.get("video_indexer_account_name") + }) + debug_print(f"[VIDEO INDEXER] Managed identity authentication requires: endpoint, location, account_id, resource_group, subscription_id, account_name") + missing_settings = [key for key, value in required_settings.items() if not value] if missing_settings: - debug_print(f"[VIDEO INDEXER] ERROR: Missing required settings: {missing_settings}") + debug_print(f"[VIDEO INDEXER] ERROR: Missing required settings for {auth_type} authentication: {missing_settings}") update_callback(status=f"VIDEO: missing settings - {', '.join(missing_settings)}") return 0 @@ -405,14 +421,24 @@ def to_seconds(ts: str) -> float: # 2) Upload video to Indexer try: url = f"{vi_ep}/{vi_loc}/Accounts/{vi_acc}/Videos" - params = {"accessToken": token, "name": original_filename} + + # Use the access token in the URL parameters + headers = {} + # Request comprehensive indexing including audio transcript + params = { + "accessToken": token, + "name": original_filename, + "indexingPreset": "Default", # Includes video + audio insights + "streamingPreset": "NoStreaming" + } + debug_print(f"[VIDEO INDEXER] Using managed identity access token authentication") debug_print(f"[VIDEO INDEXER] Upload URL: {url}") debug_print(f"[VIDEO INDEXER] Upload params: {params}") debug_print(f"[VIDEO INDEXER] Starting file upload for: {original_filename}") with open(temp_file_path, "rb") as f: - resp = requests.post(url, params=params, files={"file": f}) + resp = requests.post(url, params=params, headers=headers, files={"file": f}) debug_print(f"[VIDEO INDEXER] Upload response status: {resp.status_code}") @@ -461,10 +487,14 @@ def to_seconds(ts: str) -> float: return 0 # 3) Poll until ready + # Don't use includeInsights parameter - it filters what's returned. We want everything. index_url = ( f"{vi_ep}/{vi_loc}/Accounts/{vi_acc}/Videos/{vid}/Index" - f"?accessToken={token}&includeInsights=Transcript&includeStreamingUrls=false" + f"?accessToken={token}" ) + poll_headers = {} + debug_print(f"[VIDEO INDEXER] Using managed identity access token for polling") + debug_print(f"[VIDEO INDEXER] Requesting full insights (no filtering)") debug_print(f"[VIDEO INDEXER] Index polling URL: {index_url}") debug_print(f"[VIDEO INDEXER] Starting processing polling for video ID: {vid}") @@ -477,7 +507,7 @@ def to_seconds(ts: str) -> float: debug_print(f"[VIDEO INDEXER] Polling attempt {poll_count}/{max_polls}") try: - r = requests.get(index_url) + r = requests.get(index_url, headers=poll_headers) debug_print(f"[VIDEO INDEXER] Poll response status: {r.status_code}") if r.status_code in (401, 404): @@ -540,14 +570,137 @@ def to_seconds(ts: str) -> float: # 4) Extract transcript & OCR debug_print(f"[VIDEO INDEXER] Starting insights extraction for video ID: {vid}") + debug_print(f"[VIDEO INDEXER] Extracting insights from completed video") insights = info.get("insights", {}) + if not insights: + debug_print(f"[VIDEO INDEXER] ERROR: No insights object in response") + debug_print(f"[VIDEO INDEXER] Response info keys: {list(info.keys())}") + return 0 + + # Get video duration from insights (primary) or info (fallback) + video_duration = insights.get("duration") or info.get("duration", "00:00:00") + video_duration_seconds = to_seconds(video_duration) if video_duration else 0 + debug_print(f"[VIDEO INDEXER] Video duration: {video_duration} ({video_duration_seconds} seconds)") + + # Log raw insights JSON for complete visibility (debug only) + import json + print(f"\n[VIDEO] ===== RAW INSIGHTS JSON =====", flush=True) + try: + insights_json = json.dumps(insights, indent=2, ensure_ascii=False) + # Truncate if too long (show first 10000 chars) + if len(insights_json) > 10000: + print(f"{insights_json[:10000]}\n... (truncated, total length: {len(insights_json)} chars)", flush=True) + else: + print(insights_json, flush=True) + except Exception as e: + print(f"[VIDEO] Could not serialize insights to JSON: {e}", flush=True) + print(f"[VIDEO] ===== END RAW INSIGHTS =====\n", flush=True) + + debug_print(f"[VIDEO INDEXER] Insights keys available: {list(insights.keys())}") + print(f"[VIDEO] Available insight types: {', '.join(list(insights.keys())[:15])}...", flush=True) + + # Debug: Show sample structures for all insight types + print(f"\n[VIDEO] ===== SAMPLE DATA STRUCTURES =====", flush=True) + + transcript_data = insights.get("transcript", []) + if transcript_data: + print(f"[VIDEO] TRANSCRIPT sample: {transcript_data[0]}", flush=True) + + ocr_data = insights.get("ocr", []) + if ocr_data: + print(f"[VIDEO] OCR sample: {ocr_data[0]}", flush=True) + + keywords_data_debug = insights.get("keywords", []) + if keywords_data_debug: + print(f"[VIDEO] KEYWORDS sample: {keywords_data_debug[0]}", flush=True) + + labels_data_debug = insights.get("labels", []) + if labels_data_debug: + debug_print(f"[VIDEO INDEXER] LABELS sample: {labels_data_debug[0]}") + + topics_data_debug = insights.get("topics", []) + if topics_data_debug: + debug_print(f"[VIDEO INDEXER] TOPICS sample: {topics_data_debug[0]}") + + audio_effects_data_debug = insights.get("audioEffects", []) + if audio_effects_data_debug: + debug_print(f"[VIDEO INDEXER] AUDIO_EFFECTS sample: {audio_effects_data_debug[0]}") + + emotions_data_debug = insights.get("emotions", []) + if emotions_data_debug: + debug_print(f"[VIDEO INDEXER] EMOTIONS sample: {emotions_data_debug[0]}") + + sentiments_data_debug = insights.get("sentiments", []) + if sentiments_data_debug: + debug_print(f"[VIDEO INDEXER] SENTIMENTS sample: {sentiments_data_debug[0]}") + + scenes_data_debug = insights.get("scenes", []) + if scenes_data_debug: + debug_print(f"[VIDEO INDEXER] SCENES sample: {scenes_data_debug[0]}") + + shots_data_debug = insights.get("shots", []) + if shots_data_debug: + debug_print(f"[VIDEO INDEXER] SHOTS sample: {shots_data_debug[0]}") + + faces_data_debug = insights.get("faces", []) + if faces_data_debug: + debug_print(f"[VIDEO INDEXER] FACES sample: {faces_data_debug[0]}") + + namedLocations_data_debug = insights.get("namedLocations", []) + if namedLocations_data_debug: + debug_print(f"[VIDEO INDEXER] NAMED_LOCATIONS sample: {namedLocations_data_debug[0]}") + + # Check for other potential label sources + brands_data_debug = insights.get("brands", []) + if brands_data_debug: + debug_print(f"[VIDEO INDEXER] BRANDS sample: {brands_data_debug[0]}") + + visualContentModeration_debug = insights.get("visualContentModeration", []) + if visualContentModeration_debug: + debug_print(f"[VIDEO INDEXER] VISUAL_MODERATION sample: {visualContentModeration_debug[0]}") + + # Show total counts for all available insights + print(f"[VIDEO] COUNTS:", flush=True) + for key in insights.keys(): + value = insights.get(key, []) + if isinstance(value, list): + print(f" {key}: {len(value)} items", flush=True) + + print(f"[VIDEO] ===== END SAMPLE DATA =====\n", flush=True) + transcript = insights.get("transcript", []) ocr_blocks = insights.get("ocr", []) + keywords_data = insights.get("keywords", []) + labels_data = insights.get("labels", []) + topics_data = insights.get("topics", []) + audio_effects_data = insights.get("audioEffects", []) + emotions_data = insights.get("emotions", []) + sentiments_data = insights.get("sentiments", []) + named_people_data = insights.get("namedPeople", []) + named_locations_data = insights.get("namedLocations", []) + speakers_data = insights.get("speakers", []) + detected_objects_data = insights.get("detectedObjects", []) debug_print(f"[VIDEO INDEXER] Transcript segments found: {len(transcript)}") debug_print(f"[VIDEO INDEXER] OCR blocks found: {len(ocr_blocks)}") + debug_print(f"[VIDEO INDEXER] Keywords found: {len(keywords_data)}") + debug_print(f"[VIDEO INDEXER] Labels found: {len(labels_data)}") + debug_print(f"[VIDEO INDEXER] Topics found: {len(topics_data)}") + debug_print(f"[VIDEO INDEXER] Audio effects found: {len(audio_effects_data)}") + debug_print(f"[VIDEO INDEXER] Emotions found: {len(emotions_data)}") + debug_print(f"[VIDEO INDEXER] Sentiments found: {len(sentiments_data)}") + debug_print(f"[VIDEO INDEXER] Named people found: {len(named_people_data)}") + debug_print(f"[VIDEO INDEXER] Named locations found: {len(named_locations_data)}") + debug_print(f"[VIDEO INDEXER] Speakers found: {len(speakers_data)}") + debug_print(f"[VIDEO INDEXER] Detected objects found: {len(detected_objects_data)}") + debug_print(f"[VIDEO INDEXER] Insights extracted - Transcript: {len(transcript)}, OCR: {len(ocr_blocks)}, Keywords: {len(keywords_data)}, Labels: {len(labels_data)}, Topics: {len(topics_data)}, Audio: {len(audio_effects_data)}, Emotions: {len(emotions_data)}, Sentiments: {len(sentiments_data)}, People: {len(named_people_data)}, Locations: {len(named_locations_data)}, Objects: {len(detected_objects_data)}") + + if len(transcript) == 0: + debug_print(f"[VIDEO INDEXER] WARNING: No transcript data available") + debug_print(f"[VIDEO INDEXER] Available insights keys: {list(insights.keys())}") + # Build context lists for transcript and OCR speech_context = [ {"text": seg["text"].strip(), "start": inst["start"]} for seg in transcript if seg.get("text", "").strip() @@ -558,45 +711,368 @@ def to_seconds(ts: str) -> float: for block in ocr_blocks if block.get("text", "").strip() for inst in block.get("instances", []) ] + + # Build context lists for additional insights + keywords_context = [ + {"text": kw.get("name", ""), "start": inst["start"]} + for kw in keywords_data if kw.get("name", "").strip() + for inst in kw.get("instances", []) + ] + labels_context = [ + {"text": label.get("name", ""), "start": inst["start"]} + for label in labels_data if label.get("name", "").strip() + for inst in label.get("instances", []) + ] + topics_context = [ + {"text": topic.get("name", ""), "start": inst["start"]} + for topic in topics_data if topic.get("name", "").strip() + for inst in topic.get("instances", []) + ] + audio_effects_context = [ + {"text": ae.get("audioEffectType", ""), "start": inst["start"]} + for ae in audio_effects_data if ae.get("audioEffectType", "").strip() + for inst in ae.get("instances", []) + ] + emotions_context = [ + {"text": emotion.get("type", ""), "start": inst["start"]} + for emotion in emotions_data if emotion.get("type", "").strip() + for inst in emotion.get("instances", []) + ] + sentiments_context = [ + {"text": sentiment.get("sentimentType", ""), "start": inst["start"]} + for sentiment in sentiments_data if sentiment.get("sentimentType", "").strip() + for inst in sentiment.get("instances", []) + ] + named_people_context = [ + {"text": person.get("name", ""), "start": inst["start"]} + for person in named_people_data if person.get("name", "").strip() + for inst in person.get("instances", []) + ] + named_locations_context = [ + {"text": location.get("name", ""), "start": inst["start"]} + for location in named_locations_data if location.get("name", "").strip() + for inst in location.get("instances", []) + ] + detected_objects_context = [ + {"text": obj.get("type", ""), "start": inst["start"]} + for obj in detected_objects_data if obj.get("type", "").strip() + for inst in obj.get("instances", []) + ] debug_print(f"[VIDEO INDEXER] Speech context items: {len(speech_context)}") debug_print(f"[VIDEO INDEXER] OCR context items: {len(ocr_context)}") + debug_print(f"[VIDEO INDEXER] Keywords context items: {len(keywords_context)}") + debug_print(f"[VIDEO INDEXER] Labels context items: {len(labels_context)}") + debug_print(f"[VIDEO INDEXER] Topics context items: {len(topics_context)}") + debug_print(f"[VIDEO INDEXER] Audio effects context items: {len(audio_effects_context)}") + debug_print(f"[VIDEO INDEXER] Emotions context items: {len(emotions_context)}") + debug_print(f"[VIDEO INDEXER] Sentiments context items: {len(sentiments_context)}") + debug_print(f"[VIDEO INDEXER] Named people context items: {len(named_people_context)}") + debug_print(f"[VIDEO INDEXER] Named locations context items: {len(named_locations_context)}") + debug_print(f"[VIDEO INDEXER] Detected objects context items: {len(detected_objects_context)}") + debug_print(f"[VIDEO INDEXER] Context built - Speech: {len(speech_context)}, OCR: {len(ocr_context)}, Keywords: {len(keywords_context)}, Labels: {len(labels_context)}, People: {len(named_people_context)}, Locations: {len(named_locations_context)}, Objects: {len(detected_objects_context)}") + + if len(speech_context) > 0: + debug_print(f"[VIDEO INDEXER] First speech item: {speech_context[0]}") + # Sort all contexts by timestamp speech_context.sort(key=lambda x: to_seconds(x["start"])) ocr_context.sort(key=lambda x: to_seconds(x["start"])) + keywords_context.sort(key=lambda x: to_seconds(x["start"])) + labels_context.sort(key=lambda x: to_seconds(x["start"])) + topics_context.sort(key=lambda x: to_seconds(x["start"])) + audio_effects_context.sort(key=lambda x: to_seconds(x["start"])) + emotions_context.sort(key=lambda x: to_seconds(x["start"])) + sentiments_context.sort(key=lambda x: to_seconds(x["start"])) + named_people_context.sort(key=lambda x: to_seconds(x["start"])) + named_locations_context.sort(key=lambda x: to_seconds(x["start"])) + detected_objects_context.sort(key=lambda x: to_seconds(x["start"])) debug_print(f"[VIDEO INDEXER] Starting 30-second chunk processing") + debug_print(f"[VIDEO INDEXER] Starting time-based chunk processing - Video duration: {video_duration_seconds}s") + debug_print(f"[VIDEO INDEXER] Available insights - Speech: {len(speech_context)}, OCR: {len(ocr_context)}, Keywords: {len(keywords_context)}, Labels: {len(labels_context)}") + + # Check if we have any content at all + total_insights = len(speech_context) + len(ocr_context) + len(keywords_context) + len(labels_context) + len(topics_context) + len(audio_effects_context) + len(emotions_context) + len(sentiments_context) + len(named_people_context) + len(named_locations_context) + len(detected_objects_context) + + if total_insights == 0 and video_duration_seconds == 0: + debug_print(f"[VIDEO INDEXER] ERROR: No insights and no duration information available") + update_callback(status="VIDEO: no data available") + return 0 + + # Use video duration to create time-based chunks, even without speech + if video_duration_seconds == 0: + debug_print(f"[VIDEO INDEXER] WARNING: No video duration available, estimating from insights") + # Estimate duration from the latest timestamp in any insight + max_timestamp = 0 + for context_list in [speech_context, ocr_context, keywords_context, labels_context, topics_context, audio_effects_context, emotions_context, sentiments_context, named_people_context, named_locations_context, detected_objects_context]: + if context_list: + max_ts = max(to_seconds(item["start"]) for item in context_list) + max_timestamp = max(max_timestamp, max_ts) + video_duration_seconds = max_timestamp + 30 # Add buffer + debug_print(f"[VIDEO INDEXER] Estimated duration: {video_duration_seconds}s") + + # Create chunks based on time intervals (30 seconds each) + num_chunks = int(video_duration_seconds / 30) + (1 if video_duration_seconds % 30 > 0 else 0) + debug_print(f"[VIDEO INDEXER] Will create {num_chunks} time-based chunks") total = 0 idx_s = 0 n_s = len(speech_context) idx_o = 0 n_o = len(ocr_context) + idx_kw = 0 + n_kw = len(keywords_context) + idx_lbl = 0 + n_lbl = len(labels_context) + idx_top = 0 + n_top = len(topics_context) + idx_ae = 0 + n_ae = len(audio_effects_context) + idx_emo = 0 + n_emo = len(emotions_context) + idx_sent = 0 + n_sent = len(sentiments_context) + idx_people = 0 + n_people = len(named_people_context) + idx_locations = 0 + n_locations = len(named_locations_context) + idx_objects = 0 + n_objects = len(detected_objects_context) + + # Process chunks in 30-second intervals based on video duration + for chunk_num in range(num_chunks): + window_start = chunk_num * 30.0 + window_end = min((chunk_num + 1) * 30.0, video_duration_seconds) + + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} window: {window_start}s to {window_end}s") - while idx_s < n_s: - window_start = to_seconds(speech_context[idx_s]["start"]) - window_end = window_start + 30.0 - + # Collect speech for this time window speech_lines = [] - while idx_s < n_s and to_seconds(speech_context[idx_s]["start"]) <= window_end: - speech_lines.append(speech_context[idx_s]["text"]) + while idx_s < n_s and to_seconds(speech_context[idx_s]["start"]) < window_end: + if to_seconds(speech_context[idx_s]["start"]) >= window_start: + speech_lines.append(speech_context[idx_s]["text"]) idx_s += 1 + if idx_s < n_s and to_seconds(speech_context[idx_s]["start"]) >= window_end: + break + + # Reset idx_s if we went past window_end + while idx_s > 0 and idx_s < n_s and to_seconds(speech_context[idx_s]["start"]) >= window_end: + idx_s -= 1 + if idx_s < n_s and to_seconds(speech_context[idx_s]["start"]) < window_end: + idx_s += 1 + + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} speech lines collected: {len(speech_lines)}") + # Collect OCR for this time window ocr_lines = [] - while idx_o < n_o and to_seconds(ocr_context[idx_o]["start"]) <= window_end: - ocr_lines.append(ocr_context[idx_o]["text"]) + while idx_o < n_o and to_seconds(ocr_context[idx_o]["start"]) < window_end: + if to_seconds(ocr_context[idx_o]["start"]) >= window_start: + ocr_lines.append(ocr_context[idx_o]["text"]) idx_o += 1 - - start_ts = speech_context[total]["start"] + if idx_o < n_o and to_seconds(ocr_context[idx_o]["start"]) >= window_end: + break + + while idx_o > 0 and idx_o < n_o and to_seconds(ocr_context[idx_o]["start"]) >= window_end: + idx_o -= 1 + if idx_o < n_o and to_seconds(ocr_context[idx_o]["start"]) < window_end: + idx_o += 1 + + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} OCR lines collected: {len(ocr_lines)}") + + # Collect keywords for this time window + chunk_keywords = [] + while idx_kw < n_kw and to_seconds(keywords_context[idx_kw]["start"]) < window_end: + if to_seconds(keywords_context[idx_kw]["start"]) >= window_start: + chunk_keywords.append(keywords_context[idx_kw]["text"]) + idx_kw += 1 + if idx_kw < n_kw and to_seconds(keywords_context[idx_kw]["start"]) >= window_end: + break + while idx_kw > 0 and idx_kw < n_kw and to_seconds(keywords_context[idx_kw]["start"]) >= window_end: + idx_kw -= 1 + if idx_kw < n_kw and to_seconds(keywords_context[idx_kw]["start"]) < window_end: + idx_kw += 1 + + # Collect labels for this time window + chunk_labels = [] + while idx_lbl < n_lbl and to_seconds(labels_context[idx_lbl]["start"]) < window_end: + if to_seconds(labels_context[idx_lbl]["start"]) >= window_start: + chunk_labels.append(labels_context[idx_lbl]["text"]) + idx_lbl += 1 + if idx_lbl < n_lbl and to_seconds(labels_context[idx_lbl]["start"]) >= window_end: + break + while idx_lbl > 0 and idx_lbl < n_lbl and to_seconds(labels_context[idx_lbl]["start"]) >= window_end: + idx_lbl -= 1 + if idx_lbl < n_lbl and to_seconds(labels_context[idx_lbl]["start"]) < window_end: + idx_lbl += 1 + + # Collect topics for this time window + chunk_topics = [] + while idx_top < n_top and to_seconds(topics_context[idx_top]["start"]) < window_end: + if to_seconds(topics_context[idx_top]["start"]) >= window_start: + chunk_topics.append(topics_context[idx_top]["text"]) + idx_top += 1 + if idx_top < n_top and to_seconds(topics_context[idx_top]["start"]) >= window_end: + break + while idx_top > 0 and idx_top < n_top and to_seconds(topics_context[idx_top]["start"]) >= window_end: + idx_top -= 1 + if idx_top < n_top and to_seconds(topics_context[idx_top]["start"]) < window_end: + idx_top += 1 + + # Collect audio effects for this time window + chunk_audio_effects = [] + while idx_ae < n_ae and to_seconds(audio_effects_context[idx_ae]["start"]) < window_end: + if to_seconds(audio_effects_context[idx_ae]["start"]) >= window_start: + chunk_audio_effects.append(audio_effects_context[idx_ae]["text"]) + idx_ae += 1 + if idx_ae < n_ae and to_seconds(audio_effects_context[idx_ae]["start"]) >= window_end: + break + while idx_ae > 0 and idx_ae < n_ae and to_seconds(audio_effects_context[idx_ae]["start"]) >= window_end: + idx_ae -= 1 + if idx_ae < n_ae and to_seconds(audio_effects_context[idx_ae]["start"]) < window_end: + idx_ae += 1 + + # Collect emotions for this time window + chunk_emotions = [] + while idx_emo < n_emo and to_seconds(emotions_context[idx_emo]["start"]) < window_end: + if to_seconds(emotions_context[idx_emo]["start"]) >= window_start: + chunk_emotions.append(emotions_context[idx_emo]["text"]) + idx_emo += 1 + if idx_emo < n_emo and to_seconds(emotions_context[idx_emo]["start"]) >= window_end: + break + while idx_emo > 0 and idx_emo < n_emo and to_seconds(emotions_context[idx_emo]["start"]) >= window_end: + idx_emo -= 1 + if idx_emo < n_emo and to_seconds(emotions_context[idx_emo]["start"]) < window_end: + idx_emo += 1 + + # Collect sentiments for this time window + chunk_sentiments = [] + while idx_sent < n_sent and to_seconds(sentiments_context[idx_sent]["start"]) < window_end: + if to_seconds(sentiments_context[idx_sent]["start"]) >= window_start: + chunk_sentiments.append(sentiments_context[idx_sent]["text"]) + idx_sent += 1 + if idx_sent < n_sent and to_seconds(sentiments_context[idx_sent]["start"]) >= window_end: + break + while idx_sent > 0 and idx_sent < n_sent and to_seconds(sentiments_context[idx_sent]["start"]) >= window_end: + idx_sent -= 1 + if idx_sent < n_sent and to_seconds(sentiments_context[idx_sent]["start"]) < window_end: + idx_sent += 1 + + # Collect named people for this time window + chunk_people = [] + while idx_people < n_people and to_seconds(named_people_context[idx_people]["start"]) < window_end: + if to_seconds(named_people_context[idx_people]["start"]) >= window_start: + chunk_people.append(named_people_context[idx_people]["text"]) + idx_people += 1 + if idx_people < n_people and to_seconds(named_people_context[idx_people]["start"]) >= window_end: + break + while idx_people > 0 and idx_people < n_people and to_seconds(named_people_context[idx_people]["start"]) >= window_end: + idx_people -= 1 + if idx_people < n_people and to_seconds(named_people_context[idx_people]["start"]) < window_end: + idx_people += 1 + + # Collect named locations for this time window + chunk_locations = [] + while idx_locations < n_locations and to_seconds(named_locations_context[idx_locations]["start"]) < window_end: + if to_seconds(named_locations_context[idx_locations]["start"]) >= window_start: + chunk_locations.append(named_locations_context[idx_locations]["text"]) + idx_locations += 1 + if idx_locations < n_locations and to_seconds(named_locations_context[idx_locations]["start"]) >= window_end: + break + while idx_locations > 0 and idx_locations < n_locations and to_seconds(named_locations_context[idx_locations]["start"]) >= window_end: + idx_locations -= 1 + if idx_locations < n_locations and to_seconds(named_locations_context[idx_locations]["start"]) < window_end: + idx_locations += 1 + + # Collect detected objects for this time window + chunk_objects = [] + while idx_objects < n_objects and to_seconds(detected_objects_context[idx_objects]["start"]) < window_end: + if to_seconds(detected_objects_context[idx_objects]["start"]) >= window_start: + chunk_objects.append(detected_objects_context[idx_objects]["text"]) + idx_objects += 1 + if idx_objects < n_objects and to_seconds(detected_objects_context[idx_objects]["start"]) >= window_end: + break + while idx_objects > 0 and idx_objects < n_objects and to_seconds(detected_objects_context[idx_objects]["start"]) >= window_end: + idx_objects -= 1 + if idx_objects < n_objects and to_seconds(detected_objects_context[idx_objects]["start"]) < window_end: + idx_objects += 1 + + # Format timestamp as HH:MM:SS + hours = int(window_start // 3600) + minutes = int((window_start % 3600) // 60) + seconds = int(window_start % 60) + start_ts = f"{hours:02d}:{minutes:02d}:{seconds:02d}.000" + chunk_text = " ".join(speech_lines).strip() ocr_text = " ".join(ocr_lines).strip() + + # Build enhanced chunk text with insights appended + if chunk_text: + # Has speech - append insights to it + insight_parts = [] + if chunk_keywords: + insight_parts.append(f"Keywords: {', '.join(chunk_keywords)}") + if chunk_labels: + insight_parts.append(f"Visual elements: {', '.join(chunk_labels)}") + if chunk_topics: + insight_parts.append(f"Topics: {', '.join(chunk_topics)}") + if chunk_audio_effects: + insight_parts.append(f"Audio: {', '.join(chunk_audio_effects)}") + if chunk_emotions: + insight_parts.append(f"Emotions: {', '.join(chunk_emotions)}") + if chunk_sentiments: + insight_parts.append(f"Sentiment: {', '.join(chunk_sentiments)}") + if chunk_people: + insight_parts.append(f"People: {', '.join(chunk_people)}") + if chunk_locations: + insight_parts.append(f"Locations: {', '.join(chunk_locations)}") + if chunk_objects: + insight_parts.append(f"Objects: {', '.join(chunk_objects)}") + + if insight_parts: + chunk_text = f"{chunk_text}\n\n{' | '.join(insight_parts)}" + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} enhanced with {len(insight_parts)} insight types") + else: + # No speech - build chunk text from other insights + insight_parts = [] + if ocr_text: + insight_parts.append(f"Visual text: {ocr_text}") + if chunk_keywords: + insight_parts.append(f"Keywords: {', '.join(chunk_keywords)}") + if chunk_labels: + insight_parts.append(f"Visual elements: {', '.join(chunk_labels)}") + if chunk_topics: + insight_parts.append(f"Topics: {', '.join(chunk_topics)}") + if chunk_audio_effects: + insight_parts.append(f"Audio: {', '.join(chunk_audio_effects)}") + if chunk_emotions: + insight_parts.append(f"Emotions: {', '.join(chunk_emotions)}") + if chunk_sentiments: + insight_parts.append(f"Sentiment: {', '.join(chunk_sentiments)}") + if chunk_people: + insight_parts.append(f"People: {', '.join(chunk_people)}") + if chunk_locations: + insight_parts.append(f"Locations: {', '.join(chunk_locations)}") + if chunk_objects: + insight_parts.append(f"Objects: {', '.join(chunk_objects)}") + + chunk_text = ". ".join(insight_parts) if insight_parts else "[No content detected]" + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} has no speech, using insights as text: {chunk_text[:100]}...") - debug_print(f"[VIDEO INDEXER] Processing chunk {total + 1} at timestamp {start_ts}") - debug_print(f"[VIDEO INDEXER] Chunk text length: {len(chunk_text)}, OCR text length: {len(ocr_text)}") + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} at timestamp {start_ts}") + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} text length: {len(chunk_text)}, OCR text length: {len(ocr_text)}") + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} insights - Keywords: {len(chunk_keywords)}, Labels: {len(chunk_labels)}, Topics: {len(chunk_topics)}, Audio: {len(chunk_audio_effects)}, Emotions: {len(chunk_emotions)}, Sentiments: {len(chunk_sentiments)}, People: {len(chunk_people)}, Locations: {len(chunk_locations)}, Objects: {len(chunk_objects)}") + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1}: timestamp={start_ts}, text_len={len(chunk_text)}, ocr_len={len(ocr_text)}, insights={len(chunk_keywords)}kw/{len(chunk_labels)}lbl/{len(chunk_topics)}top") + + # Skip truly empty chunks (no content at all) + if chunk_text == "[No content detected]" and not any([chunk_keywords, chunk_labels, chunk_topics, chunk_audio_effects, chunk_emotions, chunk_sentiments, chunk_people, chunk_locations, chunk_objects]): + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} is completely empty, skipping") + continue - update_callback(current_file_chunk=total+1, status=f"VIDEO: saving chunk @ {start_ts}") + update_callback(current_file_chunk=chunk_num+1, status=f"VIDEO: saving chunk @ {start_ts}") try: + debug_print(f"[VIDEO INDEXER] Calling save_video_chunk for chunk {chunk_num + 1}") save_video_chunk( page_text_content=chunk_text, ocr_chunk_text=ocr_text, @@ -606,12 +1082,14 @@ def to_seconds(ts: str) -> float: document_id=document_id, group_id=group_id ) - debug_print(f"[VIDEO INDEXER] Chunk {total + 1} saved successfully") + debug_print(f"[VIDEO INDEXER] Chunk {chunk_num + 1} saved successfully") + total += 1 except Exception as e: - debug_print(f"[VIDEO INDEXER] Failed to save chunk {total + 1}: {str(e)}") - print(f"[VIDEO] CHUNK SAVE ERROR for chunk {total + 1}: {e}", flush=True) - - total += 1 + debug_print(f"[VIDEO INDEXER] Failed to save chunk {chunk_num + 1}: {str(e)}") + import traceback + debug_print(f"[VIDEO INDEXER] Chunk save traceback: {traceback.format_exc()}") + + debug_print(f"[VIDEO INDEXER] Chunk processing complete - Total chunks saved: {total}") # Extract metadata if enabled and chunks were processed settings = get_settings() @@ -1036,13 +1514,47 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, chunk_summary = "" author = [] title = "" + + # Check if this document has vision analysis and append it to chunk_text + vision_analysis = metadata.get('vision_analysis') + enhanced_chunk_text = page_text_content + + if vision_analysis: + debug_print(f"[SAVE_CHUNKS] Document {document_id} has vision analysis, appending to chunk_text") + # Format vision analysis as structured text for better searchability + vision_text_parts = [] + vision_text_parts.append("\n\n=== AI Vision Analysis ===") + vision_text_parts.append(f"Model: {vision_analysis.get('model', 'unknown')}") + + if vision_analysis.get('description'): + vision_text_parts.append(f"\nDescription: {vision_analysis['description']}") + + if vision_analysis.get('objects'): + objects_list = vision_analysis['objects'] + if isinstance(objects_list, list): + vision_text_parts.append(f"\nObjects Detected: {', '.join(objects_list)}") + else: + vision_text_parts.append(f"\nObjects Detected: {objects_list}") + + if vision_analysis.get('text'): + vision_text_parts.append(f"\nVisible Text: {vision_analysis['text']}") + + if vision_analysis.get('analysis'): + vision_text_parts.append(f"\nContextual Analysis: {vision_analysis['analysis']}") + + vision_text = "\n".join(vision_text_parts) + enhanced_chunk_text = page_text_content + vision_text + + debug_print(f"[SAVE_CHUNKS] Enhanced chunk_text length: {len(enhanced_chunk_text)} (original: {len(page_text_content)}, vision: {len(vision_text)})") + else: + debug_print(f"[SAVE_CHUNKS] No vision analysis found for document {document_id}") if is_public_workspace: chunk_document = { "id": chunk_id, "document_id": document_id, "chunk_id": str(page_number), - "chunk_text": page_text_content, + "chunk_text": enhanced_chunk_text, "embedding": embedding, "file_name": file_name, "chunk_keywords": chunk_keywords, @@ -1063,7 +1575,7 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, "id": chunk_id, "document_id": document_id, "chunk_id": str(page_number), - "chunk_text": page_text_content, + "chunk_text": enhanced_chunk_text, "embedding": embedding, "file_name": file_name, "chunk_keywords": chunk_keywords, @@ -1086,7 +1598,7 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, "id": chunk_id, "document_id": document_id, "chunk_id": str(page_number), - "chunk_text": page_text_content, + "chunk_text": enhanced_chunk_text, "embedding": embedding, "file_name": file_name, "chunk_keywords": chunk_keywords, @@ -1123,6 +1635,57 @@ def save_chunks(page_text_content, page_number, file_name, user_id, document_id, print(f"Error uploading chunk document for document {document_id}: {e}") raise +def get_document_metadata_for_citations(document_id, user_id=None, group_id=None, public_workspace_id=None): + """ + Retrieve keywords and abstract from a document for creating metadata citations. + Used to enhance search results with additional context from document metadata. + + Args: + document_id: The document's unique identifier + user_id: User ID (for personal documents) + group_id: Group ID (for group documents) + public_workspace_id: Public workspace ID (for public documents) + + Returns: + dict: Dictionary with 'keywords' and 'abstract' fields, or None if document not found + """ + is_group = group_id is not None + is_public_workspace = public_workspace_id is not None + + # Determine the correct container + if is_public_workspace: + cosmos_container = cosmos_public_documents_container + elif is_group: + cosmos_container = cosmos_group_documents_container + else: + cosmos_container = cosmos_user_documents_container + + try: + # Read the document directly by ID + document_item = cosmos_container.read_item( + item=document_id, + partition_key=document_id + ) + + # Extract keywords and abstract + keywords = document_item.get('keywords', []) + abstract = document_item.get('abstract', '') + + # Return only if we have actual content + if keywords or abstract: + return { + 'keywords': keywords if keywords else [], + 'abstract': abstract if abstract else '', + 'file_name': document_item.get('file_name', 'Unknown') + } + + return None + + except Exception as e: + # Document not found or error reading - return None silently + # This is expected for documents without metadata + return None + def get_all_chunks(document_id, user_id, group_id=None, public_workspace_id=None): is_group = group_id is not None is_public_workspace = public_workspace_id is not None @@ -1624,58 +2187,8 @@ def delete_document(user_id, document_id, group_id=None, public_workspace_id=Non # Get the file name from the document to use for blob deletion file_name = document_item.get('file_name') - file_ext = os.path.splitext(file_name)[1].lower() if file_name else None - # First try to delete video from Video Indexer if applicable - if file_ext in ('.mp4', '.mov', '.avi', '.mkv', '.flv'): - debug_print(f"[VIDEO INDEXER DELETE] Video file detected, attempting Video Indexer deletion for document: {document_id}") - try: - settings = get_settings() - vi_ep = settings.get("video_indexer_endpoint") - vi_loc = settings.get("video_indexer_location") - vi_acc = settings.get("video_indexer_account_id") - - debug_print(f"[VIDEO INDEXER DELETE] Configuration - Endpoint: {vi_ep}, Location: {vi_loc}, Account ID: {vi_acc}") - - if not all([vi_ep, vi_loc, vi_acc]): - debug_print(f"[VIDEO INDEXER DELETE] Missing video indexer configuration, skipping deletion") - print("Missing video indexer configuration; skipping Video Indexer deletion.") - else: - debug_print(f"[VIDEO INDEXER DELETE] Acquiring authentication token") - token = get_video_indexer_account_token(settings) - debug_print(f"[VIDEO INDEXER DELETE] Token acquired successfully") - - # You need to store the video ID in the document metadata when uploading - video_id = document_item.get("video_indexer_id") - debug_print(f"[VIDEO INDEXER DELETE] Video ID from document metadata: {video_id}") - - if video_id: - delete_url = f"{vi_ep}/{vi_loc}/Accounts/{vi_acc}/Videos/{video_id}?accessToken={token}" - debug_print(f"[VIDEO INDEXER DELETE] Delete URL: {delete_url}") - - resp = requests.delete(delete_url, timeout=60) - debug_print(f"[VIDEO INDEXER DELETE] Delete response status: {resp.status_code}") - - if resp.status_code != 200: - debug_print(f"[VIDEO INDEXER DELETE] Delete response text: {resp.text}") - - resp.raise_for_status() - debug_print(f"[VIDEO INDEXER DELETE] Successfully deleted video ID: {video_id}") - print(f"Deleted video from Video Indexer: {video_id}") - else: - debug_print(f"[VIDEO INDEXER DELETE] No video_indexer_id found in document metadata") - print("No video_indexer_id found in document metadata; skipping Video Indexer deletion.") - except requests.exceptions.RequestException as e: - debug_print(f"[VIDEO INDEXER DELETE] Request error: {str(e)}") - if hasattr(e, 'response') and e.response is not None: - debug_print(f"[VIDEO INDEXER DELETE] Error response status: {e.response.status_code}") - debug_print(f"[VIDEO INDEXER DELETE] Error response text: {e.response.text}") - print(f"Error deleting video from Video Indexer: {e}") - except Exception as e: - debug_print(f"[VIDEO INDEXER DELETE] Unexpected error: {str(e)}") - print(f"Error deleting video from Video Indexer: {e}") - - # Second try to delete from blob storage + # Delete from blob storage try: if file_name: delete_from_blob_storage(document_id, user_id, file_name, group_id, public_workspace_id) @@ -2470,6 +2983,146 @@ def estimate_word_count(text): return 0 return len(text.split()) +def analyze_image_with_vision_model(image_path, user_id, document_id, settings): + """ + Analyze image using GPT-4 Vision or similar multimodal model. + + Args: + image_path: Path to image file + user_id: User ID for logging + document_id: Document ID for tracking + settings: Application settings + + Returns: + dict: { + 'description': 'AI-generated image description', + 'objects': ['list', 'of', 'detected', 'objects'], + 'text': 'any text visible in image', + 'analysis': 'detailed analysis' + } or None if vision analysis is disabled or fails + """ + if not settings.get('enable_multimodal_vision', False): + return None + + try: + # Convert image to base64 + with open(image_path, 'rb') as img_file: + image_bytes = img_file.read() + base64_image = base64.b64encode(image_bytes).decode('utf-8') + + # Determine image mime type + mime_type = mimetypes.guess_type(image_path)[0] or 'image/jpeg' + + # Get vision model settings + vision_model = settings.get('multimodal_vision_model', 'gpt-4o') + + if not vision_model: + print(f"Warning: Multi-modal vision enabled but no model selected") + return None + + # Initialize client (reuse GPT configuration) + enable_gpt_apim = settings.get('enable_gpt_apim', False) + + if enable_gpt_apim: + gpt_client = AzureOpenAI( + api_version=settings.get('azure_apim_gpt_api_version'), + azure_endpoint=settings.get('azure_apim_gpt_endpoint'), + api_key=settings.get('azure_apim_gpt_subscription_key') + ) + else: + # Use managed identity or key + auth_type = settings.get('azure_openai_gpt_authentication_type', 'key') + if auth_type == 'managed_identity': + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + cognitive_services_scope + ) + gpt_client = AzureOpenAI( + api_version=settings.get('azure_openai_gpt_api_version'), + azure_endpoint=settings.get('azure_openai_gpt_endpoint'), + azure_ad_token_provider=token_provider + ) + else: + gpt_client = AzureOpenAI( + api_version=settings.get('azure_openai_gpt_api_version'), + azure_endpoint=settings.get('azure_openai_gpt_endpoint'), + api_key=settings.get('azure_openai_gpt_key') + ) + + # Create vision prompt + print(f"Analyzing image with vision model: {vision_model}") + + response = gpt_client.chat.completions.create( + model=vision_model, + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": """Analyze this image and provide: +1. A detailed description of what you see +2. List any objects, people, or notable elements +3. Extract any visible text (OCR) +4. Provide contextual analysis or insights + +Format your response as JSON with these keys: +{ + "description": "...", + "objects": ["...", "..."], + "text": "...", + "analysis": "..." +}""" + }, + { + "type": "image_url", + "image_url": { + "url": f"data:{mime_type};base64,{base64_image}" + } + } + ] + } + ], + max_tokens=1000 + ) + + # Parse response + content = response.choices[0].message.content + + debug_print(f"[VISION_ANALYSIS] Raw response for {document_id}: {content[:500]}...") + + # Try to parse as JSON, fallback to raw text + try: + # Clean up potential markdown code fences + content_cleaned = clean_json_codeFence(content) + vision_analysis = json.loads(content_cleaned) + debug_print(f"[VISION_ANALYSIS] Parsed JSON successfully for {document_id}") + except Exception as parse_error: + debug_print(f"[VISION_ANALYSIS] Vision response not valid JSON: {parse_error}") + print(f"Vision response not valid JSON, using raw text") + vision_analysis = { + 'description': content, + 'raw_response': content + } + + # Add model info to analysis + vision_analysis['model'] = vision_model + + debug_print(f"[VISION_ANALYSIS] Complete analysis for {document_id}:") + debug_print(f" Model: {vision_model}") + debug_print(f" Description: {vision_analysis.get('description', 'N/A')[:200]}...") + debug_print(f" Objects: {vision_analysis.get('objects', [])}") + debug_print(f" Text: {vision_analysis.get('text', 'N/A')[:100]}...") + + print(f"Vision analysis completed for document: {document_id}") + return vision_analysis + + except Exception as e: + print(f"Error in vision analysis for {document_id}: {str(e)}") + import traceback + traceback.print_exc() + return None + def upload_to_blob(temp_file_path, user_id, document_id, blob_filename, update_callback, group_id=None, public_workspace_id=None): """Uploads the file to Azure Blob Storage.""" @@ -2585,6 +3238,355 @@ def process_txt(document_id, user_id, temp_file_path, original_filename, enable_ return total_chunks_saved +def process_xml(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None): + """Processes XML files using RecursiveCharacterTextSplitter for structured content.""" + is_group = group_id is not None + is_public_workspace = public_workspace_id is not None + + update_callback(status="Processing XML file...") + total_chunks_saved = 0 + # Character-based chunking for XML structure preservation + max_chunk_size_chars = 4000 + + if enable_enhanced_citations: + args = { + "temp_file_path": temp_file_path, + "user_id": user_id, + "document_id": document_id, + "blob_filename": original_filename, + "update_callback": update_callback + } + + if is_group: + args["group_id"] = group_id + elif is_public_workspace: + args["public_workspace_id"] = public_workspace_id + + upload_to_blob(**args) + + try: + # Read XML content + try: + with open(temp_file_path, 'r', encoding='utf-8') as f: + xml_content = f.read() + except Exception as e: + raise Exception(f"Error reading XML file {original_filename}: {e}") + + # Use RecursiveCharacterTextSplitter with XML-aware separators + # This preserves XML structure better than simple word splitting + xml_splitter = RecursiveCharacterTextSplitter( + chunk_size=max_chunk_size_chars, + chunk_overlap=0, + length_function=len, + separators=["\n\n", "\n", ">", " ", ""], # XML-friendly separators + is_separator_regex=False + ) + + # Split the XML content + final_chunks = xml_splitter.split_text(xml_content) + + initial_chunk_count = len(final_chunks) + update_callback(number_of_pages=initial_chunk_count) + + for idx, chunk_content in enumerate(final_chunks, start=1): + # Skip empty chunks + if not chunk_content or not chunk_content.strip(): + print(f"Skipping empty XML chunk {idx}/{initial_chunk_count}") + continue + + update_callback( + current_file_chunk=idx, + status=f"Saving chunk {idx}/{initial_chunk_count}..." + ) + args = { + "page_text_content": chunk_content, + "page_number": total_chunks_saved + 1, + "file_name": original_filename, + "user_id": user_id, + "document_id": document_id + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + save_chunks(**args) + total_chunks_saved += 1 + + # Final update with actual chunks saved + if total_chunks_saved != initial_chunk_count: + update_callback(number_of_pages=total_chunks_saved) + print(f"Adjusted final chunk count from {initial_chunk_count} to {total_chunks_saved} after skipping empty chunks.") + + except Exception as e: + print(f"Error during XML processing for {original_filename}: {type(e).__name__}: {e}") + raise Exception(f"Failed processing XML file {original_filename}: {e}") + + return total_chunks_saved + +def process_yaml(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None): + """Processes YAML files using RecursiveCharacterTextSplitter for structured content.""" + is_group = group_id is not None + is_public_workspace = public_workspace_id is not None + + update_callback(status="Processing YAML file...") + total_chunks_saved = 0 + # Character-based chunking for YAML structure preservation + max_chunk_size_chars = 4000 + + if enable_enhanced_citations: + args = { + "temp_file_path": temp_file_path, + "user_id": user_id, + "document_id": document_id, + "blob_filename": original_filename, + "update_callback": update_callback + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + upload_to_blob(**args) + + try: + # Read YAML content + try: + with open(temp_file_path, 'r', encoding='utf-8') as f: + yaml_content = f.read() + except Exception as e: + raise Exception(f"Error reading YAML file {original_filename}: {e}") + + # Use RecursiveCharacterTextSplitter with YAML-aware separators + # This preserves YAML structure better than simple word splitting + yaml_splitter = RecursiveCharacterTextSplitter( + chunk_size=max_chunk_size_chars, + chunk_overlap=0, + length_function=len, + separators=["\n\n", "\n", "- ", " ", ""], # YAML-friendly separators + is_separator_regex=False + ) + + # Split the YAML content + final_chunks = yaml_splitter.split_text(yaml_content) + + initial_chunk_count = len(final_chunks) + update_callback(number_of_pages=initial_chunk_count) + + for idx, chunk_content in enumerate(final_chunks, start=1): + # Skip empty chunks + if not chunk_content or not chunk_content.strip(): + print(f"Skipping empty YAML chunk {idx}/{initial_chunk_count}") + continue + + update_callback( + current_file_chunk=idx, + status=f"Saving chunk {idx}/{initial_chunk_count}..." + ) + args = { + "page_text_content": chunk_content, + "page_number": total_chunks_saved + 1, + "file_name": original_filename, + "user_id": user_id, + "document_id": document_id + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + save_chunks(**args) + total_chunks_saved += 1 + + # Final update with actual chunks saved + if total_chunks_saved != initial_chunk_count: + update_callback(number_of_pages=total_chunks_saved) + print(f"Adjusted final chunk count from {initial_chunk_count} to {total_chunks_saved} after skipping empty chunks.") + + except Exception as e: + print(f"Error during YAML processing for {original_filename}: {type(e).__name__}: {e}") + raise Exception(f"Failed processing YAML file {original_filename}: {e}") + + return total_chunks_saved + +def process_log(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None): + """Processes LOG files using line-based chunking to maintain log record integrity.""" + is_group = group_id is not None + is_public_workspace = public_workspace_id is not None + + update_callback(status="Processing LOG file...") + total_chunks_saved = 0 + target_words_per_chunk = 1000 # Word-based chunking for better semantic grouping + + if enable_enhanced_citations: + args = { + "temp_file_path": temp_file_path, + "user_id": user_id, + "document_id": document_id, + "blob_filename": original_filename, + "update_callback": update_callback + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + upload_to_blob(**args) + + try: + with open(temp_file_path, 'r', encoding='utf-8') as f: + content = f.read() + + # Split by lines to maintain log record integrity + lines = content.splitlines(keepends=True) # Keep line endings + + if not lines: + raise Exception(f"LOG file {original_filename} is empty") + + # Chunk by accumulating lines until reaching target word count + final_chunks = [] + current_chunk_lines = [] + current_chunk_word_count = 0 + + for line in lines: + line_word_count = len(line.split()) + + # If adding this line exceeds target AND we already have content + if current_chunk_word_count + line_word_count > target_words_per_chunk and current_chunk_lines: + # Finalize current chunk + final_chunks.append("".join(current_chunk_lines)) + # Start new chunk with current line + current_chunk_lines = [line] + current_chunk_word_count = line_word_count + else: + # Add line to current chunk + current_chunk_lines.append(line) + current_chunk_word_count += line_word_count + + # Add the last remaining chunk if it has content + if current_chunk_lines: + final_chunks.append("".join(current_chunk_lines)) + + num_chunks = len(final_chunks) + update_callback(number_of_pages=num_chunks) + + for idx, chunk_content in enumerate(final_chunks, start=1): + if chunk_content.strip(): + update_callback( + current_file_chunk=idx, + status=f"Saving chunk {idx}/{num_chunks}..." + ) + args = { + "page_text_content": chunk_content, + "page_number": idx, + "file_name": original_filename, + "user_id": user_id, + "document_id": document_id + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + save_chunks(**args) + total_chunks_saved += 1 + + except Exception as e: + raise Exception(f"Failed processing LOG file {original_filename}: {e}") + + return total_chunks_saved + +def process_doc(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None): + """ + Processes .doc and .docm files using docx2txt library. + Note: .docx files still use Document Intelligence for better formatting preservation. + """ + is_group = group_id is not None + is_public_workspace = public_workspace_id is not None + + update_callback(status=f"Processing {original_filename.split('.')[-1].upper()} file...") + total_chunks_saved = 0 + target_words_per_chunk = 400 # Consistent with other text-based chunking + + if enable_enhanced_citations: + args = { + "temp_file_path": temp_file_path, + "user_id": user_id, + "document_id": document_id, + "blob_filename": original_filename, + "update_callback": update_callback + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + upload_to_blob(**args) + + try: + # Import docx2txt here to avoid dependency issues if not installed + try: + import docx2txt + except ImportError: + raise Exception("docx2txt library is required for .doc and .docm file processing. Install with: pip install docx2txt") + + # Extract text from .doc or .docm file + try: + text_content = docx2txt.process(temp_file_path) + except Exception as e: + raise Exception(f"Error extracting text from {original_filename}: {e}") + + if not text_content or not text_content.strip(): + raise Exception(f"No text content extracted from {original_filename}") + + # Split into words for chunking + words = text_content.split() + if not words: + raise Exception(f"No text content found in {original_filename}") + + # Create chunks of target_words_per_chunk words + final_chunks = [] + for i in range(0, len(words), target_words_per_chunk): + chunk_words = words[i:i + target_words_per_chunk] + chunk_text = " ".join(chunk_words) + final_chunks.append(chunk_text) + + num_chunks = len(final_chunks) + update_callback(number_of_pages=num_chunks) + + for idx, chunk_content in enumerate(final_chunks, start=1): + if chunk_content.strip(): + update_callback( + current_file_chunk=idx, + status=f"Saving chunk {idx}/{num_chunks}..." + ) + args = { + "page_text_content": chunk_content, + "page_number": idx, + "file_name": original_filename, + "user_id": user_id, + "document_id": document_id + } + + if is_public_workspace: + args["public_workspace_id"] = public_workspace_id + elif is_group: + args["group_id"] = group_id + + save_chunks(**args) + total_chunks_saved += 1 + + except Exception as e: + raise Exception(f"Failed processing {original_filename}: {e}") + + return total_chunks_saved + def process_html(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None): """Processes HTML files.""" is_group = group_id is not None @@ -3093,11 +4095,11 @@ def process_tabular(document_id, user_id, temp_file_path, original_filename, fil total_chunks_saved = process_single_tabular_sheet(**args) - elif file_ext in ('.xlsx', '.xls'): + elif file_ext in ('.xlsx', '.xls', '.xlsm'): # Process Excel (potentially multiple sheets) excel_file = pandas.ExcelFile( temp_file_path, - engine='openpyxl' if file_ext == '.xlsx' else 'xlrd' + engine='openpyxl' if file_ext in ('.xlsx', '.xlsm') else 'xlrd' ) sheet_names = excel_file.sheet_names base_name, ext = os.path.splitext(original_filename) @@ -3401,6 +4403,71 @@ def process_di_document(document_id, user_id, temp_file_path, original_filename, # Don't fail the whole process, just update status update_callback(status=f"Processing complete (metadata extraction warning)") + # --- Multi-Modal Vision Analysis (for images only) --- + if is_image and enable_enhanced_citations: + enable_multimodal_vision = settings.get('enable_multimodal_vision', False) + if enable_multimodal_vision: + try: + update_callback(status="Performing AI vision analysis...") + + vision_analysis = analyze_image_with_vision_model( + temp_file_path, + user_id, + document_id, + settings + ) + + if vision_analysis: + print(f"Vision analysis completed for image: {original_filename}") + + # Update document with vision analysis results + update_fields = { + 'vision_analysis': vision_analysis, + 'vision_description': vision_analysis.get('description', ''), + 'vision_objects': vision_analysis.get('objects', []), + 'vision_extracted_text': vision_analysis.get('text', ''), + 'status': "AI vision analysis completed" + } + update_callback(**update_fields) + + # Save vision analysis as separate blob for citations + vision_json_path = temp_file_path + '_vision.json' + try: + with open(vision_json_path, 'w', encoding='utf-8') as f: + json.dump(vision_analysis, f, indent=2) + + vision_blob_filename = f"{os.path.splitext(original_filename)[0]}_vision_analysis.json" + + upload_blob_args = { + "temp_file_path": vision_json_path, + "user_id": user_id, + "document_id": document_id, + "blob_filename": vision_blob_filename, + "update_callback": update_callback + } + + if is_public_workspace: + upload_blob_args["public_workspace_id"] = public_workspace_id + elif is_group: + upload_blob_args["group_id"] = group_id + + upload_to_blob(**upload_blob_args) + print(f"Vision analysis saved to blob storage: {vision_blob_filename}") + + finally: + if os.path.exists(vision_json_path): + os.remove(vision_json_path) + else: + print(f"Vision analysis returned no results for: {original_filename}") + update_callback(status="Vision analysis completed (no results)") + + except Exception as e: + print(f"Warning: Error in vision analysis for {document_id}: {str(e)}") + import traceback + traceback.print_exc() + # Don't fail the whole process, just update status + update_callback(status=f"Processing complete (vision analysis warning)") + return total_final_chunks_processed def _get_content_type(path: str) -> str: @@ -3631,8 +4698,9 @@ def update_doc_callback(**kwargs): update_doc_callback(status=f"Processing file {original_filename}, type: {file_ext}") # --- 1. Dispatch to appropriate handler based on file type --- - di_supported_extensions = ('.pdf', '.docx', '.doc', '.pptx', '.ppt', '.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif') - tabular_extensions = ('.csv', '.xlsx', '.xls') + # Note: .doc and .docm are handled separately by process_doc() using docx2txt + di_supported_extensions = ('.pdf', '.docx', '.pptx', '.ppt', '.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif') + tabular_extensions = ('.csv', '.xlsx', '.xls', '.xlsm') is_group = group_id is not None @@ -3653,6 +4721,14 @@ def update_doc_callback(**kwargs): if file_ext == '.txt': total_chunks_saved = process_txt(**{k: v for k, v in args.items() if k != "file_ext"}) + elif file_ext == '.xml': + total_chunks_saved = process_xml(**{k: v for k, v in args.items() if k != "file_ext"}) + elif file_ext in ('.yaml', '.yml'): + total_chunks_saved = process_yaml(**{k: v for k, v in args.items() if k != "file_ext"}) + elif file_ext == '.log': + total_chunks_saved = process_log(**{k: v for k, v in args.items() if k != "file_ext"}) + elif file_ext in ('.doc', '.docm'): + total_chunks_saved = process_doc(**{k: v for k, v in args.items() if k != "file_ext"}) elif file_ext == '.html': total_chunks_saved = process_html(**{k: v for k, v in args.items() if k != "file_ext"}) elif file_ext == '.md': diff --git a/application/single_app/functions_search.py b/application/single_app/functions_search.py index cbdff52c..a3936d1c 100644 --- a/application/single_app/functions_search.py +++ b/application/single_app/functions_search.py @@ -46,18 +46,22 @@ def hybrid_search(query, user_id, document_id=None, top_n=12, doc_scope="all", a select=["id", "chunk_text", "chunk_id", "file_name", "user_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] ) - group_results = search_client_group.search( - search_text=query, - vector_queries=[vector_query], - filter=( - f"(group_id eq '{active_group_id}' or shared_group_ids/any(g: g eq '{active_group_id},approved')) and document_id eq '{document_id}'" - ), - query_type="semantic", - semantic_configuration_name="nexus-group-index-semantic-configuration", - query_caption="extractive", - query_answer="extractive", - select=["id", "chunk_text", "chunk_id", "file_name", "group_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] - ) + # Only search group index if active_group_id is provided + if active_group_id: + group_results = search_client_group.search( + search_text=query, + vector_queries=[vector_query], + filter=( + f"(group_id eq '{active_group_id}' or shared_group_ids/any(g: g eq '{active_group_id},approved')) and document_id eq '{document_id}'" + ), + query_type="semantic", + semantic_configuration_name="nexus-group-index-semantic-configuration", + query_caption="extractive", + query_answer="extractive", + select=["id", "chunk_text", "chunk_id", "file_name", "group_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] + ) + else: + group_results = [] # Get visible public workspace IDs from user settings visible_public_workspace_ids = get_user_visible_public_workspace_ids_from_settings(user_id) @@ -97,18 +101,22 @@ def hybrid_search(query, user_id, document_id=None, top_n=12, doc_scope="all", a select=["id", "chunk_text", "chunk_id", "file_name", "user_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] ) - group_results = search_client_group.search( - search_text=query, - vector_queries=[vector_query], - filter=( - f"(group_id eq '{active_group_id}' or shared_group_ids/any(g: g eq '{active_group_id},approved'))" - ), - query_type="semantic", - semantic_configuration_name="nexus-group-index-semantic-configuration", - query_caption="extractive", - query_answer="extractive", - select=["id", "chunk_text", "chunk_id", "file_name", "group_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] - ) + # Only search group index if active_group_id is provided + if active_group_id: + group_results = search_client_group.search( + search_text=query, + vector_queries=[vector_query], + filter=( + f"(group_id eq '{active_group_id}' or shared_group_ids/any(g: g eq '{active_group_id},approved'))" + ), + query_type="semantic", + semantic_configuration_name="nexus-group-index-semantic-configuration", + query_caption="extractive", + query_answer="extractive", + select=["id", "chunk_text", "chunk_id", "file_name", "group_id", "version", "chunk_sequence", "upload_date", "document_classification", "page_number", "author", "chunk_keywords", "title", "chunk_summary"] + ) + else: + group_results = [] # Get visible public workspace IDs from user settings visible_public_workspace_ids = get_user_visible_public_workspace_ids_from_settings(user_id) diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 712a8d1c..d292af7b 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -134,6 +134,10 @@ def get_settings(): 'number_of_historical_messages_to_summarize': 10, 'enable_summarize_content_history_beyond_conversation_history_limit': False, + # Multi-Modal Vision Analysis + 'enable_multimodal_vision': False, + 'multimodal_vision_model': '', + # Document Classification 'enable_document_classification': False, 'document_classification_categories': [ @@ -215,11 +219,10 @@ def get_settings(): 'video_indexer_endpoint': video_indexer_endpoint, 'video_indexer_location': '', 'video_indexer_account_id': '', - 'video_indexer_api_key': '', 'video_indexer_resource_group': '', 'video_indexer_subscription_id': '', 'video_indexer_account_name': '', - 'video_indexer_arm_api_version': '2021-11-10-preview', + 'video_indexer_arm_api_version': '2024-01-01', 'video_index_timeout': 600, # Audio file settings with Azure speech service diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 8e6aa196..c88ab9cc 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -606,8 +606,10 @@ def chat_api(): "doc_scope": document_scope, } - # Add active_group_id when document scope is 'group' or chat_type is 'group' - if (document_scope == 'group' or chat_type == 'group') and active_group_id: + # Add active_group_id when: + # 1. Document scope is 'group' or chat_type is 'group', OR + # 2. Document scope is 'all' and groups are enabled (so group search can be included) + if active_group_id and (document_scope == 'group' or document_scope == 'all' or chat_type == 'group'): search_args["active_group_id"] = active_group_id @@ -705,6 +707,151 @@ def chat_api(): # Reorder hybrid citations list in descending order based on page_number hybrid_citations_list.sort(key=lambda x: x.get('page_number', 0), reverse=True) + # --- NEW: Extract metadata (keywords/abstract) for additional citations --- + # Only if extract_metadata is enabled + if settings.get('enable_extract_meta_data', False): + from functions_documents import get_document_metadata_for_citations + + # Track which documents we've already processed to avoid duplicates + processed_doc_ids = set() + + for doc in search_results: + # Get document ID (from the chunk's document reference) + # AI Search chunks contain references to their parent document + doc_id = doc.get('id', '').split('_')[0] if doc.get('id') else None + + # Skip if we've already processed this document + if not doc_id or doc_id in processed_doc_ids: + continue + + processed_doc_ids.add(doc_id) + + # Determine workspace type from the search result fields + doc_user_id = doc.get('user_id') + doc_group_id = doc.get('group_id') + doc_public_workspace_id = doc.get('public_workspace_id') + + # Query Cosmos for this document's metadata + metadata = get_document_metadata_for_citations( + document_id=doc_id, + user_id=doc_user_id if doc_user_id else None, + group_id=doc_group_id if doc_group_id else None, + public_workspace_id=doc_public_workspace_id if doc_public_workspace_id else None + ) + + # If we have metadata with content, create additional citations + if metadata: + file_name = metadata.get('file_name', 'Unknown') + keywords = metadata.get('keywords', []) + abstract = metadata.get('abstract', '') + + # Create citation for keywords if they exist + if keywords and len(keywords) > 0: + keywords_text = ', '.join(keywords) if isinstance(keywords, list) else str(keywords) + keywords_citation_id = f"{doc_id}_keywords" + + keywords_citation = { + "file_name": file_name, + "citation_id": keywords_citation_id, + "page_number": "Metadata", # Special page identifier + "chunk_id": keywords_citation_id, + "chunk_sequence": 9999, # High number to sort to end + "score": 0.0, # No relevance score for metadata + "group_id": doc_group_id, + "version": doc.get('version', 'N/A'), + "classification": doc.get('document_classification'), + "metadata_type": "keywords", # Flag this as metadata citation + "metadata_content": keywords_text + } + hybrid_citations_list.append(keywords_citation) + combined_documents.append(keywords_citation) # Add to combined_documents too + + # Add keywords to retrieved content for the model + keywords_context = f"Document Keywords ({file_name}): {keywords_text}" + retrieved_texts.append(keywords_context) + + # Create citation for abstract if it exists + if abstract and len(abstract.strip()) > 0: + abstract_citation_id = f"{doc_id}_abstract" + + abstract_citation = { + "file_name": file_name, + "citation_id": abstract_citation_id, + "page_number": "Metadata", # Special page identifier + "chunk_id": abstract_citation_id, + "chunk_sequence": 9998, # High number to sort to end + "score": 0.0, # No relevance score for metadata + "group_id": doc_group_id, + "version": doc.get('version', 'N/A'), + "classification": doc.get('document_classification'), + "metadata_type": "abstract", # Flag this as metadata citation + "metadata_content": abstract + } + hybrid_citations_list.append(abstract_citation) + combined_documents.append(abstract_citation) # Add to combined_documents too + + # Add abstract to retrieved content for the model + abstract_context = f"Document Abstract ({file_name}): {abstract}" + retrieved_texts.append(abstract_context) + + # Create citation for vision analysis if it exists + vision_analysis = metadata.get('vision_analysis') + if vision_analysis: + vision_citation_id = f"{doc_id}_vision" + + # Format vision analysis for citation display + vision_description = vision_analysis.get('description', '') + vision_objects = vision_analysis.get('objects', []) + vision_text = vision_analysis.get('text', '') + + vision_content = f"AI Vision Analysis:\n" + if vision_description: + vision_content += f"Description: {vision_description}\n" + if vision_objects: + vision_content += f"Objects: {', '.join(vision_objects)}\n" + if vision_text: + vision_content += f"Text in Image: {vision_text}\n" + + vision_citation = { + "file_name": file_name, + "citation_id": vision_citation_id, + "page_number": "AI Vision", # Special page identifier + "chunk_id": vision_citation_id, + "chunk_sequence": 9997, # High number to sort to end (before keywords/abstract) + "score": 0.0, # No relevance score for vision analysis + "group_id": doc_group_id, + "version": doc.get('version', 'N/A'), + "classification": doc.get('document_classification'), + "metadata_type": "vision", # Flag this as vision citation + "metadata_content": vision_content + } + hybrid_citations_list.append(vision_citation) + combined_documents.append(vision_citation) # Add to combined_documents too + + # Add vision analysis to retrieved content for the model + vision_context = f"AI Vision Analysis ({file_name}): {vision_content}" + retrieved_texts.append(vision_context) + + # Update the system prompt with the enhanced content including metadata + if retrieved_texts: + retrieved_content = "\n\n".join(retrieved_texts) + system_prompt_search = f"""You are an AI assistant. Use the following retrieved document excerpts to answer the user's question. Cite sources using the format (Source: filename, Page: page number). + + Retrieved Excerpts: + {retrieved_content} + + Based *only* on the information provided above, answer the user's query. If the answer isn't in the excerpts, say so. + + Example + User: What is the policy on double dipping? + Assistant: The policy prohibits entities from using federal funds received through one program to apply for additional funds through another program, commonly known as 'double dipping' (Source: PolicyDocument.pdf, Page: 12) + """ + # Update the system message with enhanced content and updated documents array + if system_messages_for_augmentation: + system_messages_for_augmentation[-1]['content'] = system_prompt_search + system_messages_for_augmentation[-1]['documents'] = combined_documents + # --- END NEW METADATA CITATIONS --- + # Update conversation classifications if new ones were found if list(classifications_found) != conversation_item.get('classification', []): conversation_item['classification'] = list(classifications_found) @@ -1139,13 +1286,66 @@ def chat_api(): 'role': 'system', # Represent file as system info 'content': f"[User uploaded a file named '{filename}'. Content preview:\n{display_content}]\nUse this file context if relevant." }) - # elif role == 'image': # If you want to represent image generation prompts/results - # prompt = message.get('prompt', 'User generated an image.') - # img_url = message.get('content', '') # URL is in content - # conversation_history_for_api.append({ - # 'role': 'system', - # 'content': f"[Assistant generated an image based on the prompt: '{prompt}'. Image URL: {img_url}]" - # }) + elif role == 'image': # Handle image uploads with extracted text and vision analysis + filename = message.get('filename', 'uploaded_image') + is_user_upload = message.get('metadata', {}).get('is_user_upload', False) + + if is_user_upload: + # This is a user-uploaded image with extracted text and vision analysis + # IMPORTANT: Do NOT include message.get('content') as it contains base64 image data + # which would consume excessive tokens. Only use extracted_text and vision_analysis. + extracted_text = message.get('extracted_text', '') + vision_analysis = message.get('vision_analysis', {}) + + # Build comprehensive context from OCR and vision analysis (NO BASE64!) + image_context_parts = [f"[User uploaded an image named '{filename}'.]"] + + if extracted_text: + # Include OCR text from Document Intelligence + extracted_preview = extracted_text[:max_file_content_length_in_history] + if len(extracted_text) > max_file_content_length_in_history: + extracted_preview += "..." + image_context_parts.append(f"\n\nExtracted Text (OCR):\n{extracted_preview}") + + if vision_analysis: + # Include AI vision analysis + image_context_parts.append("\n\nAI Vision Analysis:") + + if vision_analysis.get('description'): + image_context_parts.append(f"\nDescription: {vision_analysis['description']}") + + if vision_analysis.get('objects'): + objects_str = ', '.join(vision_analysis['objects']) + image_context_parts.append(f"\nObjects detected: {objects_str}") + + if vision_analysis.get('text'): + image_context_parts.append(f"\nText visible in image: {vision_analysis['text']}") + + if vision_analysis.get('contextual_analysis'): + image_context_parts.append(f"\nContextual analysis: {vision_analysis['contextual_analysis']}") + + image_context_content = ''.join(image_context_parts) + "\n\nUse this image information to answer questions about the uploaded image." + + # Verify we're not accidentally including base64 data + if 'data:image/' in image_context_content or ';base64,' in image_context_content: + print(f"WARNING: Base64 image data detected in chat history for {filename}! Removing to save tokens.") + # This should never happen, but safety check just in case + image_context_content = f"[User uploaded an image named '{filename}' - image data excluded from chat history to conserve tokens]" + + debug_print(f"[IMAGE_CONTEXT] Adding user-uploaded image to history: {filename}, context length: {len(image_context_content)} chars") + conversation_history_for_api.append({ + 'role': 'system', + 'content': image_context_content + }) + else: + # This is a system-generated image (DALL-E, etc.) + # Don't include the image data URL in history either + prompt = message.get('prompt', 'User requested image generation.') + debug_print(f"[IMAGE_CONTEXT] Adding system-generated image to history: {prompt[:100]}...") + conversation_history_for_api.append({ + 'role': 'system', + 'content': f"[Assistant generated an image based on the prompt: '{prompt}']" + }) # Ignored roles: 'safety', 'blocked', 'system' (if they are only for augmentation/summary) diff --git a/application/single_app/route_backend_documents.py b/application/single_app/route_backend_documents.py index 447bc032..ec2594b1 100644 --- a/application/single_app/route_backend_documents.py +++ b/application/single_app/route_backend_documents.py @@ -20,11 +20,15 @@ def get_file_content(): user_id = get_current_user_id() conversation_id = data.get('conversation_id') file_id = data.get('file_id') + + debug_print(f"[GET_FILE_CONTENT] Starting - user_id={user_id}, conversation_id={conversation_id}, file_id={file_id}") if not user_id: + debug_print(f"[GET_FILE_CONTENT] ERROR: User not authenticated") return jsonify({'error': 'User not authenticated'}), 401 if not conversation_id or not file_id: + debug_print(f"[GET_FILE_CONTENT] ERROR: Missing conversation_id or file_id") return jsonify({'error': 'Missing conversation_id or id'}), 400 try: @@ -57,36 +61,52 @@ def get_file_content(): add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="File not found in conversation") return jsonify({'error': 'File not found in conversation'}), 404 + debug_print(f"[GET_FILE_CONTENT] Found {len(items)} items for file_id={file_id}") + debug_print(f"[GET_FILE_CONTENT] First item structure: {json.dumps(items[0], default=str, indent=2)}") add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="File found, processing content: " + str(items)) items_sorted = sorted(items, key=lambda x: x.get('chunk_index', 0)) filename = items_sorted[0].get('filename', 'Untitled') is_table = items_sorted[0].get('is_table', False) + debug_print(f"[GET_FILE_CONTENT] Filename: {filename}, is_table: {is_table}") add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="Combining file content from chunks, filename: " + filename + ", is_table: " + str(is_table)) combined_parts = [] - for it in items_sorted: + for idx, it in enumerate(items_sorted): fc = it.get('file_content', '') + debug_print(f"[GET_FILE_CONTENT] Chunk {idx}: file_content type={type(fc).__name__}, len={len(fc) if hasattr(fc, '__len__') else 'N/A'}") if isinstance(fc, list): + debug_print(f"[GET_FILE_CONTENT] Processing list of {len(fc)} items") # If file_content is a list of dicts, join their 'content' fields text_chunks = [] - for chunk in fc: - text_chunks.append(chunk.get('content', '')) + for chunk_idx, chunk in enumerate(fc): + debug_print(f"[GET_FILE_CONTENT] List item {chunk_idx} type: {type(chunk).__name__}") + if isinstance(chunk, dict): + text_chunks.append(chunk.get('content', '')) + elif isinstance(chunk, str): + text_chunks.append(chunk) + else: + debug_print(f"[GET_FILE_CONTENT] Unexpected chunk type in list: {type(chunk).__name__}") combined_parts.append("\n".join(text_chunks)) elif isinstance(fc, str): + debug_print(f"[GET_FILE_CONTENT] Processing string content") # If it's already a string, just append combined_parts.append(fc) else: # If it's neither a list nor a string, handle as needed (e.g., skip or log) + debug_print(f"[GET_FILE_CONTENT] WARNING: Unexpected file_content type: {type(fc).__name__}, value: {fc}") pass combined_content = "\n".join(combined_parts) + debug_print(f"[GET_FILE_CONTENT] Combined content length: {len(combined_content)}") if not combined_content: add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="Combined file content is empty") + debug_print(f"[GET_FILE_CONTENT] ERROR: Combined content is empty") return jsonify({'error': 'File content not found'}), 404 + debug_print(f"[GET_FILE_CONTENT] Successfully returning file content") return jsonify({ 'file_content': combined_content, 'filename': filename, @@ -94,6 +114,8 @@ def get_file_content(): }), 200 except Exception as e: + debug_print(f"[GET_FILE_CONTENT] EXCEPTION: {str(e)}") + debug_print(f"[GET_FILE_CONTENT] Traceback: {traceback.format_exc()}") add_file_task_to_file_processing_log(document_id=file_id, user_id=user_id, content="Error retrieving file content: " + str(e)) return jsonify({'error': f'Error retrieving file content: {str(e)}'}), 500 diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 449ba546..7ef92bf3 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -276,6 +276,9 @@ def test_connection(): elif test_type == 'azure_doc_intelligence': return _test_azure_doc_intelligence_connection(data) + elif test_type == 'multimodal_vision': + return _test_multimodal_vision_connection(data) + elif test_type == 'chunking_api': # If you have a chunking API test, implement it here. return jsonify({'message': 'Chunking API connection successful'}), 200 @@ -285,6 +288,86 @@ def test_connection(): except Exception as e: return jsonify({'error': str(e)}), 500 + +def _test_multimodal_vision_connection(payload): + """Test multi-modal vision analysis with a sample image.""" + enable_apim = payload.get('enable_apim', False) + vision_model = payload.get('vision_model') + + if not vision_model: + return jsonify({'error': 'No vision model specified'}), 400 + + # Create a simple test image (1x1 red pixel PNG) + test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" + + try: + if enable_apim: + apim_data = payload.get('apim', {}) + endpoint = apim_data.get('endpoint') + api_version = apim_data.get('api_version') + subscription_key = apim_data.get('subscription_key') + + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=subscription_key + ) + else: + direct_data = payload.get('direct', {}) + endpoint = direct_data.get('endpoint') + api_version = direct_data.get('api_version') + auth_type = direct_data.get('auth_type', 'key') + + if auth_type == 'managed_identity': + token_provider = get_bearer_token_provider( + DefaultAzureCredential(), + cognitive_services_scope + ) + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + azure_ad_token_provider=token_provider + ) + else: + api_key = direct_data.get('key') + gpt_client = AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=api_key + ) + + # Test vision analysis with simple prompt + response = gpt_client.chat.completions.create( + model=vision_model, + messages=[ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What color is this image? Just say the color." + }, + { + "type": "image_url", + "image_url": { + "url": f"data:image/png;base64,{test_image_base64}" + } + } + ] + } + ], + max_tokens=50 + ) + + result = response.choices[0].message.content + + return jsonify({ + 'message': 'Multi-modal vision connection successful', + 'details': f'Model responded: {result}' + }), 200 + + except Exception as e: + return jsonify({'error': f'Vision test failed: {str(e)}'}), 500 def get_index_client() -> SearchIndexClient: """ diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 178cb434..937933ef 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -631,11 +631,10 @@ def is_valid_url(url): 'video_indexer_endpoint': form_data.get('video_indexer_endpoint', video_indexer_endpoint).strip(), 'video_indexer_location': form_data.get('video_indexer_location', '').strip(), 'video_indexer_account_id': form_data.get('video_indexer_account_id', '').strip(), - 'video_indexer_api_key': form_data.get('video_indexer_api_key', '').strip(), 'video_indexer_resource_group': form_data.get('video_indexer_resource_group', '').strip(), 'video_indexer_subscription_id': form_data.get('video_indexer_subscription_id', '').strip(), 'video_indexer_account_name': form_data.get('video_indexer_account_name', '').strip(), - 'video_indexer_arm_api_version': form_data.get('video_indexer_arm_api_version', '2021-11-10-preview').strip(), + 'video_indexer_arm_api_version': form_data.get('video_indexer_arm_api_version', '2024-01-01').strip(), 'video_index_timeout': int(form_data.get('video_index_timeout', 600)), # Audio file settings with Azure speech service @@ -646,6 +645,10 @@ def is_valid_url(url): 'metadata_extraction_model': form_data.get('metadata_extraction_model', '').strip(), + # Multi-modal vision settings + 'enable_multimodal_vision': form_data.get('enable_multimodal_vision') == 'on', + 'multimodal_vision_model': form_data.get('multimodal_vision_model', '').strip(), + # --- Banner fields --- 'classification_banner_enabled': classification_banner_enabled, 'classification_banner_text': classification_banner_text, diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index ed12ec6c..cc5e8603 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -117,10 +117,78 @@ def upload_file(): extracted_content = '' is_table = False + vision_analysis = None + image_base64_url = None # For storing base64-encoded images try: + # Check if this is an image file + is_image_file = file_ext in ['.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif'] + if file_ext in ['.pdf', '.docx', '.pptx', '.html', '.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.heif']: - extracted_content = extract_content_with_azure_di(temp_file_path) + extracted_content_raw = extract_content_with_azure_di(temp_file_path) + + # Convert pages_data list to string + if isinstance(extracted_content_raw, list): + extracted_content = "\n\n".join([ + f"[Page {page.get('page_number', 'N/A')}]\n{page.get('content', '')}" + for page in extracted_content_raw + ]) + else: + extracted_content = str(extracted_content_raw) + + # NEW: For images, convert to base64 for inline display + if is_image_file: + try: + with open(temp_file_path, 'rb') as img_file: + image_bytes = img_file.read() + base64_image = base64.b64encode(image_bytes).decode('utf-8') + + # Detect mime type + mime_type = mimetypes.guess_type(temp_file_path)[0] or 'image/png' + + # Create data URL + image_base64_url = f"data:{mime_type};base64,{base64_image}" + print(f"Converted image to base64: {filename}, size: {len(image_base64_url)} bytes") + except Exception as b64_error: + print(f"Warning: Failed to convert image to base64: {b64_error}") + + # Perform vision analysis for images if enabled + if is_image_file and settings.get('enable_multimodal_vision', False): + try: + from functions_documents import analyze_image_with_vision_model + + vision_analysis = analyze_image_with_vision_model( + temp_file_path, + user_id, + f"chat_upload_{int(time.time())}", + settings + ) + + if vision_analysis: + # Combine DI OCR with vision analysis + vision_description = vision_analysis.get('description', '') + vision_objects = vision_analysis.get('objects', []) + vision_text = vision_analysis.get('text', '') + + extracted_content += f"\n\n=== AI Vision Analysis ===\n" + extracted_content += f"Description: {vision_description}\n" + if vision_objects: + extracted_content += f"Objects detected: {', '.join(vision_objects)}\n" + if vision_text: + extracted_content += f"Text visible in image: {vision_text}\n" + + print(f"Vision analysis added to chat upload: {filename}") + except Exception as vision_error: + print(f"Warning: Vision analysis failed for chat upload: {vision_error}") + # Continue without vision analysis + + elif file_ext in ['.doc', '.docm']: + # Use docx2txt for .doc and .docm files + try: + import docx2txt + extracted_content = docx2txt.process(temp_file_path) + except ImportError: + return jsonify({'error': 'docx2txt library required for .doc/.docm files'}), 500 elif file_ext == '.txt': extracted_content = extract_text_file(temp_file_path) elif file_ext == '.md': @@ -129,7 +197,10 @@ def upload_file(): with open(temp_file_path, 'r', encoding='utf-8') as f: parsed_json = json.load(f) extracted_content = json.dumps(parsed_json, indent=2) - elif file_ext in ['.csv', '.xls', '.xlsx']: + elif file_ext in ['.xml', '.yaml', '.yml', '.log']: + # Handle XML, YAML, and LOG files as text for inline chat + extracted_content = extract_text_file(temp_file_path) + elif file_ext in ['.csv', '.xls', '.xlsx', '.xlsm']: extracted_content = extract_table_file(temp_file_path, file_ext) is_table = True else: @@ -142,18 +213,120 @@ def upload_file(): try: file_message_id = f"{conversation_id}_file_{int(time.time())}_{random.randint(1000,9999)}" - file_message = { - 'id': file_message_id, - 'conversation_id': conversation_id, - 'role': 'file', - 'filename': filename, - 'file_content': extracted_content, - 'is_table': is_table, - 'timestamp': datetime.utcnow().isoformat(), - 'model_deployment_name': None - } + + # For images with base64 data, store as 'image' role (like system-generated images) + if image_base64_url: + # Check if image data is too large for a single Cosmos document (2MB limit) + # Use 1.5MB as safe limit for base64 content + max_content_size = 1500000 # 1.5MB in bytes + + if len(image_base64_url) > max_content_size: + print(f"Large image detected ({len(image_base64_url)} bytes), splitting across multiple documents") + + # Extract base64 part for splitting + data_url_prefix = image_base64_url.split(',')[0] + ',' + base64_content = image_base64_url.split(',')[1] + + # Calculate chunks + chunk_size = max_content_size - len(data_url_prefix) - 200 # Room for JSON overhead + chunks = [base64_content[i:i+chunk_size] for i in range(0, len(base64_content), chunk_size)] + total_chunks = len(chunks) + + print(f"Splitting into {total_chunks} chunks of max {chunk_size} bytes each") + + # Create main image document with first chunk + main_image_doc = { + 'id': file_message_id, + 'conversation_id': conversation_id, + 'role': 'image', + 'content': f"{data_url_prefix}{chunks[0]}", + 'filename': filename, + 'prompt': f"User uploaded: {filename}", + 'created_at': datetime.utcnow().isoformat(), + 'timestamp': datetime.utcnow().isoformat(), + 'model_deployment_name': None, + 'metadata': { + 'is_chunked': True, + 'total_chunks': total_chunks, + 'chunk_index': 0, + 'original_size': len(image_base64_url), + 'is_user_upload': True + } + } + + # Add vision analysis and extracted text if available + if vision_analysis: + main_image_doc['vision_analysis'] = vision_analysis + if extracted_content: + main_image_doc['extracted_text'] = extracted_content + + cosmos_messages_container.upsert_item(main_image_doc) + + # Create chunk documents + for i in range(1, total_chunks): + chunk_doc = { + 'id': f"{file_message_id}_chunk_{i}", + 'conversation_id': conversation_id, + 'role': 'image_chunk', + 'content': chunks[i], + 'parent_message_id': file_message_id, + 'created_at': datetime.utcnow().isoformat(), + 'timestamp': datetime.utcnow().isoformat(), + 'metadata': { + 'is_chunk': True, + 'chunk_index': i, + 'total_chunks': total_chunks, + 'parent_message_id': file_message_id + } + } + cosmos_messages_container.upsert_item(chunk_doc) + + print(f"Created {total_chunks} chunked image documents for {filename}") + else: + # Small enough to store in single document + image_message = { + 'id': file_message_id, + 'conversation_id': conversation_id, + 'role': 'image', + 'content': image_base64_url, + 'filename': filename, + 'prompt': f"User uploaded: {filename}", + 'created_at': datetime.utcnow().isoformat(), + 'timestamp': datetime.utcnow().isoformat(), + 'model_deployment_name': None, + 'metadata': { + 'is_chunked': False, + 'original_size': len(image_base64_url), + 'is_user_upload': True + } + } + + # Add vision analysis and extracted text if available + if vision_analysis: + image_message['vision_analysis'] = vision_analysis + if extracted_content: + image_message['extracted_text'] = extracted_content + + cosmos_messages_container.upsert_item(image_message) + print(f"Created single image document for {filename}") + else: + # Non-image file or failed to convert to base64, store as 'file' role + file_message = { + 'id': file_message_id, + 'conversation_id': conversation_id, + 'role': 'file', + 'filename': filename, + 'file_content': extracted_content, + 'is_table': is_table, + 'timestamp': datetime.utcnow().isoformat(), + 'model_deployment_name': None + } + + # Add vision analysis if available + if vision_analysis: + file_message['vision_analysis'] = vision_analysis - cosmos_messages_container.upsert_item(file_message) + cosmos_messages_container.upsert_item(file_message) conversation_item['last_updated'] = datetime.utcnow().isoformat() cosmos_conversations_container.upsert_item(conversation_item) @@ -426,10 +599,10 @@ def view_document(): # Define supported types for direct viewing/handling is_pdf = file_ext == '.pdf' - is_word = file_ext in ('.docx', '.doc') + is_word = file_ext in ('.docx', '.doc', '.docm') is_ppt = file_ext in ('.pptx', '.ppt') is_image = file_ext in ('.jpg', '.jpeg', '.png', '.bmp', '.tiff', '.tif', '.gif', '.webp') # Added more image types - is_text = file_ext in ('.txt', '.md', '.csv', '.json', '.log', '.xml', '.html', '.htm') # Common text-based types + is_text = file_ext in ('.txt', '.md', '.csv', '.json', '.log', '.xml', '.yaml', '.yml', '.html', '.htm') # Common text-based types try: # Download the file to the specified location diff --git a/application/single_app/route_frontend_group_workspaces.py b/application/single_app/route_frontend_group_workspaces.py index 523799e8..97fbbc6b 100644 --- a/application/single_app/route_frontend_group_workspaces.py +++ b/application/single_app/route_frontend_group_workspaces.py @@ -46,6 +46,18 @@ def group_workspaces(): ) ) legacy_count = legacy_docs_from_cosmos[0] if legacy_docs_from_cosmos else 0 + + # Build allowed extensions string + allowed_extensions = [ + "txt", "pdf", "doc", "docm", "docx", "xlsx", "xls", "xlsm","csv", "pptx", "html", + "jpg", "jpeg", "png", "bmp", "tiff", "tif", "heif", "md", "json", + "xml", "yaml", "yml", "log" + ] + if enable_video_file_support in [True, 'True', 'true']: + allowed_extensions += ["mp4", "mov", "avi", "wmv", "mkv", "webm"] + if enable_audio_file_support in [True, 'True', 'true']: + allowed_extensions += ["mp3", "wav", "ogg", "aac", "flac", "m4a"] + allowed_extensions_str = "Allowed: " + ", ".join(allowed_extensions) return render_template( 'group_workspaces.html', @@ -55,7 +67,8 @@ def group_workspaces(): enable_video_file_support=enable_video_file_support, enable_audio_file_support=enable_audio_file_support, enable_file_sharing=enable_file_sharing, - legacy_docs_count=legacy_count + legacy_docs_count=legacy_count, + allowed_extensions=allowed_extensions_str ) @app.route('/set_active_group', methods=['POST']) diff --git a/application/single_app/route_frontend_public_workspaces.py b/application/single_app/route_frontend_public_workspaces.py index df88ddd7..0b9b208e 100644 --- a/application/single_app/route_frontend_public_workspaces.py +++ b/application/single_app/route_frontend_public_workspaces.py @@ -71,8 +71,9 @@ def public_workspaces(): # Build allowed extensions string as in workspace.html allowed_extensions = [ - "txt", "pdf", "docx", "xlsx", "xls", "csv", "pptx", "html", - "jpg", "jpeg", "png", "bmp", "tiff", "tif", "heif", "md", "json" + "txt", "pdf", "doc", "docm", "docx", "xlsx", "xls", "xlsm","csv", "pptx", "html", + "jpg", "jpeg", "png", "bmp", "tiff", "tif", "heif", "md", "json", + "xml", "yaml", "yml", "log" ] if enable_video_file_support in [True, 'True', 'true']: allowed_extensions += ["mp4", "mov", "avi", "wmv", "mkv", "webm"] diff --git a/application/single_app/route_frontend_workspace.py b/application/single_app/route_frontend_workspace.py index 0bdf289a..1407ac15 100644 --- a/application/single_app/route_frontend_workspace.py +++ b/application/single_app/route_frontend_workspace.py @@ -44,6 +44,18 @@ def workspace(): ) ) legacy_count = legacy_docs_from_cosmos[0] if legacy_docs_from_cosmos else 0 + + # Build allowed extensions string + allowed_extensions = [ + "txt", "pdf", "doc", "docm", "docx", "xlsx", "xls", "xlsm","csv", "pptx", "html", + "jpg", "jpeg", "png", "bmp", "tiff", "tif", "heif", "md", "json", + "xml", "yaml", "yml", "log" + ] + if enable_video_file_support in [True, 'True', 'true']: + allowed_extensions += ["mp4", "mov", "avi", "wmv", "mkv", "webm"] + if enable_audio_file_support in [True, 'True', 'true']: + allowed_extensions += ["mp3", "wav", "ogg", "aac", "flac", "m4a"] + allowed_extensions_str = "Allowed: " + ", ".join(allowed_extensions) return render_template( 'workspace.html', @@ -53,7 +65,8 @@ def workspace(): enable_video_file_support=enable_video_file_support, enable_audio_file_support=enable_audio_file_support, enable_file_sharing=enable_file_sharing, - legacy_docs_count=legacy_count + legacy_docs_count=legacy_count, + allowed_extensions=allowed_extensions_str ) \ No newline at end of file diff --git a/application/single_app/static/js/admin/admin_settings.js b/application/single_app/static/js/admin/admin_settings.js index d3b72980..5361583a 100644 --- a/application/single_app/static/js/admin/admin_settings.js +++ b/application/single_app/static/js/admin/admin_settings.js @@ -2251,6 +2251,66 @@ function setupTestButtons() { } }); } + + const testVisionBtn = document.getElementById('test_multimodal_vision_button'); + if (testVisionBtn) { + testVisionBtn.addEventListener('click', async () => { + const resultDiv = document.getElementById('test_multimodal_vision_result'); + resultDiv.innerHTML = 'Testing Vision Analysis...'; + + const visionModel = document.getElementById('multimodal_vision_model').value; + + if (!visionModel) { + resultDiv.innerHTML = 'Please select a vision model first'; + return; + } + + const enableApim = document.getElementById('enable_gpt_apim').checked; + + const payload = { + test_type: 'multimodal_vision', + enable_apim: enableApim, + vision_model: visionModel + }; + + if (enableApim) { + payload.apim = { + endpoint: document.getElementById('azure_apim_gpt_endpoint').value, + subscription_key: document.getElementById('azure_apim_gpt_subscription_key').value, + api_version: document.getElementById('azure_apim_gpt_api_version').value, + deployment: visionModel + }; + } else { + payload.direct = { + endpoint: document.getElementById('azure_openai_gpt_endpoint').value, + auth_type: document.getElementById('azure_openai_gpt_authentication_type').value, + key: document.getElementById('azure_openai_gpt_key').value, + api_version: document.getElementById('azure_openai_gpt_api_version').value, + deployment: visionModel + }; + } + + try { + const resp = await fetch('/api/admin/settings/test_connection', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }); + const data = await resp.json(); + if (resp.ok) { + resultDiv.innerHTML = `
+ Success!
+ ${data.message}
+ ${data.details || ''} +
`; + } else { + resultDiv.innerHTML = `${data.error || 'Error testing Vision Analysis'}`; + } + } catch (err) { + resultDiv.innerHTML = `Error: ${err.message}`; + } + }); + } } function toggleEnhancedCitation(isEnabled) { @@ -2352,10 +2412,91 @@ if (extractToggle) { }); } +// Multi-Modal Vision UI +const visionToggle = document.getElementById('enable_multimodal_vision'); +const visionModelDiv = document.getElementById('multimodal_vision_model_settings'); +const visionSelect = document.getElementById('multimodal_vision_model'); + +function populateVisionModels() { + if (!visionSelect) return; + + // remember previously chosen value + const prev = visionSelect.getAttribute('data-prev') || ''; + + // clear out old options (except the placeholder) + visionSelect.innerHTML = ''; + + if (document.getElementById('enable_gpt_apim').checked) { + // use comma-separated APIM deployments + const text = document.getElementById('azure_apim_gpt_deployment').value || ''; + text.split(',') + .map(s => s.trim()) + .filter(s => s) + .forEach(d => { + const opt = new Option(d, d); + visionSelect.add(opt); + }); + } else { + // use direct GPT selected deployments - filter for vision-capable models + (window.gptSelected || []).forEach(m => { + // Only include models with vision capabilities + // Vision-enabled models per Azure OpenAI docs: + // - o-series reasoning models (o1, o3, etc.) + // - GPT-5 series + // - GPT-4.1 series + // - GPT-4.5 + // - GPT-4o series (gpt-4o, gpt-4o-mini) + // - GPT-4 vision models (gpt-4-vision, gpt-4-turbo-vision) + const modelNameLower = (m.modelName || '').toLowerCase(); + const isVisionCapable = + modelNameLower.includes('vision') || // gpt-4-vision, gpt-4-turbo-vision + modelNameLower.includes('gpt-4o') || // gpt-4o, gpt-4o-mini + modelNameLower.includes('gpt-4.1') || // gpt-4.1 series + modelNameLower.includes('gpt-4.5') || // gpt-4.5 + modelNameLower.includes('gpt-5') || // gpt-5 series + modelNameLower.match(/^o\d+/) || // o1, o3, etc. (o-series) + modelNameLower.includes('o1-') || // o1-preview, o1-mini + modelNameLower.includes('o3-'); // o3-mini, etc. + + if (isVisionCapable) { + const label = `${m.deploymentName} (${m.modelName})`; + const opt = new Option(label, m.deploymentName); + visionSelect.add(opt); + } + }); + } + + // restore previous + if (prev) { + visionSelect.value = prev; + } +} + +if (visionToggle && visionModelDiv) { + // show/hide the model dropdown + visionModelDiv.style.display = visionToggle.checked ? 'block' : 'none'; + visionToggle.addEventListener('change', () => { + visionModelDiv.style.display = visionToggle.checked ? 'block' : 'none'; + markFormAsModified(); + }); +} + +// Listen for vision model selection changes +if (visionSelect) { + visionSelect.addEventListener('change', () => { + // Update data-prev to remember the selection + visionSelect.setAttribute('data-prev', visionSelect.value); + markFormAsModified(); + }); +} + // when APIM‐toggle flips, repopulate const apimToggle = document.getElementById('enable_gpt_apim'); if (apimToggle) { - apimToggle.addEventListener('change', populateExtractionModels); + apimToggle.addEventListener('change', () => { + populateExtractionModels(); + populateVisionModels(); + }); } // on load, stash previous & populate @@ -2364,6 +2505,10 @@ document.addEventListener('DOMContentLoaded', () => { extractSelect.setAttribute('data-prev', extractSelect.value); populateExtractionModels(); } + if (visionSelect) { + visionSelect.setAttribute('data-prev', visionSelect.value); + populateVisionModels(); + } }); diff --git a/application/single_app/static/js/chat/chat-citations.js b/application/single_app/static/js/chat/chat-citations.js index 9f24d000..a69619c9 100644 --- a/application/single_app/static/js/chat/chat-citations.js +++ b/application/single_app/static/js/chat/chat-citations.js @@ -226,6 +226,64 @@ export function showImagePopup(imageSrc) { modal.show(); } +export function showMetadataModal(metadataType, metadataContent, fileName) { + // Create or reuse the metadata modal + let modalContainer = document.getElementById("metadata-modal"); + if (!modalContainer) { + modalContainer = document.createElement("div"); + modalContainer.id = "metadata-modal"; + modalContainer.classList.add("modal", "fade"); + modalContainer.tabIndex = -1; + modalContainer.setAttribute("aria-hidden", "true"); + + modalContainer.innerHTML = ` +
+
+
+ + +
+
+
+ File: +
+
+ Type: +
+
+ Content: +
+
+
+
+
+ `; + document.body.appendChild(modalContainer); + } + + // Update modal content + const modalTitle = modalContainer.querySelector("#metadata-modal-title"); + const fileNameEl = modalContainer.querySelector("#metadata-file-name"); + const metadataTypeEl = modalContainer.querySelector("#metadata-type"); + const metadataContentEl = modalContainer.querySelector("#metadata-content"); + + if (modalTitle) { + modalTitle.textContent = `Document Metadata - ${metadataType.charAt(0).toUpperCase() + metadataType.slice(1)}`; + } + if (fileNameEl) { + fileNameEl.textContent = fileName; + } + if (metadataTypeEl) { + metadataTypeEl.textContent = metadataType.charAt(0).toUpperCase() + metadataType.slice(1); + } + if (metadataContentEl) { + metadataContentEl.textContent = metadataContent; + } + + const modal = new bootstrap.Modal(modalContainer); + modal.show(); +} + export function showAgentCitationModal(toolName, toolArgs, toolResult) { // Create or reuse the agent citation modal let modalContainer = document.getElementById("agent-citation-modal"); @@ -460,6 +518,18 @@ if (chatboxEl) { return; } + // Check if this is a metadata citation + const isMetadata = target.getAttribute("data-is-metadata") === "true"; + if (isMetadata) { + // Show metadata content directly in a modal + const metadataType = target.getAttribute("data-metadata-type"); + const metadataContent = target.getAttribute("data-metadata-content"); + const fileName = citationId.split('_')[0]; // Extract filename from citation ID + + showMetadataModal(metadataType, metadataContent, fileName); + return; + } + const { docId, pageNumber } = parseDocIdAndPage(citationId); // Safety check: Ensure docId and pageNumber were parsed correctly diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index b5419eee..79a439d7 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -359,12 +359,21 @@ function createCitationsHtml( const displayText = `${escapeHtml(cite.file_name)}, Page ${ cite.page_number || "N/A" }`; + + // Check if this is a metadata citation + const isMetadata = cite.metadata_type ? true : false; + const metadataType = cite.metadata_type || ''; + const metadataContent = cite.metadata_content || ''; + citationsHtml += ` - ${displayText} + ${displayText} `; }); } @@ -475,7 +484,8 @@ export function loadMessages(conversationId) { } else if (msg.role === "image") { // Validate image URL before calling appendMessage if (msg.content && msg.content !== 'null' && msg.content.trim() !== '') { - appendMessage("image", msg.content, msg.model_deployment_name, msg.id, false, [], [], [], msg.agent_display_name, msg.agent_name); + // Pass the full message object for images that may have metadata (uploaded images) + appendMessage("image", msg.content, msg.model_deployment_name, msg.id, false, [], [], [], msg.agent_display_name, msg.agent_name, msg); } else { console.error(`[loadMessages] Invalid image URL for message ${msg.id}: "${msg.content}"`); // Show error message instead of broken image @@ -502,7 +512,8 @@ export function appendMessage( webCitations = [], agentCitations = [], agentDisplayName = null, - agentName = null + agentName = null, + fullMessageObject = null ) { if (!chatbox || sender === "System") return; @@ -756,8 +767,15 @@ export function appendMessage( // Make sure this matches the case used in loadMessages/actuallySendMessage messageClass = "image-message"; // Use a distinct class if needed, or reuse ai-message + // Check if this is a user-uploaded image with metadata + const isUserUpload = fullMessageObject?.metadata?.is_user_upload || false; + const hasExtractedText = fullMessageObject?.extracted_text || false; + const hasVisionAnalysis = fullMessageObject?.vision_analysis || false; + // Use agent display name if available, otherwise show AI with model - if (agentDisplayName) { + if (isUserUpload) { + senderLabel = "Uploaded Image"; + } else if (agentDisplayName) { senderLabel = agentDisplayName; } else if (modelName) { senderLabel = `AI (${modelName})`; @@ -765,14 +783,28 @@ export function appendMessage( senderLabel = "Image"; } - avatarImg = "/static/images/ai-avatar.png"; // Or a specific image icon - avatarAltText = "Generated Image"; + avatarImg = isUserUpload ? "/static/images/user-avatar.png" : "/static/images/ai-avatar.png"; + avatarAltText = isUserUpload ? "Uploaded Image" : "Generated Image"; // Validate image URL before creating img tag if (messageContent && messageContent !== 'null' && messageContent.trim() !== '') { - messageContentHtml = `Generated Image`; + messageContentHtml = `${isUserUpload ? 'Uploaded' : 'Generated'} Image`; + + // Add info button for uploaded images with extracted text or vision analysis + if (isUserUpload && (hasExtractedText || hasVisionAnalysis)) { + const infoContainerId = `image-info-${messageId || Date.now()}`; + messageContentHtml += ` +
+ +
+ `; + } } else { - messageContentHtml = `
Failed to generate image - invalid response from image service
`; + messageContentHtml = `
Failed to ${isUserUpload ? 'load' : 'generate'} image - invalid response from image service
`; } } else if (sender === "safety") { messageClass = "safety-message"; @@ -858,6 +890,17 @@ export function appendMessage( if (sender === "You") { attachUserMessageEventListeners(messageDiv, messageId, messageContent); } + + // Add event listener for image info button (uploaded images) + if (sender === "image" && fullMessageObject?.metadata?.is_user_upload) { + const imageInfoBtn = messageDiv.querySelector('.image-info-btn'); + if (imageInfoBtn) { + imageInfoBtn.addEventListener('click', () => { + toggleImageInfo(messageDiv, messageId, fullMessageObject); + }); + } + } + scrollChatToBottom(); } // End of the large 'else' block for non-AI messages } @@ -1842,3 +1885,118 @@ if (modelSelect) { saveUserSetting({ 'preferredModelDeployment': selectedModel }); }); } + +/** + * Toggle the image info drawer for uploaded images + * Shows extracted text (OCR) and vision analysis + */ +function toggleImageInfo(messageDiv, messageId, fullMessageObject) { + const toggleBtn = messageDiv.querySelector('.image-info-btn'); + const targetId = toggleBtn.getAttribute('aria-controls'); + const infoContainer = messageDiv.querySelector(`#${targetId}`); + + if (!infoContainer) { + console.error(`Image info container not found for targetId: ${targetId}`); + return; + } + + const isExpanded = infoContainer.style.display !== "none"; + + // Store current scroll position to maintain user's view + const currentScrollTop = document.getElementById('chat-messages-container')?.scrollTop || window.pageYOffset; + + if (isExpanded) { + // Hide the info + infoContainer.style.display = "none"; + toggleBtn.setAttribute("aria-expanded", false); + toggleBtn.title = "View extracted text & analysis"; + toggleBtn.innerHTML = ' View Text'; + } else { + // Show the info + infoContainer.style.display = "block"; + toggleBtn.setAttribute("aria-expanded", true); + toggleBtn.title = "Hide extracted text & analysis"; + toggleBtn.innerHTML = ' Hide Text'; + + // Load image info if not already loaded + if (infoContainer.innerHTML.includes('Loading image information...')) { + loadImageInfo(fullMessageObject, infoContainer); + } + } + + // Restore scroll position after DOM changes + setTimeout(() => { + if (document.getElementById('chat-messages-container')) { + document.getElementById('chat-messages-container').scrollTop = currentScrollTop; + } else { + window.scrollTo(0, currentScrollTop); + } + }, 10); +} + +/** + * Load image extracted text and vision analysis into the info drawer + */ +function loadImageInfo(fullMessageObject, container) { + const extractedText = fullMessageObject?.extracted_text || ''; + const visionAnalysis = fullMessageObject?.vision_analysis || null; + const filename = fullMessageObject?.filename || 'Image'; + + let content = '
'; + + // Filename + content += `
Filename: ${escapeHtml(filename)}
`; + + // Extracted Text (OCR from Document Intelligence) + if (extractedText && extractedText.trim()) { + content += '
'; + content += 'Extracted Text (OCR):'; + content += '
'; + content += escapeHtml(extractedText); + content += '
'; + } + + // Vision Analysis (AI-generated description, objects, text) + if (visionAnalysis) { + content += '
'; + content += 'AI Vision Analysis:'; + + if (visionAnalysis.model_name) { + content += `
Model: ${escapeHtml(visionAnalysis.model_name)}
`; + } + + if (visionAnalysis.description) { + content += '
Description:
'; + content += escapeHtml(visionAnalysis.description); + content += '
'; + } + + if (visionAnalysis.objects && Array.isArray(visionAnalysis.objects) && visionAnalysis.objects.length > 0) { + content += '
Objects Detected:
'; + content += visionAnalysis.objects.map(obj => `${escapeHtml(obj)}`).join(''); + content += '
'; + } + + if (visionAnalysis.text && visionAnalysis.text.trim()) { + content += '
Text Visible in Image:
'; + content += escapeHtml(visionAnalysis.text); + content += '
'; + } + + if (visionAnalysis.contextual_analysis && visionAnalysis.contextual_analysis.trim()) { + content += '
Contextual Analysis:
'; + content += escapeHtml(visionAnalysis.contextual_analysis); + content += '
'; + } + + content += '
'; + } + + content += '
'; + + if (!extractedText && !visionAnalysis) { + content = '
No extracted text or analysis available for this image.
'; + } + + container.innerHTML = content; +} diff --git a/application/single_app/templates/_video_indexer_info.html b/application/single_app/templates/_video_indexer_info.html index 6bd5f509..0042068b 100644 --- a/application/single_app/templates/_video_indexer_info.html +++ b/application/single_app/templates/_video_indexer_info.html @@ -12,7 +12,12 @@

${ws.name}

-

${ws.description || ""}

-

Owner: ${owner.displayName} (${owner.email})

- `); + // Update profile hero + updateProfileHero(ws, owner); // Determine role if (userId === owner.userId) { @@ -174,12 +262,25 @@ function loadWorkspaceInfo(callback) { $("#editWorkspaceContainer").show(); $("#editWorkspaceName").val(ws.name); $("#editWorkspaceDescription").val(ws.description); + + // Set selected color + const color = ws.heroColor || '#0078d4'; + $("#selectedColor").val(color); + updateHeroColor(color); + $(`.color-option[data-color="${color}"]`).addClass('selected'); + } + + // Show member actions for non-owners + if (currentUserRole !== "Owner" && currentUserRole) { + $("#memberActionsContainer").show(); } // Admin & Owner UI if (currentUserRole === "Owner" || currentUserRole === "Admin") { $("#addMemberBtn").show(); + $("#addBulkMemberBtn").show(); $("#pendingRequestsSection").show(); + $("#activityTimelineSection").show(); loadPendingRequests(); } @@ -194,7 +295,8 @@ function loadWorkspaceInfo(callback) { function updateWorkspaceInfo() { const data = { name: $("#editWorkspaceName").val().trim(), - description: $("#editWorkspaceDescription").val().trim() + description: $("#editWorkspaceDescription").val().trim(), + heroColor: $("#selectedColor").val() }; $.ajax({ url: `/api/public_workspaces/${workspaceId}`, @@ -223,8 +325,18 @@ function loadMembers(searchTerm = "", roleFilter = "") { $.get(url) .done(function (members) { const rows = members.map(m => { + const isOwner = m.role === "Owner"; + const checkboxHtml = isOwner || (currentUserRole !== "Owner" && currentUserRole !== "Admin") + ? '' + : ``; + return ` + ${checkboxHtml} ${m.displayName || "(no name)"}
${m.email || ""} @@ -235,10 +347,14 @@ function loadMembers(searchTerm = "", roleFilter = "") { `; }).join(""); $("#membersTable tbody").html(rows); + + // Reset selection UI + $("#selectAllMembers").prop("checked", false); + updateBulkActionsBar(); }) .fail(function () { $("#membersTable tbody").html( - `Failed to load members.` + `Failed to load members.` ); }); } @@ -436,3 +552,743 @@ function addMemberDirectly() { } }); } + +// --- New Functions for Profile Hero and Stats --- + +// Update profile hero section +function updateProfileHero(workspace, owner) { + const initial = workspace.name ? workspace.name.charAt(0).toUpperCase() : 'W'; + $('#workspaceInitial').text(initial); + $('#workspaceHeroName').text(workspace.name || 'Unnamed Workspace'); + $('#workspaceOwnerName').text(owner.displayName || 'Unknown'); + $('#workspaceOwnerEmail').text(owner.email || 'N/A'); + $('#workspaceHeroDescription').text(workspace.description || 'No description provided'); + + // Apply hero color + const color = workspace.heroColor || '#0078d4'; + updateHeroColor(color); +} + +// Update hero color +function updateHeroColor(color) { + const darker = adjustColorBrightness(color, -30); + document.documentElement.style.setProperty('--hero-color', color); + document.documentElement.style.setProperty('--hero-color-dark', darker); +} + +// Adjust color brightness +function adjustColorBrightness(color, percent) { + const num = parseInt(color.replace('#', ''), 16); + const amt = Math.round(2.55 * percent); + const R = (num >> 16) + amt; + const G = (num >> 8 & 0x00FF) + amt; + const B = (num & 0x0000FF) + amt; + return '#' + (0x1000000 + (R < 255 ? R < 1 ? 0 : R : 255) * 0x10000 + + (G < 255 ? G < 1 ? 0 : G : 255) * 0x100 + + (B < 255 ? B < 1 ? 0 : B : 255)) + .toString(16).slice(1); +} + +// Initialize color picker +function initializeColorPicker() { + $('.color-option').on('click', function() { + $('.color-option').removeClass('selected'); + $(this).addClass('selected'); + const color = $(this).data('color'); + $('#selectedColor').val(color); + updateHeroColor(color); + }); +} + +// Load workspace stats +let documentChart, storageChart, tokenChart; + +function loadWorkspaceStats() { + // Load stats data + $.get(`/api/public_workspaces/${workspaceId}/stats`) + .done(function(stats) { + updateStatCards(stats); + updateCharts(stats); + // Load activity timeline if user has permission + if (currentUserRole === "Owner" || currentUserRole === "Admin") { + loadActivityTimeline(50); + } + }) + .fail(function() { + console.error('Failed to load workspace stats'); + $('#stat-documents').text('N/A'); + $('#stat-storage').text('N/A'); + $('#stat-tokens').text('N/A'); + $('#stat-members').text('N/A'); + }); +} + +// Update stat cards +function updateStatCards(stats) { + $('#stat-documents').text(stats.totalDocuments || 0); + $('#stat-storage').text(formatBytes(stats.storageUsed || 0)); + $('#stat-tokens').text(formatNumber(stats.totalTokens || 0)); + $('#stat-members').text(stats.totalMembers || 0); +} + +// Update charts +function updateCharts(stats) { + // Document Activity Chart - Two bars for uploads and deletes + const docCtx = document.getElementById('documentChart'); + if (docCtx) { + if (documentChart) documentChart.destroy(); + documentChart = new Chart(docCtx, { + type: 'bar', + data: { + labels: stats.documentActivity?.labels || [], + datasets: [ + { + label: 'Uploads', + data: stats.documentActivity?.uploads || [], + backgroundColor: 'rgba(13, 202, 240, 0.8)', + borderColor: 'rgb(13, 202, 240)', + borderWidth: 1 + }, + { + label: 'Deletes', + data: stats.documentActivity?.deletes || [], + backgroundColor: 'rgba(220, 53, 69, 0.8)', + borderColor: 'rgb(220, 53, 69)', + borderWidth: 1 + } + ] + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + display: true, + position: 'top' + } + }, + scales: { + y: { + beginAtZero: true, + ticks: { precision: 0 } + } + } + } + }); + } + + // Storage Usage Chart (Doughnut) - AI Search and Blob Storage + const storageCtx = document.getElementById('storageChart'); + if (storageCtx) { + if (storageChart) storageChart.destroy(); + const aiSearch = stats.storage?.ai_search_size || 0; + const blobStorage = stats.storage?.storage_account_size || 0; + + storageChart = new Chart(storageCtx, { + type: 'doughnut', + data: { + labels: ['AI Search', 'Blob Storage'], + datasets: [{ + data: [aiSearch, blobStorage], + backgroundColor: [ + 'rgb(13, 110, 253)', + 'rgb(23, 162, 184)' + ], + borderWidth: 2 + }] + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { position: 'bottom' }, + tooltip: { + callbacks: { + label: function(context) { + return context.label + ': ' + formatBytes(context.parsed); + } + } + } + } + } + }); + } + + // Token Usage Chart + const tokenCtx = document.getElementById('tokenChart'); + if (tokenCtx) { + if (tokenChart) tokenChart.destroy(); + tokenChart = new Chart(tokenCtx, { + type: 'bar', + data: { + labels: stats.tokenUsage?.labels || [], + datasets: [{ + label: 'Tokens Used', + data: stats.tokenUsage?.data || [], + backgroundColor: 'rgba(255, 193, 7, 0.7)', + borderColor: 'rgb(255, 193, 7)', + borderWidth: 1 + }] + }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { display: false } + }, + scales: { + y: { + beginAtZero: true, + ticks: { + callback: function(value) { + return formatNumber(value); + } + } + } + } + } + }); + } +} + +// Load activity timeline +function loadActivityTimeline(limit = 50) { + $.get(`/api/public_workspaces/${workspaceId}/activity?limit=${limit}`) + .done(function(activities) { + if (!activities || activities.length === 0) { + $('#activityTimeline').html('

No recent activity

'); + return; + } + + const html = activities.map(activity => renderActivityItem(activity)).join(''); + $('#activityTimeline').html(html); + }) + .fail(function(xhr) { + if (xhr.status === 403) { + $('#activityTimeline').html('

Access denied - Only workspace owners and admins can view activity timeline

'); + } else { + $('#activityTimeline').html('

Failed to load activity

'); + } + }); +} + +// Render activity item +function renderActivityItem(activity) { + const icons = { + 'document_creation': 'file-earmark-arrow-up', + 'document_deletion': 'file-earmark-x', + 'token_usage': 'cpu', + 'user_login': 'box-arrow-in-right' + }; + + const colors = { + 'document_creation': 'success', + 'document_deletion': 'danger', + 'token_usage': 'primary', + 'user_login': 'info' + }; + + const activityType = activity.activity_type || 'unknown'; + const icon = icons[activityType] || 'circle'; + const color = colors[activityType] || 'secondary'; + const time = formatRelativeTime(activity.timestamp || activity.created_at); + + // Generate description based on activity type + let description = ''; + let title = activityType.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase()); + + if (activityType === 'document_creation' && activity.document) { + description = `File: ${activity.document.file_name || 'Unknown'}`; + } else if (activityType === 'document_deletion' && activity.document_metadata) { + description = `File: ${activity.document_metadata.file_name || 'Unknown'}`; + } else if (activityType === 'token_usage' && activity.usage) { + description = `Tokens: ${formatNumber(activity.usage.total_tokens || 0)}`; + } else if (activityType === 'user_login') { + description = 'User logged in'; + } + + const activityJson = JSON.stringify(activity); + + return ` +
+
+
+ +
+
+
+
${title}
+ ${time} +
+

${description}

+
+
+
+ `; +} + +// Format bytes +function formatBytes(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i]; +} + +// Format number with commas +function formatNumber(num) { + return num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ','); +} + +// Show raw activity in modal +function showRawActivity(element) { + try { + const activityJson = element.getAttribute('data-activity'); + const activity = JSON.parse(activityJson); + const modalBody = document.getElementById('rawActivityModalBody'); + modalBody.innerHTML = `
${JSON.stringify(activity, null, 2)}
`; + $('#rawActivityModal').modal('show'); + } catch (error) { + console.error('Error showing raw activity:', error); + } +} + +// Copy raw activity to clipboard +function copyRawActivityToClipboard() { + const modalBody = document.getElementById('rawActivityModalBody'); + const text = modalBody.textContent; + navigator.clipboard.writeText(text).then(() => { + alert('Activity data copied to clipboard!'); + }).catch(err => { + console.error('Failed to copy:', err); + }); +} + +// Make functions globally available +window.showRawActivity = showRawActivity; +window.copyRawActivityToClipboard = copyRawActivityToClipboard; + +// Format relative time +function formatRelativeTime(timestamp) { + const now = new Date(); + const date = new Date(timestamp); + const diffMs = now - date; + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMs / 3600000); + const diffDays = Math.floor(diffMs / 86400000); + + if (diffMins < 1) return 'Just now'; + if (diffMins < 60) return `${diffMins}m ago`; + if (diffHours < 24) return `${diffHours}h ago`; + if (diffDays < 7) return `${diffDays}d ago`; + return date.toLocaleDateString(); +} + +// ============================================================================ +// CSV Bulk Member Upload Functions +// ============================================================================ + +let csvParsedData = []; + +function downloadCsvExample() { + const csvContent = `userId,displayName,email,role +00000000-0000-0000-0000-000000000001,John Smith,john.smith@contoso.com,user +00000000-0000-0000-0000-000000000002,Jane Doe,jane.doe@contoso.com,admin +00000000-0000-0000-0000-000000000003,Bob Johnson,bob.johnson@contoso.com,document_manager`; + + const blob = new Blob([csvContent], { type: 'text/csv' }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'bulk_members_example.csv'; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + window.URL.revokeObjectURL(url); +} + +function showCsvConfig() { + const modal = new bootstrap.Modal(document.getElementById('csvFormatInfoModal')); + modal.show(); +} + +function validateGuid(guid) { + return ValidationUtils.validateGuid(guid); +} + +function validateEmail(email) { + const emailRegex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; + return emailRegex.test(email); +} + +function handleCsvFileSelect(event) { + const file = event.target.files[0]; + if (!file) { + $("#csvNextBtn").prop("disabled", true); + $("#csvValidationResults").hide(); + $("#csvErrorDetails").hide(); + return; + } + + const reader = new FileReader(); + reader.onload = function (e) { + const text = e.target.result; + const lines = text.split(/\r?\n/).filter(line => line.trim()); + + $("#csvErrorDetails").hide(); + $("#csvValidationResults").hide(); + + // Validate header + if (lines.length < 2) { + showCsvError("CSV must contain at least a header row and one data row"); + return; + } + + const header = lines[0].toLowerCase().trim(); + if (header !== "userid,displayname,email,role") { + showCsvError("Invalid header. Expected: userId,displayName,email,role"); + return; + } + + // Validate row count + const dataRows = lines.slice(1); + if (dataRows.length > 1000) { + showCsvError(`Too many rows. Maximum 1,000 members allowed (found ${dataRows.length})`); + return; + } + + // Parse and validate rows + csvParsedData = []; + const errors = []; + const validRoles = ['user', 'admin', 'document_manager']; + + for (let i = 0; i < dataRows.length; i++) { + const rowNum = i + 2; // +2 because header is row 1 + const row = dataRows[i].split(','); + + if (row.length !== 4) { + errors.push(`Row ${rowNum}: Expected 4 columns, found ${row.length}`); + continue; + } + + const userId = row[0].trim(); + const displayName = row[1].trim(); + const email = row[2].trim(); + const role = row[3].trim().toLowerCase(); + + if (!userId || !displayName || !email || !role) { + errors.push(`Row ${rowNum}: All fields are required`); + continue; + } + + if (!validateGuid(userId)) { + errors.push(`Row ${rowNum}: Invalid GUID format for userId`); + continue; + } + + if (!validateEmail(email)) { + errors.push(`Row ${rowNum}: Invalid email format`); + continue; + } + + if (!validRoles.includes(role)) { + errors.push(`Row ${rowNum}: Invalid role '${role}'. Must be: user, admin, or document_manager`); + continue; + } + + csvParsedData.push({ userId, displayName, email, role }); + } + + if (errors.length > 0) { + showCsvError(`Found ${errors.length} validation error(s):\n` + errors.slice(0, 10).join('\n') + + (errors.length > 10 ? `\n... and ${errors.length - 10} more` : '')); + return; + } + + // Show validation success + const sampleRows = csvParsedData.slice(0, 3); + $("#csvValidationDetails").html(` +

✓ Valid CSV file detected

+

Total members to add: ${csvParsedData.length}

+

Sample data (first 3):

+
    + ${sampleRows.map(row => `
  • ${row.displayName} (${row.email})
    • `).join('')} +
+ `); + $("#csvValidationResults").show(); + $("#csvNextBtn").prop("disabled", false); + }; + + reader.readAsText(file); +} + +function showCsvError(message) { + $("#csvErrorList").html(`
${escapeHtml(message)}
`); + $("#csvErrorDetails").show(); + $("#csvNextBtn").prop("disabled", true); + csvParsedData = []; +} + +function startCsvUpload() { + if (csvParsedData.length === 0) { + alert("No valid data to upload"); + return; + } + + // Switch to stage 2 + $("#csvStage1").hide(); + $("#csvStage2").show(); + $("#csvNextBtn").hide(); + $("#csvCancelBtn").hide(); + $("#csvModalClose").hide(); + + // Upload members + uploadCsvMembers(); +} + +async function uploadCsvMembers() { + let successCount = 0; + let failedCount = 0; + let skippedCount = 0; + const failures = []; + + for (let i = 0; i < csvParsedData.length; i++) { + const member = csvParsedData[i]; + const progress = Math.round(((i + 1) / csvParsedData.length) * 100); + + updateCsvProgress(progress, `Processing ${i + 1} of ${csvParsedData.length}: ${member.displayName}`); + + try { + const response = await fetch(`/api/public_workspaces/${workspaceId}/members`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + userId: member.userId, + displayName: member.displayName, + email: member.email, + role: member.role + }) + }); + + const data = await response.json(); + + if (response.ok && data.success) { + successCount++; + } else if (data.error && data.error.includes('already a member')) { + skippedCount++; + } else { + failedCount++; + failures.push(`${member.displayName}: ${data.error || 'Unknown error'}`); + } + } catch (error) { + failedCount++; + failures.push(`${member.displayName}: ${error.message}`); + } + } + + // Show summary + showCsvSummary(successCount, failedCount, skippedCount, failures); +} + +function updateCsvProgress(percentage, statusText) { + $("#csvProgressBar").css("width", percentage + "%"); + $("#csvProgressBar").attr("aria-valuenow", percentage); + $("#csvProgressText").text(percentage + "%"); + $("#csvStatusText").text(statusText); +} + +function showCsvSummary(successCount, failedCount, skippedCount, failures) { + $("#csvStage2").hide(); + $("#csvStage3").show(); + $("#csvDoneBtn").show(); + + let summaryHtml = ` +

Upload Summary:

+
    +
  • ✅ Successfully added: ${successCount}
    • +
  • ⏭️ Skipped (already members): ${skippedCount}
    • +
  • ❌ Failed: ${failedCount}
    • +
+ `; + + if (failures.length > 0) { + summaryHtml += ` +
+

Failed Members:

+
    + ${failures.slice(0, 10).map(f => `
  • ${escapeHtml(f)}
    • `).join('')} + ${failures.length > 10 ? `
  • ... and ${failures.length - 10} more
    • ` : ''} +
+ `; + } + + $("#csvSummary").html(summaryHtml); +} + +function resetCsvModal() { + // Reset to stage 1 + $("#csvStage1").show(); + $("#csvStage2").hide(); + $("#csvStage3").hide(); + $("#csvNextBtn").show(); + $("#csvNextBtn").prop("disabled", true); + $("#csvCancelBtn").show(); + $("#csvDoneBtn").hide(); + $("#csvModalClose").show(); + $("#csvValidationResults").hide(); + $("#csvErrorDetails").hide(); + $("#csvFileInput").val(''); + csvParsedData = []; + + // Reset progress + updateCsvProgress(0, 'Ready'); +} + +// ============================================================================ +// Bulk Member Actions Functions +// ============================================================================ + +function getSelectedMembers() { + const selected = []; + $(".member-checkbox:checked").each(function () { + selected.push({ + userId: $(this).data("user-id"), + name: $(this).data("user-name"), + email: $(this).data("user-email"), + role: $(this).data("user-role") + }); + }); + return selected; +} + +function updateBulkActionsBar() { + const selectedCount = $(".member-checkbox:checked").length; + if (selectedCount > 0) { + $("#selectedCount").text(selectedCount); + $("#bulkActionsBar").show(); + } else { + $("#bulkActionsBar").hide(); + } +} + +function updateSelectAllCheckbox() { + const totalCheckboxes = $(".member-checkbox").length; + const checkedCheckboxes = $(".member-checkbox:checked").length; + + if (totalCheckboxes > 0 && checkedCheckboxes === totalCheckboxes) { + $("#selectAllMembers").prop("checked", true); + $("#selectAllMembers").prop("indeterminate", false); + } else if (checkedCheckboxes > 0) { + $("#selectAllMembers").prop("checked", false); + $("#selectAllMembers").prop("indeterminate", true); + } else { + $("#selectAllMembers").prop("checked", false); + $("#selectAllMembers").prop("indeterminate", false); + } +} + +async function bulkAssignRole() { + const selectedMembers = getSelectedMembers(); + const newRole = $("#bulkRoleSelect").val(); + + if (selectedMembers.length === 0) { + alert("No members selected"); + return; + } + + // Close modal and show progress + $("#bulkAssignRoleModal").modal("hide"); + + let successCount = 0; + let failedCount = 0; + const failures = []; + + for (let i = 0; i < selectedMembers.length; i++) { + const member = selectedMembers[i]; + + try { + const response = await fetch(`/api/public_workspaces/${workspaceId}/members/${member.userId}`, { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ role: newRole }) + }); + + const data = await response.json(); + + if (response.ok && data.success) { + successCount++; + } else { + failedCount++; + failures.push(`${member.name}: ${data.error || 'Unknown error'}`); + } + } catch (error) { + failedCount++; + failures.push(`${member.name}: ${error.message}`); + } + } + + // Show summary + let message = `Role assignment complete:\n✅ Success: ${successCount}\n❌ Failed: ${failedCount}`; + if (failures.length > 0) { + message += "\n\nFailed members:\n" + failures.slice(0, 5).join("\n"); + if (failures.length > 5) { + message += `\n... and ${failures.length - 5} more`; + } + } + alert(message); + + // Reload members and clear selection + loadMembers(); +} + +async function bulkRemoveMembers() { + const selectedMembers = getSelectedMembers(); + + if (selectedMembers.length === 0) { + alert("No members selected"); + return; + } + + // Close modal + $("#bulkRemoveMembersModal").modal("hide"); + + let successCount = 0; + let failedCount = 0; + const failures = []; + + for (let i = 0; i < selectedMembers.length; i++) { + const member = selectedMembers[i]; + + try { + const response = await fetch(`/api/public_workspaces/${workspaceId}/members/${member.userId}`, { + method: 'DELETE' + }); + + const data = await response.json(); + + if (response.ok && data.success) { + successCount++; + } else { + failedCount++; + failures.push(`${member.name}: ${data.error || 'Unknown error'}`); + } + } catch (error) { + failedCount++; + failures.push(`${member.name}: ${error.message}`); + } + } + + // Show summary + let message = `Member removal complete:\n✅ Success: ${successCount}\n❌ Failed: ${failedCount}`; + if (failures.length > 0) { + message += "\n\nFailed removals:\n" + failures.slice(0, 5).join("\n"); + if (failures.length > 5) { + message += `\n... and ${failures.length - 5} more`; + } + } + alert(message); + + // Reload members and clear selection + loadMembers(); +} diff --git a/application/single_app/static/js/public/my_public_workspaces.js b/application/single_app/static/js/public/my_public_workspaces.js index 7123d4b7..21e6d0ef 100644 --- a/application/single_app/static/js/public/my_public_workspaces.js +++ b/application/single_app/static/js/public/my_public_workspaces.js @@ -2,7 +2,7 @@ $(document).ready(function () { // Grab global active workspace ID (set via inline {% endblock %} diff --git a/application/single_app/templates/approvals.html b/application/single_app/templates/approvals.html new file mode 100644 index 00000000..a3388f23 --- /dev/null +++ b/application/single_app/templates/approvals.html @@ -0,0 +1,669 @@ +{% extends "base.html" %} + +{% block title %}Approval Requests - {{ app_settings.app_title }}{% endblock %} + +{% block content %} +
+
+
+
+

+ Approval Requests +

+ +
+ + +
+
+
+ + +
+
+
+ +
+
+ +
+
+ +
+
+ + +
+
+
+ + + + + + + + + + + + + + + + +
Request TypeGroup NameRequested ByCreatedStatusActions
+ +
Loading approvals...
+
+
+ + +
+
+ +
+ +
+
+
+
+
+
+ + + +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/application/single_app/templates/base.html b/application/single_app/templates/base.html index 854b6af3..614f7870 100644 --- a/application/single_app/templates/base.html +++ b/application/single_app/templates/base.html @@ -336,6 +336,8 @@ + + {% block scripts %}{% endblock %} diff --git a/application/single_app/templates/chats.html b/application/single_app/templates/chats.html index 0ec65b1d..ec7c8242 100644 --- a/application/single_app/templates/chats.html +++ b/application/single_app/templates/chats.html @@ -5,6 +5,9 @@ {% block head %} + {% if app_settings.enable_speech_to_text_input %} + + {% endif %} {% endblock %} @@ -560,8 +644,9 @@
Activity Trends - Real-time, does not require refresh +
+ Real-time, does not require refresh
@@ -989,10 +1074,18 @@
- - + + + + + + + + + +
@@ -1146,6 +1239,9 @@
File Upload Control
+
+ + + @@ -1473,9 +1604,20 @@
Group Ownership
+ @@ -1496,7 +1638,7 @@
Member Management
Add/remove members and assign roles
- See detailed group activity timeline @@ -1507,6 +1649,49 @@
Member Management
+ + {% if app_settings.enable_retention_policy_group %} +
+
+
Retention Policy
+
+
+
+ + Configure automatic deletion of aged conversations and documents. Set to "No automatic deletion" to keep items indefinitely. +
+
+
+ + +
+
+ + +
+
+
+
+
+
+ {% endif %} +
@@ -1649,13 +1834,16 @@
-
+
Group Members
-
- +
@@ -1663,11 +1851,8 @@
Group Members
- + - - - @@ -1678,128 +1863,970 @@
Group Members
-
- -
-
-
-
- - - - - - - - + + + + + + + {% endblock %} \ No newline at end of file diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index 7f791072..035aee68 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -145,6 +145,28 @@ #group-dropdown-button { text-align: left; } + + /* Group status badges */ + .group-status-badge { + display: inline-block; + padding: 0.2em 0.5em; + font-size: 0.75em; + font-weight: 600; + margin-left: 0.5rem; + border-radius: 0.25rem; + } + .group-status-locked { + background-color: #ffc107; + color: #000; + } + .group-status-upload-disabled { + background-color: #17a2b8; + color: #fff; + } + .group-status-inactive { + background-color: #dc3545; + color: #fff; + } {% endblock %} {% block content %}
@@ -189,6 +211,11 @@

Group Workspace

+ +
+ +
+
+ + +
MemberName RoleJoinedLast ActiveActions
@@ -417,17 +464,7 @@
Group Documents
- + @@ -1091,7 +1128,8 @@
Currently Shared With:
try { groupSimplemde = new SimpleMDE({ element: groupPromptContentEl, - spellChecker: false + spellChecker: false, + autoDownloadFontAwesome: false }); } catch (e) { console.error("Failed to initialize SimpleMDE for group prompts:", e); @@ -1374,55 +1412,37 @@
Currently Shared With:
} function updateGroupBulkActionButtons() { + const bulkActionsBar = document.getElementById("groupBulkActionsBar"); + const selectedCountSpan = document.getElementById("groupSelectedCount"); const deleteBtn = document.getElementById("group-delete-selected-btn"); const removeBtn = document.getElementById("group-remove-selected-btn"); - const bulkActions = document.getElementById("group-bulk-actions"); if (groupSelectedDocuments.size > 0) { - // Show bulk actions container - if (bulkActions) bulkActions.style.display = "inline-block"; + // Show bulk actions bar with count + if (bulkActionsBar) { + bulkActionsBar.style.display = "block"; + } + if (selectedCountSpan) { + selectedCountSpan.textContent = groupSelectedDocuments.size; + } // Check if user can manage documents (delete permission) const canManage = ["Owner", "Admin", "DocumentManager"].includes(userRoleInActiveGroup); - // For group documents, we need to determine if any selected documents - // require different actions based on user permissions - let hasOwnedDocuments = false; - let hasNonOwnedDocuments = false; - - // Check ownership of selected documents - Array.from(groupSelectedDocuments).forEach(docId => { - const docRow = document.getElementById(`group-doc-row-${docId}`); - if (docRow && docRow.__docData) { - const doc = docRow.__docData; - // In group context, we check if user has management permissions - // rather than individual document ownership - if (canManage) { - hasOwnedDocuments = true; - } else { - hasNonOwnedDocuments = true; - } - } - }); - - // Delete button: Show only if user has management permissions for group documents + // Show/hide delete and remove buttons based on permissions if (deleteBtn) { - deleteBtn.style.display = (canManage && hasOwnedDocuments) ? "inline-block" : "none"; + deleteBtn.style.display = canManage ? "inline-block" : "none"; } - // Remove button: Show for non-managers or when user wants to "remove from group" - // In group context, this could mean removing the document from the group - // but keeping it in the system (if supported by backend) + // Remove button is available for group documents if (removeBtn) { - // For now, hide remove button as group documents are typically deleted, not removed - // This can be adjusted based on backend implementation - removeBtn.style.display = "none"; + removeBtn.style.display = "inline-block"; } } else { - // No documents selected - hide everything - if (deleteBtn) deleteBtn.style.display = "none"; - if (removeBtn) removeBtn.style.display = "none"; - if (bulkActions) bulkActions.style.display = "none"; + // Hide bulk actions bar + if (bulkActionsBar) { + bulkActionsBar.style.display = "none"; + } } } @@ -1430,6 +1450,7 @@
Currently Shared With:
const table = document.getElementById("group-documents-table"); const checkboxes = document.querySelectorAll('.document-checkbox'); const expandContainers = document.querySelectorAll('.expand-collapse-container'); + const bulkActionsBar = document.getElementById("groupBulkActionsBar"); groupSelectionMode = !groupSelectionMode; @@ -1445,12 +1466,6 @@
Currently Shared With:
expandContainers.forEach(container => { container.style.display = 'none'; }); - - // Show bulk actions - const bulkActions = document.getElementById("group-bulk-actions"); - if (bulkActions) { - bulkActions.style.display = 'inline-block'; - } } else { // Exit selection mode table.classList.remove("selection-mode"); @@ -1465,21 +1480,26 @@
Currently Shared With:
container.style.display = 'inline-block'; }); - // Hide bulk actions and buttons - const bulkActions = document.getElementById("group-bulk-actions"); - if (bulkActions) { - bulkActions.style.display = 'none'; + // Hide bulk actions bar + if (bulkActionsBar) { + bulkActionsBar.style.display = 'none'; } - const deleteBtn = document.getElementById("group-delete-selected-btn"); - const removeBtn = document.getElementById("group-remove-selected-btn"); - if (deleteBtn) deleteBtn.style.display = 'none'; - if (removeBtn) removeBtn.style.display = 'none'; // Clear selected documents groupSelectedDocuments.clear(); } } + // Clear group selection + function clearGroupSelection() { + const checkboxes = document.querySelectorAll('.document-checkbox'); + checkboxes.forEach(checkbox => { + checkbox.checked = false; + }); + groupSelectedDocuments.clear(); + updateGroupBulkActionButtons(); + } + function deleteGroupSelectedDocuments() { if (groupSelectedDocuments.size === 0) return; @@ -1635,6 +1655,7 @@
Currently Shared With:
// Add event listeners for bulk action buttons const groupDeleteSelectedBtn = document.getElementById("group-delete-selected-btn"); const groupRemoveSelectedBtn = document.getElementById("group-remove-selected-btn"); + const groupClearSelectionBtn = document.getElementById("group-clear-selection-btn"); if (groupDeleteSelectedBtn) { groupDeleteSelectedBtn.addEventListener("click", deleteGroupSelectedDocuments); @@ -1642,6 +1663,9 @@
Currently Shared With:
if (groupRemoveSelectedBtn) { groupRemoveSelectedBtn.addEventListener("click", removeGroupSelectedDocuments); } + if (groupClearSelectionBtn) { + groupClearSelectionBtn.addEventListener("click", clearGroupSelection); + } // Load initial data fetchUserGroups().then(() => { @@ -1727,6 +1751,134 @@
Currently Shared With:
}); // End DOMContentLoaded /* ===================== GROUPS / ROLE / CONTEXT ===================== */ + + /** + * Update the group status alert box based on the active group's status + */ + function updateGroupStatusAlert() { + const activeGroup = userGroups.find(g => g.id === activeGroupId); + const alertBox = document.getElementById("group-status-alert"); + + if (!activeGroup || !alertBox) { + return; + } + + const status = activeGroup.status || 'active'; + const statusMessages = { + 'locked': { + type: 'warning', + icon: 'bi-lock-fill', + title: '🔒 Locked (Read-Only)', + message: 'Group is in read-only mode', + details: [ + '❌ New document uploads', + '❌ Document deletions', + '❌ Creating, editing, or deleting prompts, agents, and actions', + '✅ Viewing existing documents', + '✅ Chat and search with existing documents', + '✅ Using existing prompts, agents, and actions' + ] + }, + 'upload_disabled': { + type: 'info', + icon: 'bi-cloud-slash-fill', + title: '📁 Upload Disabled', + message: 'Restrict new content but allow other operations', + details: [ + '❌ New document uploads', + '✅ Document deletions (cleanup)', + '✅ Full chat and search functionality', + '✅ Creating, editing, and deleting prompts, agents, and actions' + ] + }, + 'inactive': { + type: 'danger', + icon: 'bi-exclamation-triangle-fill', + title: '⭕ Inactive', + message: 'Group is disabled', + details: [ + '❌ ALL operations (uploads, chat, document access)', + '❌ Creating, editing, or deleting prompts, agents, and actions', + '✅ Only admin viewing of group information', + 'Use case: Decommissioned projects, suspended groups, compliance holds' + ] + } + }; + + if (status === 'active') { + alertBox.classList.add('d-none'); + } else { + const config = statusMessages[status]; + if (config) { + alertBox.classList.remove('d-none', 'alert-warning', 'alert-info', 'alert-danger'); + alertBox.classList.add(`alert-${config.type}`); + + const detailsList = config.details.map(d => `
  • ${d}
    • `).join(''); + + alertBox.innerHTML = ` +
    + +
    + ${config.title} - ${config.message} +
      + ${detailsList} +
    +
    +
    + `; + } + } + } + + /** + * Hide/show UI elements based on the active group's status + */ + function updateGroupUIBasedOnStatus() { + const activeGroup = userGroups.find(g => g.id === activeGroupId); + + if (!activeGroup) { + return; + } + + const status = activeGroup.status || 'active'; + + // Define elements to hide/show (note: upload-section is handled by updateRoleDisplay) + const createPromptBtn = document.getElementById('create-group-prompt-btn'); + const createAgentBtn = document.getElementById('create-group-agent-btn'); + const createPluginBtn = document.getElementById('create-group-plugin-btn'); + const deleteSelectedBtn = document.getElementById('group-delete-selected-btn'); + + // Determine what operations are allowed based on status + const canCreateItems = (status === 'active'); + const canModifyItems = (status === 'active'); // locked and inactive cannot modify + const canDeleteDocs = (status === 'active' || status === 'upload_disabled'); // Can delete in upload_disabled + const canChat = (status !== 'inactive'); // Chat disabled only for inactive + + // Hide/show create buttons based on permissions + if (createPromptBtn) { + createPromptBtn.style.display = canCreateItems ? '' : 'none'; + } + if (createAgentBtn) { + createAgentBtn.style.display = canCreateItems ? '' : 'none'; + } + if (createPluginBtn) { + createPluginBtn.style.display = canCreateItems ? '' : 'none'; + } + + // Hide/show delete button for documents (shown only if can delete) + // Note: This will be further refined by role permissions in updateRoleDisplay + if (deleteSelectedBtn && !canDeleteDocs) { + deleteSelectedBtn.style.display = 'none'; + } + + // Store status globally so other functions can check it + window.currentGroupStatus = status; + window.currentGroupCanModify = canModifyItems; + window.currentGroupCanChat = canChat; + + console.log(`UI updated for group ${activeGroupId} with status: ${status}`); + } + function fetchUserGroups() { console.log("Fetching user groups..."); // Add loading state to group selector? @@ -1861,6 +2013,8 @@
    Currently Shared With:
    // Only update role element since heading element was removed activeGroupNameRoleEl.textContent = activeGroupName; updateRoleDisplay(); // Update role display and UI elements + updateGroupStatusAlert(); // Update status alert box + updateGroupUIBasedOnStatus(); // Hide/show UI elements based on status } else { // Reset state if no active group found among the user's groups activeGroupId = null; @@ -2005,7 +2159,12 @@
    Currently Shared With:
    } // Control visibility of upload section and its divider - const showUpload = canManageDocs; + // Check BOTH role AND group status - must have permission AND group must be active + const activeGroup = userGroups.find(g => g.id === activeGroupId); + const groupStatus = activeGroup ? (activeGroup.status || 'active') : 'active'; + const groupAllowsUpload = (groupStatus === 'active'); + const showUpload = canManageDocs && groupAllowsUpload; + if (uploadSection) uploadSection.style.display = showUpload ? "block" : "none"; if (uploadHr) uploadHr.style.display = showUpload ? "block" : "none"; @@ -2193,8 +2352,12 @@
    Currently Shared With:
    let actionsDropdown = ''; let chatButton = ''; - // Chat button for everyone with access (outside dropdown) - matching workspace.html - if (isComplete && !hasError) { + // Get current group status + const groupStatus = window.currentGroupStatus || 'active'; + const canChat = (groupStatus !== 'inactive'); + + // Chat button for everyone with access (outside dropdown) - but only if group allows chat + if (isComplete && !hasError && canChat) { chatButton = ` `; if ( @@ -3128,16 +3317,28 @@
    function renderGroupPromptRow(p) { const tr = document.createElement("tr"); - tr.innerHTML = ` -
    - `; + `; + } else { + actionsHtml = '—'; + } + + tr.innerHTML = ` + + `; groupPromptsTableBody.appendChild(tr); } diff --git a/application/single_app/templates/manage_group.html b/application/single_app/templates/manage_group.html index 0cd2522e..78f80cdb 100644 --- a/application/single_app/templates/manage_group.html +++ b/application/single_app/templates/manage_group.html @@ -4,123 +4,498 @@ {% block title %} Manage Group - {{app_settings.app_title }} {% endblock %} -{% block content %} +{% block head %} + +{% endblock %} +{% block content %} + +
    + +
    +
    +
    +
    + G +
    -
    - - +
    +

    Loading...

    +

    + Owner: Loading... +

    +

    + Loading... +

    +

    Loading description...

    - - +
    - -
    File Name TitleActions - - Actions
    ${p.name} + + // Check if group allows modifications + const groupStatus = window.currentGroupStatus || 'active'; + const canModify = (groupStatus === 'active'); + + let actionsHtml = ''; + if (canModify) { + actionsHtml = ` - ${p.name}${actionsHtml}
    + + + + + + + + + + + +
    + + NameRoleActions
    + + + +
    + + +
    - - - - - - - - - - - -
    NameRoleActions
    - - - + +
    + +
    +
    +
    +
    + +
    +
    -
    +
    Total Documents
    +
    +
    +
    +
    +
    + +
    +
    -
    +
    Storage Used
    +
    +
    +
    +
    +
    + +
    +
    -
    +
    Total Tokens
    +
    +
    +
    +
    +
    + +
    +
    -
    +
    Total Members
    +
    +
    +
    + + +
    +
    +
    +
    Document Activity (Last 30 Days)
    +
    + +
    +
    +
    +
    +
    +
    Storage Usage
    +
    + +
    +
    +
    +
    + +
    +
    +
    +
    Token Usage (Last 30 Days)
    +
    + +
    +
    +
    +
    + + + +
    + - + + + + + + + + + + + +
    Cannot Delete Group Yet {% endblock %} {% block scripts %} - + - + + {% endblock %} \ No newline at end of file diff --git a/application/single_app/templates/manage_public_workspace.html b/application/single_app/templates/manage_public_workspace.html index 1cb2ee33..f6dc98f3 100644 --- a/application/single_app/templates/manage_public_workspace.html +++ b/application/single_app/templates/manage_public_workspace.html @@ -1,76 +1,460 @@ {% extends "base.html" %} {% block title %}Manage Public Workspace – {{ app_settings.app_title }}{% endblock %} -{% block content %} -
    -

    Manage Public Workspace

    -
    - - -
    -
    - - - - - +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    +
    +
    + +
    Processing...
    +
    +
    +
    +
    + + + +
    + + + + + + + + + + +{% endblock %} \ No newline at end of file diff --git a/application/single_app/templates/group_workspaces.html b/application/single_app/templates/group_workspaces.html index e88f435f..035aee68 100644 --- a/application/single_app/templates/group_workspaces.html +++ b/application/single_app/templates/group_workspaces.html @@ -145,6 +145,28 @@ #group-dropdown-button { text-align: left; } + + /* Group status badges */ + .group-status-badge { + display: inline-block; + padding: 0.2em 0.5em; + font-size: 0.75em; + font-weight: 600; + margin-left: 0.5rem; + border-radius: 0.25rem; + } + .group-status-locked { + background-color: #ffc107; + color: #000; + } + .group-status-upload-disabled { + background-color: #17a2b8; + color: #fff; + } + .group-status-inactive { + background-color: #dc3545; + color: #fff; + } {% endblock %} {% block content %}
    @@ -189,6 +211,11 @@

    Group Workspace

    + +
    + +
    + @@ -377,6 +436,26 @@
    Group Documents
    + + + @@ -385,17 +464,7 @@
    Group Documents
    - + @@ -560,6 +629,106 @@
    Group Prompts
    + + {% if settings.enable_semantic_kernel and settings.allow_group_agents %} + +
    +
    +
    +
    Group Agents
    + +
    +
    + You do not have permission to manage group agents. +
    +
    + +
    +
    File Name TitleActions - - Actions
    + + + + + + + + + + + + +
    Display NameDescriptionActions
    + + Select a group to load agents. +
    + +
    + + + {% endif %} + + {% if settings.enable_semantic_kernel and settings.allow_group_plugins %} + +
    +
    + You do not have permission to manage group actions. +
    +
    + +
    + + {% endif %} @@ -729,6 +898,13 @@ {% endblock %} {% block scripts %} - + - + + {% endblock %} \ No newline at end of file diff --git a/application/single_app/templates/manage_public_workspace.html b/application/single_app/templates/manage_public_workspace.html index 1cb2ee33..f6dc98f3 100644 --- a/application/single_app/templates/manage_public_workspace.html +++ b/application/single_app/templates/manage_public_workspace.html @@ -1,76 +1,460 @@ {% extends "base.html" %} {% block title %}Manage Public Workspace – {{ app_settings.app_title }}{% endblock %} -{% block content %} -
    -

    Manage Public Workspace

    -
    - - -
    -
    - - - - -