From 26c460e2785a1886c0890831998a88845c42bf67 Mon Sep 17 00:00:00 2001 From: Terry J Hawkins Date: Thu, 20 Feb 2025 15:45:42 -0600 Subject: [PATCH] Updated mergeRows in EventDataTable and AttributeDataTable. 1. Want to ensure the importRows code path is only triggered by the initial data load 2. Changed criteria to look at the number of events instead of the number of eventData or AttributeData rows --- src/org/labkey/snd/query/AttributeDataTable.java | 12 ++++++++++-- src/org/labkey/snd/query/EventDataTable.java | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/org/labkey/snd/query/AttributeDataTable.java b/src/org/labkey/snd/query/AttributeDataTable.java index 03fc43cd..3d0ffaa6 100644 --- a/src/org/labkey/snd/query/AttributeDataTable.java +++ b/src/org/labkey/snd/query/AttributeDataTable.java @@ -361,11 +361,19 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B Logger log = SNDManager.getLogger(configParameters, AttributeDataTable.class); List> data = getMutableData(rows, getDataIteratorContext(errors, InsertOption.MERGE, configParameters)); + // Large merge triggers importRows path - if (data.size() > SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { data.clear(); - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); } log.info("Merging rows."); diff --git a/src/org/labkey/snd/query/EventDataTable.java b/src/org/labkey/snd/query/EventDataTable.java index 30302500..5345dabc 100644 --- a/src/org/labkey/snd/query/EventDataTable.java +++ b/src/org/labkey/snd/query/EventDataTable.java @@ -141,10 +141,18 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B { return 0; } + // Large merge triggers importRows path - if (data.size() >= SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); }