From 1aaa75f468b1237c7bcf3a8a1944d22a54a1fb1b Mon Sep 17 00:00:00 2001 From: Terry J Hawkins Date: Mon, 24 Feb 2025 16:10:02 -0600 Subject: [PATCH] Updated mergeRows in EventDataTable and AttributeDataTable. (#268) 1. Want to ensure the importRows code path is only triggered by the initial data load 2. Changed criteria to look at the number of events instead of the number of eventData or AttributeData rows --- src/org/labkey/snd/query/AttributeDataTable.java | 12 ++++++++++-- src/org/labkey/snd/query/EventDataTable.java | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/org/labkey/snd/query/AttributeDataTable.java b/src/org/labkey/snd/query/AttributeDataTable.java index 03fc43c..3d0ffaa 100644 --- a/src/org/labkey/snd/query/AttributeDataTable.java +++ b/src/org/labkey/snd/query/AttributeDataTable.java @@ -361,11 +361,19 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B Logger log = SNDManager.getLogger(configParameters, AttributeDataTable.class); List> data = getMutableData(rows, getDataIteratorContext(errors, InsertOption.MERGE, configParameters)); + // Large merge triggers importRows path - if (data.size() > SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { data.clear(); - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); } log.info("Merging rows."); diff --git a/src/org/labkey/snd/query/EventDataTable.java b/src/org/labkey/snd/query/EventDataTable.java index 3030250..5345dab 100644 --- a/src/org/labkey/snd/query/EventDataTable.java +++ b/src/org/labkey/snd/query/EventDataTable.java @@ -141,10 +141,18 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B { return 0; } + // Large merge triggers importRows path - if (data.size() >= SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); }