diff --git a/src/org/labkey/snd/query/AttributeDataTable.java b/src/org/labkey/snd/query/AttributeDataTable.java index 03fc43cd..3d0ffaa6 100644 --- a/src/org/labkey/snd/query/AttributeDataTable.java +++ b/src/org/labkey/snd/query/AttributeDataTable.java @@ -361,11 +361,19 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B Logger log = SNDManager.getLogger(configParameters, AttributeDataTable.class); List> data = getMutableData(rows, getDataIteratorContext(errors, InsertOption.MERGE, configParameters)); + // Large merge triggers importRows path - if (data.size() > SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { data.clear(); - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); } log.info("Merging rows."); diff --git a/src/org/labkey/snd/query/EventDataTable.java b/src/org/labkey/snd/query/EventDataTable.java index 30302500..5345dabc 100644 --- a/src/org/labkey/snd/query/EventDataTable.java +++ b/src/org/labkey/snd/query/EventDataTable.java @@ -141,10 +141,18 @@ public int mergeRows(User user, Container container, DataIteratorBuilder rows, B { return 0; } + // Large merge triggers importRows path - if (data.size() >= SNDManager.MAX_MERGE_ROWS) + Set distinctEventIds = new HashSet<>(); + for (Map map : data) { + distinctEventIds.add(map.get("eventId")); + } + + int max_merge_rows = SNDManager.MAX_MERGE_ROWS * 10; //ensure this code path is only triggered by the initial data load + + if (distinctEventIds.size() >= max_merge_rows) { - log.info("More than " + SNDManager.MAX_MERGE_ROWS + " rows. using importRows method."); + log.info("More than " + max_merge_rows + " rows. using importRows method."); return importRows(user, container, rows, errors, configParameters, extraScriptContext); }