From ea8fec7e62d7303544b289b1cb0027c40f521dc1 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 18 Dec 2025 16:46:18 +0100 Subject: [PATCH 1/8] SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4b5649d10..54cf42a12 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2.5.1 + 2025.12.18-69441ed0724a72343c6290cd 3.0.23 From f29da7ab138f43a731f09d0c83b985c1967b54bd Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 18 Dec 2025 17:13:02 +0100 Subject: [PATCH 2/8] SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion --- .../java/step/core/timeseries/TimeSeriesCollectionsSettings.java | 1 + 1 file changed, 1 insertion(+) diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index de8d2cf70..d027785f5 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -215,6 +215,7 @@ public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(long m timeSeriesCollectionsSettings.setWeeklyEnabled(false); timeSeriesCollectionsSettings.setMainResolution(mainResolution); timeSeriesCollectionsSettings.setMainFlushInterval(mainFlushInterval); + timeSeriesCollectionsSettings.setFlushSeriesQueueSize(20000); return timeSeriesCollectionsSettings; } From e96f054789c625cb620f8cbc404e7c4979617c29 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 19 Dec 2025 10:04:37 +0100 Subject: [PATCH 3/8] SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 54cf42a12..3d1d911cf 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.18-69441ed0724a72343c6290cd + 2025.12.19-694510aa724a72343c4451d7 3.0.23 From 303f14ab9ba585c72ec49f346cebd8d0743c1bf7 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 19 Dec 2025 11:14:45 +0100 Subject: [PATCH 4/8] SED-4451 adapting TimeSeriesExecutionPlugin --- .../step/plugins/timeseries/TimeSeriesExecutionPlugin.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java index e5972f323..9ee61212b 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java @@ -59,7 +59,10 @@ public void initializeExecutionContext(ExecutionEngineContext executionEngineCon super.initializeExecutionContext(executionEngineContext, executionContext); TimeSeriesIngestionPipeline mainIngestionPipeline = timeSeries.getIngestionPipeline(); TreeMap additionalAttributes = executionContext.getObjectEnricher().getAdditionalAttributes(); - TimeSeriesIngestionPipeline ingestionPipeline = new TimeSeriesIngestionPipeline(null, new TimeSeriesIngestionPipelineSettings()) { + //Crete a wrapper of the ingestion pipeline to automatically enrich data with execution attributes + //This approach is quite error-prone and should be refactored + TimeSeriesIngestionPipeline ingestionPipeline = new TimeSeriesIngestionPipeline(null, + new TimeSeriesIngestionPipelineSettings().setResolution(mainIngestionPipeline.getResolution())) { @Override public void ingestPoint(Map attributes, long timestamp, long value) { attributes.putAll(additionalAttributes); From c5171dc5956658ec6b8f1e645e38e599a3283d6a Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 5 Jan 2026 16:52:20 +0100 Subject: [PATCH 5/8] SED-4451 index optimizations and other perf fixes --- .../measurements/MeasurementPlugin.java | 3 +- .../measurements/raw/MeasurementAccessor.java | 4 ++ .../plugins/timeseries/MetricsConstants.java | 22 ++++++++- .../TimeSeriesControllerPlugin.java | 19 ++++++-- .../plugins/timeseries/TimeSeriesService.java | 26 ++++++---- .../timeseries/api/FetchBucketsRequest.java | 9 ++++ .../core/deployment/AbstractStepServices.java | 5 ++ .../tasks/MigrationManagerTasksPlugin.java | 1 + .../tasks/V29_2_TimeSeriesNewIndexes.java | 48 +++++++++++++++++++ .../src/main/java/step/core/Constants.java | 2 +- .../aggregated/ReportNodeTimeSeries.java | 13 +++++ 11 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java diff --git a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java index 11110a0b6..2dd2eff31 100644 --- a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java +++ b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java @@ -56,6 +56,7 @@ public class MeasurementPlugin extends AbstractExecutionEnginePlugin { public static final String SCHEDULE = "schedule"; public static final String TEST_CASE = "testcase"; public static final String EXECUTION_DESCRIPTION = "execution"; + public static final String PROJECT = "project"; public static final String CTX_SCHEDULER_TASK_ID = "$schedulerTaskId"; public static final String CTX_SCHEDULE_NAME = "$scheduleName"; public static final String CTX_EXECUTION_DESCRIPTION = "$executionDescription"; @@ -66,7 +67,7 @@ public class MeasurementPlugin extends AbstractExecutionEnginePlugin { // These are used by the MeasurementControllerPlugin to "reconstruct" measures from measurements, and indicate the // "internal" fields which should NOT be added to the measure data field. Keep this in sync with the fields defined above. - static final Set MEASURE_NOT_DATA_KEYS = Set.of("_id", "project", "projectName", ATTRIBUTE_EXECUTION_ID, RN_ID, + static final Set MEASURE_NOT_DATA_KEYS = Set.of("_id", PROJECT, "projectName", ATTRIBUTE_EXECUTION_ID, RN_ID, ORIGIN, RN_STATUS, PLAN_ID, PLAN, AGENT_URL, TASK_ID, SCHEDULE, TEST_CASE, EXECUTION_DESCRIPTION); // Same use, but for defining which fields SHOULD be directly copied to the top-level fields of a measure. static final Set MEASURE_FIELDS = Set.of(NAME, BEGIN, VALUE, STATUS); diff --git a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java index 6928a4fe7..3870ea900 100644 --- a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java +++ b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java @@ -97,4 +97,8 @@ private static Document getRawMeasurement(Map o) { document.remove(MeasurementPlugin.PLAN); return document; } + + public Collection getCollection() { + return coll; + } } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java index e7b885334..c524259d7 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java @@ -5,10 +5,28 @@ import java.util.Arrays; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class MetricsConstants { - - public static final MetricAttribute STATUS_ATTRIBUTE = new MetricAttribute() + + public static String getAllAttributeNames() { + return Stream.of( + STATUS_ATTRIBUTE, + TYPE_ATRIBUTE, + TASK_ATTRIBUTE, + EXECUTION_ATTRIBUTE, + PLAN_ATTRIBUTE, + NAME_ATTRIBUTE, + ERROR_CODE_ATTRIBUTE, + EXECUTION_BOOLEAN_RESULT, + EXECUTION_RESULT + ) + .map(MetricAttribute::getName) + .collect(Collectors.joining(",")); + } + + public static final MetricAttribute STATUS_ATTRIBUTE = new MetricAttribute() .setName("rnStatus") .setType(MetricAttributeType.TEXT) .setMetadata(Map.of("knownValues", Arrays.asList("PASSED", "FAILED", "TECHNICAL_ERROR", "INTERRUPTED"))) diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java index abcd97250..452e308ca 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java @@ -35,6 +35,8 @@ import java.util.*; +import static step.core.timeseries.TimeSeriesConstants.ATTRIBUTES_PREFIX; +import static step.core.timeseries.TimeSeriesConstants.TIMESTAMP_ATTRIBUTE; import static step.plugins.measurements.MeasurementPlugin.ATTRIBUTE_EXECUTION_ID; import static step.plugins.timeseries.MetricsConstants.*; import static step.plugins.timeseries.TimeSeriesExecutionPlugin.*; @@ -45,7 +47,9 @@ public class TimeSeriesControllerPlugin extends AbstractControllerPlugin { private static final Logger logger = LoggerFactory.getLogger(TimeSeriesControllerPlugin.class); public static final String TIME_SERIES_MAIN_COLLECTION = "timeseries"; public static final String TIME_SERIES_ATTRIBUTES_PROPERTY = "timeseries.attributes"; - public static final String TIME_SERIES_ATTRIBUTES_DEFAULT = EXECUTION_ID + "," + TASK_ID + "," + PLAN_ID + ",metricType,origin,name,rnStatus,project,type"; + //We should review the usage of the following property default value, it is technically possible to set the value in properties: not sure if that really work. + //This is used to determine if we fall back to RAW measurement when we filter or group by fields that are not supported by time-series and when reingesting timeseries from RAW measurements + public static final String TIME_SERIES_ATTRIBUTES_DEFAULT = MetricsConstants.getAllAttributeNames() + ",metricType,origin,project"; // Following properties are used by the UI. In the future we could remove the prefix 'plugins.' to align with other properties public static final String PARAM_KEY_EXECUTION_DASHBOARD_ID = "plugins.timeseries.execution.dashboard.id"; @@ -123,10 +127,17 @@ public ExecutionEnginePlugin getExecutionEnginePlugin() { public void initializeData(GlobalContext context) throws Exception { super.initializeData(context); timeSeries.createIndexes(new LinkedHashSet<>(List.of(new IndexField(ATTRIBUTE_EXECUTION_ID, Order.ASC, String.class)))); + IndexField metricTypeIndexField = new IndexField(ATTRIBUTES_PREFIX + METRIC_TYPE, Order.ASC, String.class); + IndexField beginIndexField = new IndexField(TIMESTAMP_ATTRIBUTE, Order.ASC, Long.class); timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( - new IndexField("attributes.taskId", Order.ASC, String.class), - new IndexField("attributes.metricType", Order.ASC, String.class), - new IndexField("begin", Order.ASC, String.class) + new IndexField(ATTRIBUTES_PREFIX + TASK_ATTRIBUTE.getName(), Order.ASC, String.class), + metricTypeIndexField, + beginIndexField + ))); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + PLAN_ATTRIBUTE.getName(), Order.ASC, String.class), + metricTypeIndexField, + beginIndexField ))); List metrics = createOrUpdateMetrics(context.require(MetricTypeAccessor.class)); diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java index 72c2b3a3f..010a009ce 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java @@ -101,14 +101,19 @@ public TimeSeriesAPIResponse getMeasurements(@NotNull FetchBucketsRequest reques } private void enrichRequest(FetchBucketsRequest request) { - request.setOqlFilter(enrichOqlFilter(request.getOqlFilter())); + request.setOqlFilter(enrichOqlFilter(request.getOqlFilter(), request.isIncludeGlobalEntities())); if (request.getMaxNumberOfSeries() <= 0) { request.setMaxNumberOfSeries(maxNumberOfSeries); } } - private String enrichOqlFilter(String oqlFilter) { - String additionalOqlFilter = getObjectFilter().getOQLFilter(); + private String enrichOqlFilter(String oqlFilter, boolean includeGlobalEntities) { + String additionalOqlFilter = ""; + if (includeGlobalEntities) { + additionalOqlFilter = getObjectFilter().getOQLFilter(); + } else { + additionalOqlFilter = getRestrictedObjectFilter().getOQLFilter(); + } if (StringUtils.isNotEmpty(additionalOqlFilter)) { return (StringUtils.isNotEmpty(oqlFilter)) ? oqlFilter + " and (" + additionalOqlFilter + ")" : @@ -155,8 +160,9 @@ public boolean timeSeriesIsBuilt(@PathParam("executionId") String executionId) { @Path("/measurements-fields") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFilter) { - oqlFilter = enrichOqlFilter(oqlFilter); + public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFilter, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities) { + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getMeasurementsAttributes(oqlFilter); } @@ -168,9 +174,10 @@ public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFil public List discoverMeasurements( @QueryParam("filter") String oqlFilter, @QueryParam("limit") int limit, - @QueryParam("skip") int skip + @QueryParam("skip") int skip, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities ) { - oqlFilter = enrichOqlFilter(oqlFilter); + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getRawMeasurements(oqlFilter, skip, limit); } @@ -179,8 +186,9 @@ public List discoverMeasurements( @Path("/raw-measurements/stats") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public MeasurementsStats getRawMeasurementsStats(@QueryParam("filter") String oqlFilter) { - oqlFilter = enrichOqlFilter(oqlFilter); + public MeasurementsStats getRawMeasurementsStats(@QueryParam("filter") String oqlFilter, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities) { + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getRawMeasurementsStats(oqlFilter); } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java index ac04c8763..a22e6e058 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java @@ -14,6 +14,7 @@ public class FetchBucketsRequest { private Set collectAttributeKeys; private int collectAttributesValuesLimit; private int maxNumberOfSeries; + private boolean includeGlobalEntities; //If not set default is false public Long getStart() { return start; @@ -104,4 +105,12 @@ public int getMaxNumberOfSeries() { public void setMaxNumberOfSeries(int maxNumberOfSeries) { this.maxNumberOfSeries = maxNumberOfSeries; } + + public boolean isIncludeGlobalEntities() { + return includeGlobalEntities; + } + + public void setIncludeGlobalEntities(boolean includeGlobalEntities) { + this.includeGlobalEntities = includeGlobalEntities; + } } diff --git a/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java b/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java index 84869aa23..3d59feaa4 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java +++ b/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java @@ -79,6 +79,11 @@ protected ObjectFilter getObjectFilter() { return objectHookRegistry.getObjectFilter(getSession()); } + protected ObjectFilter getRestrictedObjectFilter() { + Session session = new RestrictedScopeSession(getSession()); + return objectHookRegistry.getObjectFilter(session); + } + protected ObjectPredicate getObjectPredicate(){ return objectHookRegistry.getObjectPredicate(getSession()); } diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java index 1a591be5a..44760db29 100644 --- a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java @@ -54,6 +54,7 @@ public void serverStart(GlobalContext context) throws Exception { migrationManager.register(V27_4_DropResolvedPlanNodesIndexForPSQLMigrationTask.class); migrationManager.register(V28_0_FixEmptyDefaultMavenSettingsMigrationTask.class); migrationManager.register(V29_0_UpdateAutomationPackageModel.class); + migrationManager.register(V29_2_TimeSeriesNewIndexes.class); } @Override diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java new file mode 100644 index 000000000..145755459 --- /dev/null +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java @@ -0,0 +1,48 @@ +package step.migration.tasks; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.core.Version; + +import step.core.collections.Collection; +import step.core.collections.CollectionFactory; +import step.core.collections.Document; +import step.core.collections.postgresql.PostgreSQLCollection; +import step.migration.MigrationContext; +import step.migration.MigrationTask; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +public class V29_2_TimeSeriesNewIndexes extends MigrationTask { + + + private static final Logger log = LoggerFactory.getLogger(V29_0_UpdateAutomationPackageModel.class); + protected AtomicInteger successCount; + + public V29_2_TimeSeriesNewIndexes(CollectionFactory collectionFactory, MigrationContext migrationContext) { + super(new Version(3,29,2), collectionFactory, migrationContext); + } + + @Override + public void runUpgradeScript() { + // Compound index created for timeseries collection with metricType, taskId and begin where wrongly using a string type for begin, we need to remove them. + // The correct ones are then created automatically in the initializeData hook as for a fresh setup + // This type of the field is only relevant for the PSQL indexes + List suffix = List.of("", "_minute", "_hour", "_day", "_week"); + suffix.forEach(s -> { + String collectionName = "timeseries" + s; + String indexName = "idx_" + collectionName + "_attributes_taskidasc_attributes_metrictypeasc_beginasc"; + Collection collection = collectionFactory.getCollection(collectionName, Document.class); + if (collection instanceof PostgreSQLCollection) { + collection.dropIndex(indexName); + log.info("Time-series index migration - dropped index {}", indexName); + } + }); + } + + @Override + public void runDowngradeScript() { + + } +} diff --git a/step-core/src/main/java/step/core/Constants.java b/step-core/src/main/java/step/core/Constants.java index 3762b32ae..66c025b3a 100644 --- a/step-core/src/main/java/step/core/Constants.java +++ b/step-core/src/main/java/step/core/Constants.java @@ -19,7 +19,7 @@ package step.core; public interface Constants { - String STEP_API_VERSION_STRING = "3.29.1"; + String STEP_API_VERSION_STRING = "3.29.2"; Version STEP_API_VERSION = new Version(STEP_API_VERSION_STRING); String STEP_YAML_SCHEMA_VERSION_STRING = "1.2.0"; diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java index 11358d3ee..7d3293176 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java @@ -11,11 +11,15 @@ import step.core.timeseries.bucket.BucketAttributes; import step.core.timeseries.ingestion.TimeSeriesIngestionPipeline; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static step.core.timeseries.TimeSeriesConstants.ATTRIBUTES_PREFIX; +import static step.core.timeseries.TimeSeriesConstants.TIMESTAMP_ATTRIBUTE; + public class ReportNodeTimeSeries implements AutoCloseable { public static final String CONF_KEY_REPORT_NODE_TIME_SERIES_ENABLED = "execution.engine.reportnodes.timeseries.enabled"; @@ -40,6 +44,15 @@ public ReportNodeTimeSeries(CollectionFactory collectionFactory, TimeSeriesColle timeSeries = new TimeSeriesBuilder().registerCollections(timeSeriesCollections).build(); ingestionPipeline = timeSeries.getIngestionPipeline(); timeSeries.createIndexes(Set.of(new IndexField(EXECUTION_ID, Order.ASC, String.class))); + IndexField beginIndexField = new IndexField(TIMESTAMP_ATTRIBUTE, Order.ASC, Long.class); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + "taskId", Order.ASC, String.class), + beginIndexField + ))); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + "planId", Order.ASC, String.class), + beginIndexField + ))); this.ingestionEnabled = ingestionEnabled; } From 135e0dc427d149562b643582287124a8552450cd Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 5 Jan 2026 17:09:57 +0100 Subject: [PATCH 6/8] SED-4451 bumping framework --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3d1d911cf..9db7832a5 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.19-694510aa724a72343c4451d7 + 2026.1.5-695bdfc4724a72343c993be9 3.0.23 From d5cbfec9b4249c63da7fba5e08d7903108f30670 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 6 Jan 2026 13:42:51 +0100 Subject: [PATCH 7/8] SED-4451 PR Feedbacks --- .../measurements/raw/MeasurementAccessor.java | 10 +++--- .../plugins/timeseries/MetricsConstants.java | 35 +++++++++++-------- .../tasks/V29_2_TimeSeriesNewIndexes.java | 4 +-- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java index 3870ea900..f6fb513b9 100644 --- a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java +++ b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java @@ -4,13 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import step.core.collections.*; +import step.core.collections.Collection; import step.core.entities.EntityManager; import step.plugins.measurements.MeasurementPlugin; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Stream; public class MeasurementAccessor { @@ -98,7 +96,7 @@ private static Document getRawMeasurement(Map o) { return document; } - public Collection getCollection() { - return coll; + public void createOrUpdateCompoundIndex(LinkedHashSet indexFields) { + coll.createOrUpdateCompoundIndex(indexFields); } } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java index c524259d7..1387db8c1 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java @@ -10,21 +10,9 @@ public class MetricsConstants { - public static String getAllAttributeNames() { - return Stream.of( - STATUS_ATTRIBUTE, - TYPE_ATRIBUTE, - TASK_ATTRIBUTE, - EXECUTION_ATTRIBUTE, - PLAN_ATTRIBUTE, - NAME_ATTRIBUTE, - ERROR_CODE_ATTRIBUTE, - EXECUTION_BOOLEAN_RESULT, - EXECUTION_RESULT - ) - .map(MetricAttribute::getName) - .collect(Collectors.joining(",")); - } + // The static method getAllAttributeNames is placed at the end of this class (to make sure all constant are initialized). + // It is used to register all metric as supported timeseries fields. + // WARNING: Do not forget to update this method when adding new constants public static final MetricAttribute STATUS_ATTRIBUTE = new MetricAttribute() .setName("rnStatus") @@ -67,4 +55,21 @@ public static String getAllAttributeNames() { .setName("result") .setType(MetricAttributeType.TEXT) .setDisplayName("Result"); + + + public static String getAllAttributeNames() { + return Stream.of( + STATUS_ATTRIBUTE, + TYPE_ATRIBUTE, + TASK_ATTRIBUTE, + EXECUTION_ATTRIBUTE, + PLAN_ATTRIBUTE, + NAME_ATTRIBUTE, + ERROR_CODE_ATTRIBUTE, + EXECUTION_BOOLEAN_RESULT, + EXECUTION_RESULT + ) + .map(MetricAttribute::getName) + .collect(Collectors.joining(",")); + } } diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java index 145755459..f6ece6125 100644 --- a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java @@ -12,13 +12,11 @@ import step.migration.MigrationTask; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; public class V29_2_TimeSeriesNewIndexes extends MigrationTask { - private static final Logger log = LoggerFactory.getLogger(V29_0_UpdateAutomationPackageModel.class); - protected AtomicInteger successCount; + private static final Logger log = LoggerFactory.getLogger(V29_2_TimeSeriesNewIndexes.class); public V29_2_TimeSeriesNewIndexes(CollectionFactory collectionFactory, MigrationContext migrationContext) { super(new Version(3,29,2), collectionFactory, migrationContext); From db7a522bb7431510b926b8f98d75f42e4a67bcdd Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 16 Jan 2026 14:39:31 +0100 Subject: [PATCH 8/8] SED-4451 adding missing TS index and fixing default resolution --- .../step/plugins/timeseries/TimeSeriesControllerPlugin.java | 4 ++++ .../step/core/timeseries/TimeSeriesCollectionsBuilder.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java index 452e308ca..dcd8cf971 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java @@ -129,6 +129,10 @@ public void initializeData(GlobalContext context) throws Exception { timeSeries.createIndexes(new LinkedHashSet<>(List.of(new IndexField(ATTRIBUTE_EXECUTION_ID, Order.ASC, String.class)))); IndexField metricTypeIndexField = new IndexField(ATTRIBUTES_PREFIX + METRIC_TYPE, Order.ASC, String.class); IndexField beginIndexField = new IndexField(TIMESTAMP_ATTRIBUTE, Order.ASC, Long.class); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + metricTypeIndexField, + beginIndexField + ))); timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( new IndexField(ATTRIBUTES_PREFIX + TASK_ATTRIBUTE.getName(), Order.ASC, String.class), metricTypeIndexField, diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 5eb8215e0..93e333b8a 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -48,7 +48,7 @@ public List getTimeSeriesCollections(String mainCollection List enabledCollections = new ArrayList<>(); int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); - addIfEnabled(enabledCollections, mainCollectionName, Duration.ofSeconds(1), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); + addIfEnabled(enabledCollections, mainCollectionName, Duration.ofMillis(collectionsSettings.getMainResolution()), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_PER_MINUTE, Duration.ofMinutes(1), collectionsSettings.getPerMinuteFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, collectionsSettings.isPerMinuteEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_HOURLY, Duration.ofHours(1), collectionsSettings.getHourlyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isHourlyEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_DAILY, Duration.ofDays(1), collectionsSettings.getDailyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isDailyEnabled());