diff --git a/Cargo.lock b/Cargo.lock index bc715500ce..a238194b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3798,6 +3798,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 2.0.12", +] + [[package]] name = "prost" version = "0.13.5" @@ -3830,6 +3845,26 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "quanta" version = "0.12.6" @@ -4658,6 +4693,7 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "prometheus", "rivet-config", "rivet-env", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 9b0a642a0c..0ded9e27f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ papaya = "0.2.1" pest_derive = "2.7" portpicker = "0.1" prettyplease = "0.2" +prometheus = "0.14" quote = "1.0" rand = "0.8" regex = "1.4" diff --git a/engine/docker/dev-host/docker-compose.yml b/engine/docker/dev-host/docker-compose.yml index 06b56e01bf..f51586ef6b 100644 --- a/engine/docker/dev-host/docker-compose.yml +++ b/engine/docker/dev-host/docker-compose.yml @@ -32,6 +32,29 @@ services: timeout: 10s retries: 10 network_mode: host + prometheus: + restart: unless-stopped + image: prom/prometheus:latest + volumes: + - prometheus-data:/prometheus + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-remote-write-receiver' + healthcheck: + test: + - CMD + - wget + - '--spider' + - '-q' + - http://127.0.0.1:9090/-/healthy + interval: 5s + timeout: 3s + retries: 3 + network_mode: host grafana: image: grafana/grafana:11.5.2 volumes: @@ -44,6 +67,8 @@ services: depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy network_mode: host postgres: restart: unless-stopped @@ -72,7 +97,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres: @@ -102,27 +128,19 @@ services: vector-server: condition: service_started network_mode: host - otel-collector-server: + otel-collector: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - ./otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy - network_mode: host - otel-collector-client: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - ./otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server: - condition: service_started + prometheus: + condition: service_healthy network_mode: host rivet-engine: build: @@ -142,7 +160,7 @@ services: depends_on: vector-client: condition: service_started - otel-collector-client: + otel-collector: condition: service_started postgres: condition: service_healthy @@ -184,6 +202,7 @@ networks: driver: bridge volumes: clickhouse-data: null + prometheus-data: null grafana-data: null postgres-data: null vector-server-data-default: null diff --git a/engine/docker/dev-host/grafana/dashboards/api.json b/engine/docker/dev-host/grafana/dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/dev-host/grafana/dashboards/api.json +++ b/engine/docker/dev-host/grafana/dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/cache.json b/engine/docker/dev-host/grafana/dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/dev-host/grafana/dashboards/cache.json +++ b/engine/docker/dev-host/grafana/dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/foundationdb.json b/engine/docker/dev-host/grafana/dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/dev-host/grafana/dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/futures.json b/engine/docker/dev-host/grafana/dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/dev-host/grafana/dashboards/futures.json +++ b/engine/docker/dev-host/grafana/dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/gasoline.json b/engine/docker/dev-host/grafana/dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/dev-host/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-host/grafana/dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/guard.json b/engine/docker/dev-host/grafana/dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/dev-host/grafana/dashboards/guard.json +++ b/engine/docker/dev-host/grafana/dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/operation.json b/engine/docker/dev-host/grafana/dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/dev-host/grafana/dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/tokio.json b/engine/docker/dev-host/grafana/dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/dev-host/grafana/dashboards/tokio.json +++ b/engine/docker/dev-host/grafana/dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/provisioning/datasources/datasources.yaml b/engine/docker/dev-host/grafana/provisioning/datasources/datasources.yaml index 3cc84bfbfb..423b7816f4 100644 --- a/engine/docker/dev-host/grafana/provisioning/datasources/datasources.yaml +++ b/engine/docker/dev-host/grafana/provisioning/datasources/datasources.yaml @@ -1,5 +1,13 @@ apiVersion: 1 datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://127.0.0.1:9090 + isDefault: true + jsonData: + timeInterval: 15s - name: ClickHouse uid: clickhouse type: grafana-clickhouse-datasource diff --git a/engine/docker/dev-host/otel-collector-client/config.yaml b/engine/docker/dev-host/otel-collector-client/config.yaml deleted file mode 100644 index 444a77a7af..0000000000 --- a/engine/docker/dev-host/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: 127.0.0.1:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-host/otel-collector-server/config.yaml b/engine/docker/dev-host/otel-collector/config.yaml similarity index 70% rename from engine/docker/dev-host/otel-collector-server/config.yaml rename to engine/docker/dev-host/otel-collector/config.yaml index c4414738d1..0480fea239 100644 --- a/engine/docker/dev-host/otel-collector-server/config.yaml +++ b/engine/docker/dev-host/otel-collector/config.yaml @@ -3,6 +3,16 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-default-0 + scrape_interval: 15s + static_configs: + - targets: + - host.docker.internal:6430 processors: resource: attributes: @@ -28,22 +38,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://127.0.0.1:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +69,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-host/prometheus/prometheus.yml b/engine/docker/dev-host/prometheus/prometheus.yml new file mode 100644 index 0000000000..edb0f179de --- /dev/null +++ b/engine/docker/dev-host/prometheus/prometheus.yml @@ -0,0 +1,4 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: [] diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/api.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/api.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/foundationdb.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/provisioning/datasources/datasources.yaml b/engine/docker/dev-multidc-multinode/core/grafana/provisioning/datasources/datasources.yaml index f66d3bf036..8c4e4e6560 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/provisioning/datasources/datasources.yaml +++ b/engine/docker/dev-multidc-multinode/core/grafana/provisioning/datasources/datasources.yaml @@ -1,5 +1,13 @@ apiVersion: 1 datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s - name: ClickHouse uid: clickhouse type: grafana-clickhouse-datasource diff --git a/engine/docker/dev-multidc-multinode/core/prometheus/prometheus.yml b/engine/docker/dev-multidc-multinode/core/prometheus/prometheus.yml new file mode 100644 index 0000000000..edb0f179de --- /dev/null +++ b/engine/docker/dev-multidc-multinode/core/prometheus/prometheus.yml @@ -0,0 +1,4 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: [] diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-client/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-client/config.yaml deleted file mode 100644 index f3077ceb1c..0000000000 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-a:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-server/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector/config.yaml similarity index 58% rename from engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-server/config.yaml rename to engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector/config.yaml index 64004c2dc7..dbce48db53 100644 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc-multinode/datacenters/dc-a/otel-collector/config.yaml @@ -3,6 +3,26 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-a-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-a-0:6430 + - job_name: rivet-engine-dc-a-1 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-a-1:6430 + - job_name: rivet-engine-dc-a-2 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-a-2:6430 processors: resource: attributes: @@ -28,22 +48,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +79,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-client/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-client/config.yaml deleted file mode 100644 index a187be889a..0000000000 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-b:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc/datacenters/dc-b/otel-collector-server/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector/config.yaml similarity index 58% rename from engine/docker/dev-multidc/datacenters/dc-b/otel-collector-server/config.yaml rename to engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector/config.yaml index 7dae8a71d0..f219601326 100644 --- a/engine/docker/dev-multidc/datacenters/dc-b/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector/config.yaml @@ -3,6 +3,26 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-b-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-b-0:6430 + - job_name: rivet-engine-dc-b-1 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-b-1:6430 + - job_name: rivet-engine-dc-b-2 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-b-2:6430 processors: resource: attributes: @@ -28,22 +48,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +79,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-client/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-client/config.yaml deleted file mode 100644 index ca6a47e900..0000000000 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-c:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-server/config.yaml b/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector/config.yaml similarity index 58% rename from engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-server/config.yaml rename to engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector/config.yaml index a4fd830662..1beb8d7f01 100644 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc-multinode/datacenters/dc-c/otel-collector/config.yaml @@ -3,6 +3,26 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-c-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-c-0:6430 + - job_name: rivet-engine-dc-c-1 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-c-1:6430 + - job_name: rivet-engine-dc-c-2 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-c-2:6430 processors: resource: attributes: @@ -28,22 +48,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +79,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc-multinode/docker-compose.yml b/engine/docker/dev-multidc-multinode/docker-compose.yml index 7fbbc3f111..9677c5c437 100644 --- a/engine/docker/dev-multidc-multinode/docker-compose.yml +++ b/engine/docker/dev-multidc-multinode/docker-compose.yml @@ -36,6 +36,35 @@ services: interval: 2s timeout: 10s retries: 10 + prometheus: + restart: unless-stopped + image: prom/prometheus:latest + volumes: + - prometheus-data:/prometheus + - ./core/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-remote-write-receiver' + ports: + - '9090:9090' + networks: + - rivet-network-dc-a-to-core + - rivet-network-dc-b-to-core + - rivet-network-dc-c-to-core + - rivet-core-network + healthcheck: + test: + - CMD + - wget + - '--spider' + - '-q' + - http://127.0.0.1:9090/-/healthy + interval: 5s + timeout: 3s + retries: 3 grafana: image: grafana/grafana:11.5.2 volumes: @@ -52,6 +81,8 @@ services: depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy postgres-dc-a: restart: unless-stopped image: postgres:17-alpine @@ -83,7 +114,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-a: @@ -122,35 +154,24 @@ services: depends_on: vector-server-dc-a: condition: service_started - otel-collector-server-dc-a: + otel-collector-dc-a: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-a/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-a/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-a - rivet-network-dc-a-to-core ports: - '4317:4317' - otel-collector-client-dc-a: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-a/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-a: - condition: service_started - networks: - - rivet-network-dc-a rivet-engine-dc-a-0: build: context: ../../.. @@ -164,12 +185,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-a:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: vector-client-dc-a: condition: service_started - otel-collector-client-dc-a: + otel-collector-dc-a: condition: service_started postgres-dc-a: condition: service_healthy @@ -207,12 +228,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-a:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: vector-client-dc-a: condition: service_started - otel-collector-client-dc-a: + otel-collector-dc-a: condition: service_started postgres-dc-a: condition: service_healthy @@ -248,12 +269,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-a:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: vector-client-dc-a: condition: service_started - otel-collector-client-dc-a: + otel-collector-dc-a: condition: service_started postgres-dc-a: condition: service_healthy @@ -352,7 +373,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-b: @@ -391,35 +413,22 @@ services: depends_on: vector-server-dc-b: condition: service_started - otel-collector-server-dc-b: + otel-collector-dc-b: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-b/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-b/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-b - rivet-network-dc-b-to-core - ports: - - '4317:4317' - otel-collector-client-dc-b: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-b/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-b: - condition: service_started - networks: - - rivet-network-dc-b rivet-engine-dc-b-0: build: context: ../../.. @@ -433,12 +442,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-b:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: vector-client-dc-b: condition: service_started - otel-collector-client-dc-b: + otel-collector-dc-b: condition: service_started postgres-dc-b: condition: service_healthy @@ -474,12 +483,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-b:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: vector-client-dc-b: condition: service_started - otel-collector-client-dc-b: + otel-collector-dc-b: condition: service_started postgres-dc-b: condition: service_healthy @@ -515,12 +524,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-b:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: vector-client-dc-b: condition: service_started - otel-collector-client-dc-b: + otel-collector-dc-b: condition: service_started postgres-dc-b: condition: service_healthy @@ -617,7 +626,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-c: @@ -656,35 +666,22 @@ services: depends_on: vector-server-dc-c: condition: service_started - otel-collector-server-dc-c: + otel-collector-dc-c: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-c/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-c/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-c - rivet-network-dc-c-to-core - ports: - - '4317:4317' - otel-collector-client-dc-c: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-c/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-c: - condition: service_started - networks: - - rivet-network-dc-c rivet-engine-dc-c-0: build: context: ../../.. @@ -698,12 +695,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-c:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: vector-client-dc-c: condition: service_started - otel-collector-client-dc-c: + otel-collector-dc-c: condition: service_started postgres-dc-c: condition: service_healthy @@ -739,12 +736,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-c:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: vector-client-dc-c: condition: service_started - otel-collector-client-dc-c: + otel-collector-dc-c: condition: service_started postgres-dc-c: condition: service_healthy @@ -780,12 +777,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-c:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: vector-client-dc-c: condition: service_started - otel-collector-client-dc-c: + otel-collector-dc-c: condition: service_started postgres-dc-c: condition: service_healthy @@ -876,6 +873,7 @@ networks: driver: bridge volumes: clickhouse-data: null + prometheus-data: null grafana-data: null postgres-data-dc-a: null vector-server-data-dc-a: null diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/api.json b/engine/docker/dev-multidc/core/grafana/dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/api.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/cache.json b/engine/docker/dev-multidc/core/grafana/dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/cache.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/foundationdb.json b/engine/docker/dev-multidc/core/grafana/dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/dev-multidc/core/grafana/dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/futures.json b/engine/docker/dev-multidc/core/grafana/dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/futures.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json b/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/guard.json b/engine/docker/dev-multidc/core/grafana/dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/guard.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/operation.json b/engine/docker/dev-multidc/core/grafana/dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/dev-multidc/core/grafana/dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/tokio.json b/engine/docker/dev-multidc/core/grafana/dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/tokio.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/provisioning/datasources/datasources.yaml b/engine/docker/dev-multidc/core/grafana/provisioning/datasources/datasources.yaml index f66d3bf036..8c4e4e6560 100644 --- a/engine/docker/dev-multidc/core/grafana/provisioning/datasources/datasources.yaml +++ b/engine/docker/dev-multidc/core/grafana/provisioning/datasources/datasources.yaml @@ -1,5 +1,13 @@ apiVersion: 1 datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s - name: ClickHouse uid: clickhouse type: grafana-clickhouse-datasource diff --git a/engine/docker/dev-multidc/core/prometheus/prometheus.yml b/engine/docker/dev-multidc/core/prometheus/prometheus.yml new file mode 100644 index 0000000000..edb0f179de --- /dev/null +++ b/engine/docker/dev-multidc/core/prometheus/prometheus.yml @@ -0,0 +1,4 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: [] diff --git a/engine/docker/dev-multidc/datacenters/dc-a/otel-collector-client/config.yaml b/engine/docker/dev-multidc/datacenters/dc-a/otel-collector-client/config.yaml deleted file mode 100644 index f3077ceb1c..0000000000 --- a/engine/docker/dev-multidc/datacenters/dc-a/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-a:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc/datacenters/dc-a/otel-collector-server/config.yaml b/engine/docker/dev-multidc/datacenters/dc-a/otel-collector/config.yaml similarity index 70% rename from engine/docker/dev-multidc/datacenters/dc-a/otel-collector-server/config.yaml rename to engine/docker/dev-multidc/datacenters/dc-a/otel-collector/config.yaml index 64004c2dc7..9ea927cbee 100644 --- a/engine/docker/dev-multidc/datacenters/dc-a/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc/datacenters/dc-a/otel-collector/config.yaml @@ -3,6 +3,16 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-a-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-a:6430 processors: resource: attributes: @@ -28,22 +38,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +69,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc/datacenters/dc-b/otel-collector-client/config.yaml b/engine/docker/dev-multidc/datacenters/dc-b/otel-collector-client/config.yaml deleted file mode 100644 index a187be889a..0000000000 --- a/engine/docker/dev-multidc/datacenters/dc-b/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-b:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-server/config.yaml b/engine/docker/dev-multidc/datacenters/dc-b/otel-collector/config.yaml similarity index 70% rename from engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-server/config.yaml rename to engine/docker/dev-multidc/datacenters/dc-b/otel-collector/config.yaml index 7dae8a71d0..3154932ed6 100644 --- a/engine/docker/dev-multidc-multinode/datacenters/dc-b/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc/datacenters/dc-b/otel-collector/config.yaml @@ -3,6 +3,16 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-b-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-b:6430 processors: resource: attributes: @@ -28,22 +38,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +69,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc/datacenters/dc-c/otel-collector-client/config.yaml b/engine/docker/dev-multidc/datacenters/dc-c/otel-collector-client/config.yaml deleted file mode 100644 index ca6a47e900..0000000000 --- a/engine/docker/dev-multidc/datacenters/dc-c/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server-dc-c:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multidc/datacenters/dc-c/otel-collector-server/config.yaml b/engine/docker/dev-multidc/datacenters/dc-c/otel-collector/config.yaml similarity index 70% rename from engine/docker/dev-multidc/datacenters/dc-c/otel-collector-server/config.yaml rename to engine/docker/dev-multidc/datacenters/dc-c/otel-collector/config.yaml index a4fd830662..23f6b12562 100644 --- a/engine/docker/dev-multidc/datacenters/dc-c/otel-collector-server/config.yaml +++ b/engine/docker/dev-multidc/datacenters/dc-c/otel-collector/config.yaml @@ -3,6 +3,16 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-dc-c-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-dc-c:6430 processors: resource: attributes: @@ -28,22 +38,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +69,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multidc/docker-compose.yml b/engine/docker/dev-multidc/docker-compose.yml index e71dfa07f2..ce49486879 100644 --- a/engine/docker/dev-multidc/docker-compose.yml +++ b/engine/docker/dev-multidc/docker-compose.yml @@ -36,6 +36,35 @@ services: interval: 2s timeout: 10s retries: 10 + prometheus: + restart: unless-stopped + image: prom/prometheus:latest + volumes: + - prometheus-data:/prometheus + - ./core/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-remote-write-receiver' + ports: + - '9090:9090' + networks: + - rivet-network-dc-a-to-core + - rivet-network-dc-b-to-core + - rivet-network-dc-c-to-core + - rivet-core-network + healthcheck: + test: + - CMD + - wget + - '--spider' + - '-q' + - http://127.0.0.1:9090/-/healthy + interval: 5s + timeout: 3s + retries: 3 grafana: image: grafana/grafana:11.5.2 volumes: @@ -52,6 +81,8 @@ services: depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy postgres-dc-a: restart: unless-stopped image: postgres:17-alpine @@ -83,7 +114,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-a: @@ -121,35 +153,24 @@ services: depends_on: vector-server-dc-a: condition: service_started - otel-collector-server-dc-a: + otel-collector-dc-a: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-a/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-a/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-a - rivet-network-dc-a-to-core ports: - '4317:4317' - otel-collector-client-dc-a: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-a/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-a: - condition: service_started - networks: - - rivet-network-dc-a rivet-engine-dc-a: build: context: ../../.. @@ -163,12 +184,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-a:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: vector-client-dc-a: condition: service_started - otel-collector-client-dc-a: + otel-collector-dc-a: condition: service_started postgres-dc-a: condition: service_healthy @@ -238,7 +259,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-b: @@ -276,35 +298,22 @@ services: depends_on: vector-server-dc-b: condition: service_started - otel-collector-server-dc-b: + otel-collector-dc-b: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-b/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-b/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-b - rivet-network-dc-b-to-core - ports: - - '4317:4317' - otel-collector-client-dc-b: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-b/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-b: - condition: service_started - networks: - - rivet-network-dc-b rivet-engine-dc-b: build: context: ../../.. @@ -318,12 +327,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-b:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: vector-client-dc-b: condition: service_started - otel-collector-client-dc-b: + otel-collector-dc-b: condition: service_started postgres-dc-b: condition: service_healthy @@ -389,7 +398,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres-dc-c: @@ -427,35 +437,22 @@ services: depends_on: vector-server-dc-c: condition: service_started - otel-collector-server-dc-c: + otel-collector-dc-c: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - >- - ./datacenters/dc-c/otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./datacenters/dc-c/otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network-dc-c - rivet-network-dc-c-to-core - ports: - - '4317:4317' - otel-collector-client-dc-c: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - >- - ./datacenters/dc-c/otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server-dc-c: - condition: service_started - networks: - - rivet-network-dc-c rivet-engine-dc-c: build: context: ../../.. @@ -469,12 +466,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client-dc-c:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: vector-client-dc-c: condition: service_started - otel-collector-client-dc-c: + otel-collector-dc-c: condition: service_started postgres-dc-c: condition: service_healthy @@ -534,6 +531,7 @@ networks: driver: bridge volumes: clickhouse-data: null + prometheus-data: null grafana-data: null postgres-data-dc-a: null vector-server-data-dc-a: null diff --git a/engine/docker/dev-multinode/docker-compose.yml b/engine/docker/dev-multinode/docker-compose.yml index 7deffc9227..3538ead70a 100644 --- a/engine/docker/dev-multinode/docker-compose.yml +++ b/engine/docker/dev-multinode/docker-compose.yml @@ -34,6 +34,33 @@ services: interval: 2s timeout: 10s retries: 10 + prometheus: + restart: unless-stopped + image: prom/prometheus:latest + volumes: + - prometheus-data:/prometheus + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-remote-write-receiver' + ports: + - '9090:9090' + networks: + - rivet-network-to-core + - rivet-core-network + healthcheck: + test: + - CMD + - wget + - '--spider' + - '-q' + - http://127.0.0.1:9090/-/healthy + interval: 5s + timeout: 3s + retries: 3 grafana: image: grafana/grafana:11.5.2 volumes: @@ -50,6 +77,8 @@ services: depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy postgres: restart: unless-stopped image: postgres:17-alpine @@ -80,7 +109,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres: @@ -116,33 +146,24 @@ services: depends_on: vector-server: condition: service_started - otel-collector-server: + otel-collector: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - ./otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network - rivet-network-to-core ports: - '4317:4317' - otel-collector-client: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - ./otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server: - condition: service_started - networks: - - rivet-network rivet-engine-0: build: context: ../../.. @@ -156,12 +177,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector:4317 stop_grace_period: 0s depends_on: vector-client: condition: service_started - otel-collector-client: + otel-collector: condition: service_started postgres: condition: service_healthy @@ -196,12 +217,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector:4317 stop_grace_period: 0s depends_on: vector-client: condition: service_started - otel-collector-client: + otel-collector: condition: service_started postgres: condition: service_healthy @@ -234,12 +255,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector:4317 stop_grace_period: 0s depends_on: vector-client: condition: service_started - otel-collector-client: + otel-collector: condition: service_started postgres: condition: service_healthy @@ -317,6 +338,7 @@ networks: driver: bridge volumes: clickhouse-data: null + prometheus-data: null grafana-data: null postgres-data: null vector-server-data-default: null diff --git a/engine/docker/dev-multinode/grafana/dashboards/api.json b/engine/docker/dev-multinode/grafana/dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/api.json +++ b/engine/docker/dev-multinode/grafana/dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/cache.json b/engine/docker/dev-multinode/grafana/dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/cache.json +++ b/engine/docker/dev-multinode/grafana/dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/foundationdb.json b/engine/docker/dev-multinode/grafana/dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/dev-multinode/grafana/dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/futures.json b/engine/docker/dev-multinode/grafana/dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/futures.json +++ b/engine/docker/dev-multinode/grafana/dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/gasoline.json b/engine/docker/dev-multinode/grafana/dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multinode/grafana/dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/guard.json b/engine/docker/dev-multinode/grafana/dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/guard.json +++ b/engine/docker/dev-multinode/grafana/dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/operation.json b/engine/docker/dev-multinode/grafana/dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/dev-multinode/grafana/dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/tokio.json b/engine/docker/dev-multinode/grafana/dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/tokio.json +++ b/engine/docker/dev-multinode/grafana/dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/provisioning/datasources/datasources.yaml b/engine/docker/dev-multinode/grafana/provisioning/datasources/datasources.yaml index f66d3bf036..8c4e4e6560 100644 --- a/engine/docker/dev-multinode/grafana/provisioning/datasources/datasources.yaml +++ b/engine/docker/dev-multinode/grafana/provisioning/datasources/datasources.yaml @@ -1,5 +1,13 @@ apiVersion: 1 datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s - name: ClickHouse uid: clickhouse type: grafana-clickhouse-datasource diff --git a/engine/docker/dev-multinode/otel-collector-client/config.yaml b/engine/docker/dev-multinode/otel-collector-client/config.yaml deleted file mode 100644 index 6e963d5edf..0000000000 --- a/engine/docker/dev-multinode/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev/otel-collector-server/config.yaml b/engine/docker/dev-multinode/otel-collector/config.yaml similarity index 58% rename from engine/docker/dev/otel-collector-server/config.yaml rename to engine/docker/dev-multinode/otel-collector/config.yaml index 7b5fc80c72..2eb31b5343 100644 --- a/engine/docker/dev/otel-collector-server/config.yaml +++ b/engine/docker/dev-multinode/otel-collector/config.yaml @@ -3,6 +3,26 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-default-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-0:6430 + - job_name: rivet-engine-default-1 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-1:6430 + - job_name: rivet-engine-default-2 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine-2:6430 processors: resource: attributes: @@ -28,22 +48,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +79,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev-multinode/prometheus/prometheus.yml b/engine/docker/dev-multinode/prometheus/prometheus.yml new file mode 100644 index 0000000000..edb0f179de --- /dev/null +++ b/engine/docker/dev-multinode/prometheus/prometheus.yml @@ -0,0 +1,4 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: [] diff --git a/engine/docker/dev/docker-compose.yml b/engine/docker/dev/docker-compose.yml index e9447777b6..6e2b881adc 100644 --- a/engine/docker/dev/docker-compose.yml +++ b/engine/docker/dev/docker-compose.yml @@ -34,6 +34,33 @@ services: interval: 2s timeout: 10s retries: 10 + prometheus: + restart: unless-stopped + image: prom/prometheus:latest + volumes: + - prometheus-data:/prometheus + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--web.enable-remote-write-receiver' + ports: + - '9090:9090' + networks: + - rivet-network-to-core + - rivet-core-network + healthcheck: + test: + - CMD + - wget + - '--spider' + - '-q' + - http://127.0.0.1:9090/-/healthy + interval: 5s + timeout: 3s + retries: 3 grafana: image: grafana/grafana:11.5.2 volumes: @@ -50,6 +77,8 @@ services: depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy postgres: restart: unless-stopped image: postgres:17-alpine @@ -80,7 +109,8 @@ services: BUILD_FRONTEND: 'true' platform: linux/amd64 restart: unless-stopped - command: sleep infinity + entrypoint: sleep + command: infinity stop_grace_period: 0s depends_on: postgres: @@ -116,33 +146,24 @@ services: depends_on: vector-server: condition: service_started - otel-collector-server: + otel-collector: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped command: '--config=/etc/otel/config.yaml' volumes: - - ./otel-collector-server/config.yaml:/etc/otel/config.yaml:ro + - ./otel-collector/config.yaml:/etc/otel/config.yaml:ro environment: - CLICKHOUSE_PASSWORD=default depends_on: clickhouse: condition: service_healthy + prometheus: + condition: service_healthy networks: - rivet-network - rivet-network-to-core ports: - '4317:4317' - otel-collector-client: - image: otel/opentelemetry-collector-contrib:latest - restart: unless-stopped - command: '--config=/etc/otel/config.yaml' - volumes: - - ./otel-collector-client/config.yaml:/etc/otel/config.yaml:ro - depends_on: - otel-collector-server: - condition: service_started - networks: - - rivet-network rivet-engine: build: context: ../../.. @@ -156,12 +177,12 @@ services: - RUST_LOG_ANSI_COLOR=1 - RIVET_OTEL_ENABLED=1 - RIVET_OTEL_SAMPLER_RATIO=1 - - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-client:4317 + - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector:4317 stop_grace_period: 0s depends_on: vector-client: condition: service_started - otel-collector-client: + otel-collector: condition: service_started postgres: condition: service_healthy @@ -211,6 +232,7 @@ networks: driver: bridge volumes: clickhouse-data: null + prometheus-data: null grafana-data: null postgres-data: null vector-server-data-default: null diff --git a/engine/docker/dev/grafana/dashboards/api.json b/engine/docker/dev/grafana/dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/dev/grafana/dashboards/api.json +++ b/engine/docker/dev/grafana/dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/cache.json b/engine/docker/dev/grafana/dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/dev/grafana/dashboards/cache.json +++ b/engine/docker/dev/grafana/dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/foundationdb.json b/engine/docker/dev/grafana/dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/dev/grafana/dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/futures.json b/engine/docker/dev/grafana/dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/dev/grafana/dashboards/futures.json +++ b/engine/docker/dev/grafana/dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/gasoline.json b/engine/docker/dev/grafana/dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/dev/grafana/dashboards/gasoline.json +++ b/engine/docker/dev/grafana/dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/guard.json b/engine/docker/dev/grafana/dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/dev/grafana/dashboards/guard.json +++ b/engine/docker/dev/grafana/dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/operation.json b/engine/docker/dev/grafana/dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/dev/grafana/dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/tokio.json b/engine/docker/dev/grafana/dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/dev/grafana/dashboards/tokio.json +++ b/engine/docker/dev/grafana/dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/dev/grafana/provisioning/datasources/datasources.yaml b/engine/docker/dev/grafana/provisioning/datasources/datasources.yaml index f66d3bf036..8c4e4e6560 100644 --- a/engine/docker/dev/grafana/provisioning/datasources/datasources.yaml +++ b/engine/docker/dev/grafana/provisioning/datasources/datasources.yaml @@ -1,5 +1,13 @@ apiVersion: 1 datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + timeInterval: 15s - name: ClickHouse uid: clickhouse type: grafana-clickhouse-datasource diff --git a/engine/docker/dev/otel-collector-client/config.yaml b/engine/docker/dev/otel-collector-client/config.yaml deleted file mode 100644 index 6e963d5edf..0000000000 --- a/engine/docker/dev/otel-collector-client/config.yaml +++ /dev/null @@ -1,39 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: - batch: - timeout: 5s - send_batch_size: 10000 -exporters: - otlp/server: - endpoint: otel-collector-server:4317 - tls: - insecure: true -service: - pipelines: - logs: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - traces: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server - metrics: - receivers: - - otlp - processors: - - batch - exporters: - - otlp/server diff --git a/engine/docker/dev-multinode/otel-collector-server/config.yaml b/engine/docker/dev/otel-collector/config.yaml similarity index 70% rename from engine/docker/dev-multinode/otel-collector-server/config.yaml rename to engine/docker/dev/otel-collector/config.yaml index 7b5fc80c72..ea66f377ba 100644 --- a/engine/docker/dev-multinode/otel-collector-server/config.yaml +++ b/engine/docker/dev/otel-collector/config.yaml @@ -3,6 +3,16 @@ receivers: protocols: grpc: endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + prometheus: + config: + scrape_configs: + - job_name: rivet-engine-default-0 + scrape_interval: 15s + static_configs: + - targets: + - rivet-engine:6430 processors: resource: attributes: @@ -28,22 +38,17 @@ exporters: logs_table_name: otel_logs traces_table_name: otel_traces timeout: 5s - metrics_tables: - gauge: - name: otel_metrics_gauge - sum: - name: otel_metrics_sum - summary: - name: otel_metrics_summary - histogram: - name: otel_metrics_histogram - exponential_histogram: - name: otel_metrics_exp_histogram retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://prometheus:9090/api/v1/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: logs: @@ -64,9 +69,9 @@ service: - clickhouse metrics: receivers: - - otlp + - prometheus processors: - resource - batch exporters: - - clickhouse + - prometheusremotewrite diff --git a/engine/docker/dev/prometheus/prometheus.yml b/engine/docker/dev/prometheus/prometheus.yml new file mode 100644 index 0000000000..edb0f179de --- /dev/null +++ b/engine/docker/dev/prometheus/prometheus.yml @@ -0,0 +1,4 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s +scrape_configs: [] diff --git a/engine/docker/template/grafana-dashboards/api.json b/engine/docker/template/grafana-dashboards/api.json index a9417891ef..87a2584411 100644 --- a/engine/docker/template/grafana-dashboards/api.json +++ b/engine/docker/template/grafana-dashboards/api.json @@ -1,1254 +1,1041 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 24, - "panels": [], - "title": "API", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 26, - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_pending'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Requests Pending", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 32, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(count_diff > 0 AND sum_diff >= 0, sum_diff / count_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n sum_val,\n count_val,\n sum_val - lagInFrame(sum_val, 1, sum_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as sum_diff,\n count_val - lagInFrame(count_val, 1, count_val) OVER (PARTITION BY method, path, datacenter ORDER BY time) as count_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Sum) as sum_val,\n max(Count) as count_val\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Duration (avg)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 33, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.95)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p95)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 34, - "interval": "15s", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{datacenter_id}} {{method}} {{path}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n quantileInterpolatedWeighted(0.99)(bound_value, count_value) as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(ResourceAttributes['rivet.datacenter'], ' ', Attributes['method'], ' ', Attributes['path']) as label,\n arrayJoin(arrayEnumerate(arrayConcat([0], ExplicitBounds, [inf]))) as idx,\n arrayConcat([0], ExplicitBounds, [inf])[idx] as bound_value,\n BucketCounts[idx] as count_value\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n )\n GROUP BY time, label\n )\n\nORDER BY label\n", - "refId": "A" - } - ], - "title": "Duration (p99)", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "zero", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 25, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path) as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_total'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 28, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', status, ' (', error_code, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '4%'\n AND Attributes['error_code'] NOT IN ('API_CANCELLED', 'CAPTCHA_CAPTCHA_REQUIRED')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (4xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 31, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.1", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n concat(datacenter, ' ', method, ' ', path, ': ', error_code, ' (', status, ')') as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n method,\n path,\n status,\n error_code,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY method, path, status, error_code, datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['method'] as method,\n Attributes['path'] as path,\n Attributes['status'] as status,\n Attributes['error_code'] as error_code,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_api_request_errors'\n AND Attributes['status'] LIKE '5%'\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, method, path, status, error_code, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY time", - "refId": "A" - } - ], - "title": "Error Rate (5xx)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "percent" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 28 - }, - "id": 35, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY label ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['status'], ' ', Attributes['error_code']) as label,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_api_request_duration'\n AND (Attributes['status'] = '200 OK' OR Attributes['status'] LIKE '5%')\n AND Attributes['path'] IN array($path)\n AND Attributes['method'] IN array($method)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, label\n )\n)\nORDER BY label\n", - "refId": "A" - } - ], - "title": "200 vs 5xx (4xx excluded)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "Project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "Datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "includeAll": true, - "label": "Path", - "multi": true, - "name": "path", - "options": [], - "query": "SELECT DISTINCT Attributes['path'] as path FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY path", - "refresh": 1, - "regex": "", - "sort": 1, - "type": "query" - }, - { - "current": { - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "includeAll": true, - "label": "Method", - "multi": true, - "name": "method", - "options": [], - "query": "SELECT DISTINCT Attributes['method'] as method FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_api_request_total' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) AND Attributes['path'] IN array($path) ORDER BY method", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "API", - "uid": "feg8y72gpm2o0c", - "version": 1, - "weekStart": "" + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "panels": [], + "title": "API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 26, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_api_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__interval])) by (rivet_datacenter, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 32, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path) /\r\nsum(rate(rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval])) by (rivet_datacenter, method, path)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 33, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p95)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 34, + "interval": "15s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_api_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"} [$__rate_interval]))\r\n by (le, rivet_datacenter, method, path)\r\n)", + "format": "heatmap", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Duration (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\"}[$__rate_interval])) by (rivet_datacenter, method, path)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}", + "range": true, + "refId": "A" + } + ], + "title": "Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",status=~\"4[0-9][0-9]\",path=~\"$path\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_api_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",status=~\"5[0-9][0-9]\",method=~\"$method\",error_code!=\"API_CANCELLED\"}[$__rate_interval])) by (rivet_datacenter, method, path, status, error_code)", + "legendFormat": "{{rivet_datacenter}} {{method}} {{path}}: {{status}} ({{error_code}})", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(\n rivet_api_request_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\",method=~\"$method\",status=~\"(200|5[0-9][0-9])\",error_code!=\"API_CANCELLED\"}\n [$__rate_interval])) by (status, error_code)", + "legendFormat": "{{method}} {{path}}: {{status}} {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "200 vs 5xx (4xx excluded)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "includeAll": true, + "label": "Path", + "multi": true, + "name": "path", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},path)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "includeAll": true, + "label": "Method", + "multi": true, + "name": "method", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_api_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",path=~\"$path\"},method)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "API", + "uid": "feg8y72gpm2o0c", + "version": 1, + "weekStart": "" } \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/cache.json b/engine/docker/template/grafana-dashboards/cache.json index 385e42ff48..802c056290 100644 --- a/engine/docker/template/grafana-dashboards/cache.json +++ b/engine/docker/template/grafana-dashboards/cache.json @@ -1,1075 +1,906 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 8, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 10 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_request_errors'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Request Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_miss_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND miss_diff >= 0, miss_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n miss_val - lagInFrame(miss_val, 1, miss_val) OVER (PARTITION BY key ORDER BY time) as miss_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_miss_total') as miss_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_miss_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Miss Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_value_empty_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(total_diff > 0 AND empty_diff >= 0, empty_diff / total_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n empty_val - lagInFrame(empty_val, 1, empty_val) OVER (PARTITION BY key ORDER BY time) as empty_diff,\n total_val - lagInFrame(total_val, 1, total_val) OVER (PARTITION BY key ORDER BY time) as total_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n sumIf(Value, MetricName = 'rivet_cache_value_empty_total') as empty_val,\n sumIf(Value, MetricName = 'rivet_cache_value_total') as total_val\n FROM otel.otel_metrics_sum\n WHERE MetricName IN ('rivet_cache_value_empty_total', 'rivet_cache_value_total')\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> ''\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Empty Rate (% of total)", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_request_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n key as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n key,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY key ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY key ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['key'] as key,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_cache_purge_value_total'\n AND Attributes['key'] IN array($key)\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, key\n )\n )\n WHERE key <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Cache Purge Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "includeAll": true, - "label": "key", - "multi": true, - "name": "key", - "options": [], - "query": "SELECT DISTINCT Attributes['key'] FROM otel.otel_metrics_sum WHERE MetricName = 'rivet_cache_request_total' AND ServiceName = 'rivet' AND ResourceAttributes['rivet.datacenter'] IN array($datacenter) ORDER BY Attributes['key']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Cache", - "uid": "c35233ed-b698-4838-9426-18e1586017f1", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_cache_request_error_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Request Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_miss_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Miss Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_value_empty_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval]) / rate(rivet_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Empty Rate (% of total)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_cache_purge_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "key/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_purge_cache_value_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",key=~\"$key\"}[$__rate_interval])) by (key)", + "instant": false, + "legendFormat": "{{key}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Purge Rate", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "hide": 0, + "includeAll": true, + "label": "Cache Key", + "multi": true, + "name": "key", + "options": [], + "query": { + "query": "label_values(rivet_cache_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},key)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cache", + "uid": "c35233ed-b698-4838-9426-18e1586017f1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/foundationdb.json b/engine/docker/template/grafana-dashboards/foundationdb.json new file mode 100644 index 0000000000..40acb297a3 --- /dev/null +++ b/engine/docker/template/grafana-dashboards/foundationdb.json @@ -0,0 +1,1845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "op/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_op_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",op=~\"reads|writes\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{op}})", + "range": true, + "refId": "A" + } + ], + "title": "Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txn/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_transaction_count_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\",state=~\"started|committed|conflicted\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} ({{state}})", + "range": true, + "refId": "A" + } + ], + "title": "Transactions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_storage_keys_queried_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Keys Queried", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_read_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "instant": false, + "legendFormat": "{{instance}} (read)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "irate(fdb_workload_bytes_written_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[1m])", + "hide": false, + "instant": false, + "legendFormat": "{{instance}} (written)", + "range": true, + "refId": "B" + } + ], + "title": "Workload Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_memory_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_memory_limit_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_process_cpu_utilization{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_pack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Packed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 2, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "k/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (type) (irate(rivet_fdb_key_unpack_count[1m]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Keys Unpacked", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Disk Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_total_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_client_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "1 - fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Database Unavailable", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_storage_data_lag_seconds{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Data Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 10, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_degraded_process_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Degraded Processes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_average_partition_size_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Partition Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_moving_data_in_flight_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} + fdb_moving_data_in_queue_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Moving Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "fdb_log_queue_disk_used_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} / on(instance) fdb_log_queue_disk_total_bytes{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Log Queue Utilization", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(fdb_database_available{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "FoundationDB", + "uid": "aef2tiqr8tnghe", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/futures.json b/engine/docker/template/grafana-dashboards/futures.json index 03880e4bef..fdca7320ea 100644 --- a/engine/docker/template/grafana-dashboards/futures.json +++ b/engine/docker/template/grafana-dashboards/futures.json @@ -1,230 +1,243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 3, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 1, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_instrumented_future_duration'\n -- AND ResourceAttributes['rivet.project'] IN array($project)\n -- AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['name'] IN array($name)\n AND Attributes['location'] IN array($location)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Instrumented Future Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "description": "", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "includeAll": true, - "label": "name", - "multi": true, - "name": "name", - "options": [], - "query": "SELECT DISTINCT Attributes['name'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['name']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "includeAll": true, - "label": "location", - "multi": true, - "name": "location", - "options": [], - "query": "SELECT DISTINCT Attributes['location'] FROM otel.otel_metrics_histogram WHERE ServiceName = 'rivet' AND MetricName = 'rivet_instrumented_future_duration' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['location']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Futures", - "uid": "ef353ektqu4g0e", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_instrumented_future_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",name=~\"$name\",location=~\"$location\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Instrumented Future Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "30s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "includeAll": true, + "label": "Name", + "multi": true, + "name": "name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "includeAll": true, + "label": "Location", + "multi": true, + "name": "location", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_instrumented_future_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},location)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Futures", + "version": 0, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/gasoline.json b/engine/docker/template/grafana-dashboards/gasoline.json index 16a9f37025..672ac40493 100644 --- a/engine/docker/template/grafana-dashboards/gasoline.json +++ b/engine/docker/template/grafana-dashboards/gasoline.json @@ -1,2477 +1,2243 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 6, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_active'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Running Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_sleeping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Sleeping Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflows", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "({{workflow_name}}) {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['workflow_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_workflow_dead'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Dead Workflow Errors", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "__auto", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n count(*) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_worker_last_ping'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label, ResourceAttributes['rivet.datacenter']\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Workers", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{signal_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['signal_name'] as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_signal_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Pending Signals", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size", - "value": "" - }, - "yBuckets": { - "mode": "size", - "scale": { - "log": 2, - "type": "log" - }, - "value": "" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "hide": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_recv_lag'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Receive Lag", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_pull_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Signals Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 13, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 14, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{worker_id}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['worker_id'] as label,\n max(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_last_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Last Pull Workflows History Duration", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 24, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_pull_workflows_history_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Pull Workflows History Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 48 - }, - "id": 16, - "panels": [], - "title": "Steps", - "type": "row" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 49 - }, - "id": 12, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_activity_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Activity Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 49 - }, - "id": 9, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{activity_name}}: {{error_code}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n\tSELECT\n $__timeInterval(TimeUnix) as time,\n concat(Attributes['activity_name'], ' (', Attributes['error'], ')') as label,\n sum(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_gasoline_activity_errors'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Activity Error Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 58 - }, - "id": 21, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Loop Iteration Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ev/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 58 - }, - "id": 25, - "interval": "15s", - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "legendFormat": "{{workflow_name}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 as collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n workflow_name as label,\n greatest(0, total_count - lagInFrame(total_count, 1, 0) OVER (PARTITION BY workflow_name ORDER BY time)) / $__interval_ms * 1000 as value\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n Attributes['workflow_name'] as workflow_name,\n sum(arraySum(BucketCounts)) as total_count\n FROM otel.otel_metrics_histogram\n WHERE MetricName = 'rivet_gasoline_loop_iteration_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY time, workflow_name\n )\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Loop Events Per Second", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "label", - "emptyValue": "null", - "rowField": "time", - "valueField": "value" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "time\\label" - } - ], - "fields": {} - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 67 - }, - "id": 17, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_message_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Message Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 67 - }, - "id": 20, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_signal_send_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Signal Send Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 76 - }, - "id": 22, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_find_workflows_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Find Workflows Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 76 - }, - "id": 19, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_gasoline_workflow_dispatch_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND Attributes['workflow_name'] IN array($workflow_name)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Workflow Dispatch Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "definition": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "includeAll": true, - "label": "workflow name", - "multi": true, - "name": "workflow_name", - "options": [], - "query": "SELECT DISTINCT Attributes['workflow_name'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND MetricName = 'rivet_gasoline_workflow_total' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY Attributes['workflow_name']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gasoline", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 7, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_active{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Running Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (workflow_name) (rivet_gasoline_workflow_sleeping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Sleeping Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (workflow_name, error_code) (rivet_gasoline_workflow_dead{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"})", + "instant": false, + "legendFormat": "({{workflow_name}}) {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Dead Workflow Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 5, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (signal_name) (rivet_gasoline_signal_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{signal_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pending Signals", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 10, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_recv_lag_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Signal Receive Lag", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_signal_pull_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Signals Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max by (worker_id) (rivet_gasoline_last_pull_workflows_history_duration{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"})", + "instant": false, + "legendFormat": "{{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Last Pull Workflows History Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 24, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_pull_workflows_history_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Pull Workflows History Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 20, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_cpu_usage_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Core Usage", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 26, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "percentunit" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_load_shedding_ratio_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Load Shedding Ratio", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 16, + "panels": [], + "title": "Steps", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 12, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_activity_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_activity_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"}[$__rate_interval])) by (activity_name, error_code)", + "legendFormat": "{{activity_name}}: {{error_code}}", + "range": true, + "refId": "A" + } + ], + "title": "Activity Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_iteration_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iteration Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 25, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(irate(rivet_gasoline_loop_iteration_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (workflow_name)", + "legendFormat": "{{workflow_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Iterations/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 76 + }, + "id": 27, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_loop_commit_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Loop Commit Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 76 + }, + "id": 17, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_message_send_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Message Send Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 22, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_find_workflows_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Find Workflows Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 19, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_workflow_dispatch_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",workflow_name=~\"$workflow_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sub Workflow Dispatch Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "includeAll": true, + "label": "Workflow Name", + "multi": true, + "name": "workflow_name", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_gasoline_workflow_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},workflow_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Gasoline", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105c", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/guard.json b/engine/docker/template/grafana-dashboards/guard.json index 6fdd9d3093..320127817a 100644 --- a/engine/docker/template/grafana-dashboards/guard.json +++ b/engine/docker/template/grafana-dashboards/guard.json @@ -1,1274 +1,1099 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 7, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_route_cache_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Route Cache Size", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_rate_limiter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Rate Limiters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH 1 AS smoothness\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n sum(Value) as value\n FROM otel.otel_metrics_gauge\n WHERE MetricName = 'rivet_guard_in_flight_counter_count'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND $__timeFilter(TimeUnix)\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "In-Flight Counters", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active TCP Connections", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_tcp_connection_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "TCP Connection Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 8 - }, - "id": 6, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_tcp_connection_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "TCP Connection Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 5, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter}}", - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 4 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as label,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_pending'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, label\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Active Proxy Requests", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": ["mean"], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "range": true, - "rawSql": "WITH\n 30 AS collector_rate_s,\n 5 AS smoothness\n\nSELECT\n time,\n label,\n avg(value) OVER (PARTITION BY label ORDER BY time ROWS BETWEEN smoothness - 1 PRECEDING AND CURRENT ROW) as value\nFROM (\n SELECT\n time,\n datacenter as label,\n if(value_diff >= 0, value_diff / time_diff, 0) as value\n FROM (\n SELECT\n time,\n datacenter,\n value,\n value - lagInFrame(value, 1, value) OVER (PARTITION BY datacenter ORDER BY time) as value_diff,\n toUnixTimestamp(time) - lagInFrame(toUnixTimestamp(time), 1, toUnixTimestamp(time)) OVER (PARTITION BY datacenter ORDER BY time) as time_diff\n FROM (\n SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['rivet.datacenter'] as datacenter,\n max(Value) as value\n FROM otel.otel_metrics_sum\n WHERE MetricName = 'rivet_guard_proxy_request_total'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000) * smoothness)\n AND TimeUnix <= $__toTime_ms()\n GROUP BY time, datacenter\n )\n )\n WHERE datacenter <> '' AND time_diff > 0\n)\nORDER BY label", - "refId": "A" - } - ], - "title": "Proxy Request Rate", - "transformations": [ - { - "id": "prepareTimeSeries", - "options": { - "format": "multi" - } - }, - { - "id": "renameByRegex", - "options": { - "regex": "^value\\s(.*)$", - "renamePattern": "$1" - } - } - ], - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 16 - }, - "id": 9, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_proxy_request_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Proxy Request Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 10, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "meta": {}, - "pluginVersion": "4.11.2", - "queryType": "table", - "rawSql": "WITH\n 30 as collector_rate_s\n\nSELECT Time, bucket, count\nFROM (\n SELECT\n Time,\n bucket,\n greatest(0, count - lagInFrame(count, 1, 0) OVER (PARTITION BY bucket ORDER BY Time)) AS count\n FROM (\n WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n case when idx = length(BucketCounts) then inf else ExplicitBounds[idx] end as bucket\n SELECT\n time_bucket as Time,\n bucket,\n sum(BucketCounts[idx]) AS count\n FROM otel.otel_metrics_histogram\n WHERE\n MetricName = 'rivet_guard_resolve_route_duration'\n AND ResourceAttributes['rivet.project'] IN array($project)\n AND ResourceAttributes['rivet.datacenter'] IN array($datacenter)\n AND TimeUnix >= $__fromTime_ms() - toIntervalMillisecond(greatest(collector_rate_s * 1000, $__interval_s * 1000))\n AND TimeUnix <= $__toTime_ms()\n AND AggregationTemporality = 2\n GROUP BY Time, bucket\n )\n)\nWHERE $__timeFilter(Time)", - "refId": "A" - } - ], - "title": "Resolve Route Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "bucket", - "emptyValue": "null", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\bucket" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - } - ], - "preload": false, - "refresh": "30s", - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "includeAll": true, - "label": "project", - "multi": true, - "name": "project", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.project'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' ORDER BY ResourceAttributes['rivet.project']", - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "includeAll": true, - "label": "datacenter", - "multi": true, - "name": "datacenter", - "options": [], - "query": "SELECT DISTINCT ResourceAttributes['rivet.datacenter'] FROM otel.otel_metrics_gauge WHERE ServiceName = 'rivet' AND ResourceAttributes['rivet.project'] IN array($project) ORDER BY ResourceAttributes['rivet.datacenter']", - "refresh": 1, - "regex": "", - "type": "query" - } - ] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Guard", - "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 115, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "In-Flight Counters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active TCP Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_tcp_connection_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 1, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_tcp_connection_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "TCP Connection Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_proxy_request_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Proxy Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "req/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_guard_proxy_request_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 9, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_proxy_request_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Proxy Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 6, + "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Resolve Route Duration", + "type": "heatmap" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 40, + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Rivet Guard", + "uid": "cen785ige8fswd", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/operation.json b/engine/docker/template/grafana-dashboards/operation.json new file mode 100644 index 0000000000..aef24f0c6f --- /dev/null +++ b/engine/docker/template/grafana-dashboards/operation.json @@ -0,0 +1,871 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 13, + "panels": [], + "title": "Worker", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 2, + "interval": "15s", + "legend": { + "show": false + }, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.1.4", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 2, + "max": "60", + "min": "0", + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(max_over_time(rivet_gasoline_operation_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__interval])) by (operation_name)", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Calls Pending", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 9, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", + "legendFormat": "{{operation_name}}: {{error_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 20, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(\r\n sum(rate(rivet_gasoline_operation_duration_sum{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name) /\r\n sum(rate(rivet_gasoline_operation_duration_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) by (operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (avg)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 21, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0\r\n", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p95)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "seconds" + } + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 22, + "interval": "15s", + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \r\n sum(rate(rivet_gasoline_operation_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",error_code!=\"1004\",operation_name=~\"$operation_name\"} [$__rate_interval])) \r\n by (le, operation_name)\r\n) > 0", + "format": "heatmap", + "legendFormat": "{{operation_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Call Duration (p99)", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "hide": 0, + "includeAll": true, + "label": "Operation Name", + "multi": true, + "name": "operation_name", + "options": [], + "query": { + "query": "label_values(rivet_gasoline_operation_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},operation_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Operation", + "uid": "c6345ab8-9146-499f-ab0b-7fb10fe5b9e1", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/tokio.json b/engine/docker/template/grafana-dashboards/tokio.json index e50028f6ff..ada2268fc8 100644 --- a/engine/docker/template/grafana-dashboards/tokio.json +++ b/engine/docker/template/grafana-dashboards/tokio.json @@ -1,1005 +1,931 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": 81, - "links": [], - "panels": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 24, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_global_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Global Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 27, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_local_queue_depth'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Queue Depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 11, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_tokio_task_poll_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Task Poll Times", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 26, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_worker_overflow_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Worker Overflow Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 23, - "interval": "15s", - "options": { - "calculate": false, - "calculation": { - "xBuckets": { - "mode": "size" - } - }, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "max": "60", - "min": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "range": true, - "rawSql": "WITH\n $__timeInterval(TimeUnix) as time_bucket,\n arrayJoin(arrayEnumerate(BucketCounts)) AS idx,\n ExplicitBounds AS bounds\nSELECT\n time_bucket as Time,\n bounds[idx+1] AS le,\n sum(BucketCounts[idx]) AS count\nFROM otel.otel_metrics_histogram\nWHERE\n MetricName = 'rivet_future_schedule_duration'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY Time, le\nORDER BY Time, le", - "refId": "A" - } - ], - "title": "Future Schedule Duration", - "transformations": [ - { - "id": "groupingToMatrix", - "options": { - "columnField": "le", - "emptyValue": "zero", - "rowField": "Time", - "valueField": "count" - } - }, - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "time", - "targetField": "Time\\le" - } - ], - "fields": {} - } - } - ], - "type": "heatmap" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 25, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_thread_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Thread Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "tasks/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 28, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n sum(Value) / $__interval_ms * 1000 as value\nFROM otel.otel_metrics_sum\nWHERE MetricName = 'rivet_tokio_task'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Task Spawn Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 0.1, - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 29, - "options": { - "legend": { - "calcs": ["lastNotNull"], - "displayMode": "table", - "placement": "bottom", - "showLegend": false, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.2", - "targets": [ - { - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "editorMode": "code", - "editorType": "sql", - "format": 1, - "instant": false, - "legendFormat": "{{datacenter_id}} - {{server_id}}", - "range": true, - "rawSql": "SELECT\n $__timeInterval(TimeUnix) as time,\n ResourceAttributes['datacenter_id'] as datacenter_id,\n ResourceAttributes['server_id'] as server_id,\n avg(Value) as value\nFROM otel.otel_metrics_gauge\nWHERE MetricName = 'rivet_tokio_active_task_count'\n AND $__conditionalAll(ResourceAttributes['cluster_id'], $cluster_id)\n AND $__conditionalAll(ResourceAttributes['datacenter_id'], $datacenter_id)\n AND $__conditionalAll(ResourceAttributes['server_id'], $server_id)\n AND $__conditionalAll(Attributes['pool_type'], $pool_type)\n AND $__timeFilter(TimeUnix)\nGROUP BY time, datacenter_id, server_id\nORDER BY time", - "refId": "A" - } - ], - "title": "Active Tasks", - "type": "timeseries" - } - ], - "preload": false, - "refresh": "5s", - "schemaVersion": 40, - "templating": { - "list": [ - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "description": "", - "includeAll": true, - "label": "Cluster ID", - "multi": true, - "name": "cluster_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['cluster_id'] as cluster_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY cluster_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "description": "", - "includeAll": true, - "label": "Dataceter ID", - "multi": true, - "name": "datacenter_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['datacenter_id'] as datacenter_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY datacenter_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "description": "", - "includeAll": true, - "label": "Server ID", - "multi": true, - "name": "server_id", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT ResourceAttributes['server_id'] as server_id FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY server_id", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": ["All"], - "value": ["$__all"] - }, - "datasource": { - "type": "grafana-clickhouse-datasource", - "uid": "clickhouse" - }, - "definition": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "includeAll": true, - "label": "Pool", - "multi": true, - "name": "pool_type", - "options": [], - "query": { - "qryType": 1, - "rawSql": "SELECT DISTINCT Attributes['pool_type'] as pool_type FROM otel.otel_metrics_gauge WHERE MetricName = 'rivet_tokio_thread_count' ORDER BY pool_type", - "refId": "ClickHouseVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "30", - "value": "30" - }, - "hide": 2, - "label": "Metric Export Interval (seconds)", - "name": "metric_interval", - "query": "30", - "skipUrlSync": true, - "type": "constant" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Tokio", - "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", - "version": 1, - "weekStart": "" -} + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 81, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_global_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Global Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_local_queue_depth{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Queue Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_tokio_task_poll_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Poll Times", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_worker_overflow_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Worker Overflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 23, + "interval": "15s", + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "mode": "size" + } + }, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "max": "60", + "min": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(rivet_future_schedule_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Future Schedule Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Thread Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "tasks/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rate(rivet_tokio_task_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Task Spawn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 0.1, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": false, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.5.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_tokio_active_task_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Tasks", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 40, + "templating": { + "list": [ + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(rivet_project)", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_project)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "includeAll": true, + "label": "Datacenter", + "multi": true, + "name": "datacenter", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({rivet_project=~\"$project\"},rivet_datacenter)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "description": "", + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(rivet_tokio_thread_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tokio", + "uid": "636d22f9-d18f-4086-8b45-7c50886a105d", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/engine/docker/template/src/docker-compose.ts b/engine/docker/template/src/docker-compose.ts index 5daa3b2182..5e7dd6719b 100644 --- a/engine/docker/template/src/docker-compose.ts +++ b/engine/docker/template/src/docker-compose.ts @@ -65,6 +65,32 @@ export function generateDockerCompose(context: TemplateContext) { }; volumes["clickhouse-data"] = null; + // Prometheus + services["prometheus"] = { + restart: "unless-stopped", + image: "prom/prometheus:latest", + volumes: [ + "prometheus-data:/prometheus", + `./${context.getCoreServicePath("prometheus")}/prometheus.yml:/etc/prometheus/prometheus.yml`, + ], + command: [ + "--config.file=/etc/prometheus/prometheus.yml", + "--storage.tsdb.path=/prometheus", + "--web.console.libraries=/usr/share/prometheus/console_libraries", + "--web.console.templates=/usr/share/prometheus/consoles", + "--web.enable-remote-write-receiver", + ], + ports: ["9090:9090"], + networks: [...allDcToCoreNetworkNames, CORE_NETWORK_NAME], + healthcheck: { + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:9090/-/healthy"], + interval: "5s", + timeout: "3s", + retries: 3, + }, + }; + volumes["prometheus-data"] = null; + // Grafana services["grafana"] = { image: "grafana/grafana:11.5.2", @@ -79,6 +105,7 @@ export function generateDockerCompose(context: TemplateContext) { networks: [CORE_NETWORK_NAME], depends_on: { clickhouse: { condition: "service_healthy" }, + prometheus: { condition: "service_healthy" }, }, }; volumes["grafana-data"] = null; @@ -100,12 +127,8 @@ export function generateDockerCompose(context: TemplateContext) { "vector-client", datacenter.name, ); - const otelCollectorServerServiceName = context.getServiceName( - "otel-collector-server", - datacenter.name, - ); - const otelCollectorClientServiceName = context.getServiceName( - "otel-collector-client", + const otelCollectorServiceName = context.getServiceName( + "otel-collector", datacenter.name, ); const postgresServiceName = context.getServiceName( @@ -180,7 +203,8 @@ export function generateDockerCompose(context: TemplateContext) { }, platform: "linux/amd64", restart: "unless-stopped", - command: "sleep infinity", + entrypoint: "sleep", + command: "infinity", stop_grace_period: "0s", depends_on: { //[natsServiceName]: { condition: "service_healthy" }, @@ -226,34 +250,20 @@ export function generateDockerCompose(context: TemplateContext) { }; volumes[`vector-client-data-${datacenter.name}`] = null; - services[otelCollectorServerServiceName] = { + services[otelCollectorServiceName] = { image: "otel/opentelemetry-collector-contrib:latest", restart: "unless-stopped", command: "--config=/etc/otel/config.yaml", volumes: [ - `./${context.getDatacenterServicePath("otel-collector-server", datacenter.name)}/config.yaml:/etc/otel/config.yaml:ro`, + `./${context.getDatacenterServicePath("otel-collector", datacenter.name)}/config.yaml:/etc/otel/config.yaml:ro`, ], environment: ["CLICKHOUSE_PASSWORD=default"], depends_on: { clickhouse: { condition: "service_healthy" }, + prometheus: { condition: "service_healthy" }, }, networks: [dcNetworkName, dcToCoreNetworkName], - ports: ["4317:4317"], - }; - - services[otelCollectorClientServiceName] = { - image: "otel/opentelemetry-collector-contrib:latest", - restart: "unless-stopped", - command: "--config=/etc/otel/config.yaml", - volumes: [ - `./${context.getDatacenterServicePath("otel-collector-client", datacenter.name)}/config.yaml:/etc/otel/config.yaml:ro`, - ], - depends_on: { - [otelCollectorServerServiceName]: { - condition: "service_started", - }, - }, - networks: [dcNetworkName], + ports: isPrimary ? ["4317:4317"] : undefined, }; for (let i = 0; i < datacenter.engines; i++) { @@ -278,7 +288,7 @@ export function generateDockerCompose(context: TemplateContext) { "RUST_LOG_ANSI_COLOR=1", "RIVET_OTEL_ENABLED=1", "RIVET_OTEL_SAMPLER_RATIO=1", - `RIVET_OTEL_GRPC_ENDPOINT=http://${context.getServiceHost("otel-collector-client", datacenter.name)}:4317`, + `RIVET_OTEL_GRPC_ENDPOINT=http://${context.getServiceHost("otel-collector", datacenter.name)}:4317`, // "RUST_LOG=debug,hyper=info", ], stop_grace_period: "0s", @@ -287,7 +297,7 @@ export function generateDockerCompose(context: TemplateContext) { [vectorClientServiceName]: { condition: "service_started", }, - [otelCollectorClientServiceName]: { + [otelCollectorServiceName]: { condition: "service_started", }, [postgresServiceName]: { condition: "service_healthy" }, diff --git a/engine/docker/template/src/main.ts b/engine/docker/template/src/main.ts index 81ce5df379..4a984d1f70 100644 --- a/engine/docker/template/src/main.ts +++ b/engine/docker/template/src/main.ts @@ -9,8 +9,8 @@ import { generateGitAttributes } from "./git"; import { generateReadme } from "./readme"; import { generateCoreClickhouse } from "./services/core/clickhouse"; import { generateCoreGrafana } from "./services/core/grafana"; -import { generateDatacenterOtelCollectorClient } from "./services/edge/otel-collector-client"; -import { generateDatacenterOtelCollectorServer } from "./services/edge/otel-collector-server"; +import { generateCorePrometheus } from "./services/core/prometheus"; +import { generateDatacenterOtelCollector } from "./services/edge/otel-collector"; import { generateDatacenterPostgres } from "./services/edge/postgres"; import { generateDatacenterRivetEngine } from "./services/edge/rivet-engine"; import { generateRunner } from "./services/edge/runner"; @@ -35,6 +35,7 @@ function generateTemplate(templateName: string, config: TemplateConfig) { // Generate core services generateCoreClickhouse(context); generateCoreGrafana(context); + generateCorePrometheus(context); // Generate datacenter-specific configurations for (const datacenter of config.datacenters) { @@ -42,8 +43,7 @@ function generateTemplate(templateName: string, config: TemplateConfig) { generateDatacenterRivetEngine(context, datacenter); generateDatacenterVectorServer(context, datacenter.name); generateDatacenterVectorClient(context, datacenter.name); - generateDatacenterOtelCollectorServer(context, datacenter.name); - generateDatacenterOtelCollectorClient(context, datacenter.name); + generateDatacenterOtelCollector(context, datacenter); } generateRunner(context); diff --git a/engine/docker/template/src/services/core/grafana.ts b/engine/docker/template/src/services/core/grafana.ts index 813e44ffba..0db9e06fc7 100644 --- a/engine/docker/template/src/services/core/grafana.ts +++ b/engine/docker/template/src/services/core/grafana.ts @@ -6,6 +6,8 @@ import type { TemplateContext } from "../../context"; export function generateCoreGrafana(context: TemplateContext) { const clickhouseHost = context.config.networkMode === "host" ? "127.0.0.1" : "clickhouse"; + const prometheusHost = + context.config.networkMode === "host" ? "127.0.0.1" : "prometheus"; // Grafana configuration const grafanaIni = `[server] http_port = 3000 @@ -23,10 +25,21 @@ org_role = Admin default_home_dashboard_path = /var/lib/grafana/dashboards/api.json `; - // Datasource configuration for ClickHouse + // Datasource configuration for ClickHouse and Prometheus const datasourcesConfig = { apiVersion: 1, datasources: [ + { + name: "Prometheus", + uid: "prometheus", + type: "prometheus", + access: "proxy", + url: `http://${prometheusHost}:9090`, + isDefault: true, + jsonData: { + timeInterval: "15s", + }, + }, { name: "ClickHouse", uid: "clickhouse", diff --git a/engine/docker/template/src/services/core/prometheus.ts b/engine/docker/template/src/services/core/prometheus.ts new file mode 100644 index 0000000000..64739d33a7 --- /dev/null +++ b/engine/docker/template/src/services/core/prometheus.ts @@ -0,0 +1,20 @@ +import * as yaml from "js-yaml"; +import type { TemplateContext } from "../../context"; + +export function generateCorePrometheus(context: TemplateContext) { + // Prometheus configuration with remote write enabled + // Metrics are scraped by OTEL collector and sent via remote write + const prometheusConfig = { + global: { + scrape_interval: "15s", + evaluation_interval: "15s", + }, + scrape_configs: [], + }; + + context.writeCoreServiceFile( + "prometheus", + "prometheus.yml", + yaml.dump(prometheusConfig), + ); +} diff --git a/engine/docker/template/src/services/edge/otel-collector-client.ts b/engine/docker/template/src/services/edge/otel-collector-client.ts deleted file mode 100644 index 236bef90fd..0000000000 --- a/engine/docker/template/src/services/edge/otel-collector-client.ts +++ /dev/null @@ -1,64 +0,0 @@ -import * as yaml from "js-yaml"; -import type { TemplateContext } from "../../context"; - -export function generateDatacenterOtelCollectorClient( - context: TemplateContext, - dcId: string, -) { - const otelConfig = { - receivers: { - otlp: { - protocols: { - grpc: { - endpoint: "0.0.0.0:4317", - }, - http: { - endpoint: "0.0.0.0:4318", - }, - }, - }, - }, - processors: { - batch: { - timeout: "5s", - send_batch_size: 10000, - }, - }, - exporters: { - "otlp/server": { - endpoint: `${context.getServiceHost("otel-collector-server", dcId)}:4317`, - tls: { - insecure: true, - }, - }, - }, - service: { - pipelines: { - logs: { - receivers: ["otlp"], - processors: ["batch"], - exporters: ["otlp/server"], - }, - traces: { - receivers: ["otlp"], - processors: ["batch"], - exporters: ["otlp/server"], - }, - metrics: { - receivers: ["otlp"], - processors: ["batch"], - exporters: ["otlp/server"], - }, - }, - }, - }; - - const yamlContent = yaml.dump(otelConfig); - - context.writeDatacenterServiceFile( - "otel-collector-client", - dcId, - "config.yaml", - yamlContent, - ); -} diff --git a/engine/docker/template/src/services/edge/otel-collector-server.ts b/engine/docker/template/src/services/edge/otel-collector.ts similarity index 55% rename from engine/docker/template/src/services/edge/otel-collector-server.ts rename to engine/docker/template/src/services/edge/otel-collector.ts index 5b57dc4e7c..3a4709d3c5 100644 --- a/engine/docker/template/src/services/edge/otel-collector-server.ts +++ b/engine/docker/template/src/services/edge/otel-collector.ts @@ -1,12 +1,38 @@ import * as yaml from "js-yaml"; import type { TemplateContext } from "../../context"; +import type { DatacenterConfig } from "../../config"; -export function generateDatacenterOtelCollectorServer( +export function generateDatacenterOtelCollector( context: TemplateContext, - dcName: string, + datacenter: DatacenterConfig, ) { const clickhouseHost = context.config.networkMode === "host" ? "127.0.0.1" : "clickhouse"; + const prometheusHost = + context.config.networkMode === "host" ? "127.0.0.1" : "prometheus"; + + // Build scrape configs for all engines in this datacenter + const scrapeConfigs: any[] = []; + for (let i = 0; i < datacenter.engines; i++) { + const engineHost = context.getServiceHost( + "rivet-engine", + datacenter.name, + i, + ); + scrapeConfigs.push({ + job_name: `rivet-engine-${datacenter.name}-${i}`, + scrape_interval: "15s", + static_configs: [ + { + targets: + context.config.networkMode === "host" + ? ["host.docker.internal:6430"] + : [`${engineHost}:6430`], + }, + ], + }); + } + const otelConfig = { receivers: { otlp: { @@ -14,6 +40,14 @@ export function generateDatacenterOtelCollectorServer( grpc: { endpoint: "0.0.0.0:4317", }, + http: { + endpoint: "0.0.0.0:4318", + }, + }, + }, + prometheus: { + config: { + scrape_configs: scrapeConfigs, }, }, }, @@ -27,7 +61,7 @@ export function generateDatacenterOtelCollectorServer( }, { key: "rivet.datacenter", - value: dcName, + value: datacenter.name, action: "upsert", }, ], @@ -50,23 +84,6 @@ export function generateDatacenterOtelCollectorServer( logs_table_name: "otel_logs", traces_table_name: "otel_traces", timeout: "5s", - metrics_tables: { - gauge: { - name: "otel_metrics_gauge", - }, - sum: { - name: "otel_metrics_sum", - }, - summary: { - name: "otel_metrics_summary", - }, - histogram: { - name: "otel_metrics_histogram", - }, - exponential_histogram: { - name: "otel_metrics_exp_histogram", - }, - }, retry_on_failure: { enabled: true, initial_interval: "5s", @@ -74,8 +91,22 @@ export function generateDatacenterOtelCollectorServer( max_elapsed_time: "300s", }, }, + prometheusremotewrite: { + endpoint: `http://${prometheusHost}:9090/api/v1/write`, + tls: { + insecure: true, + }, + resource_to_telemetry_conversion: { + enabled: true, + }, + }, }, service: { + // telemetry: { + // logs: { + // level: "debug", + // }, + // }, pipelines: { logs: { receivers: ["otlp"], @@ -88,9 +119,9 @@ export function generateDatacenterOtelCollectorServer( exporters: ["clickhouse"], }, metrics: { - receivers: ["otlp"], + receivers: ["prometheus"], processors: ["resource", "batch"], - exporters: ["clickhouse"], + exporters: ["prometheusremotewrite"], }, }, }, @@ -99,8 +130,8 @@ export function generateDatacenterOtelCollectorServer( const yamlContent = yaml.dump(otelConfig); context.writeDatacenterServiceFile( - "otel-collector-server", - dcName, + "otel-collector", + datacenter.name, "config.yaml", yamlContent, ); diff --git a/engine/packages/api-builder/src/metrics.rs b/engine/packages/api-builder/src/metrics.rs index 02d3aa5efb..0b2cd06bc2 100644 --- a/engine/packages/api-builder/src/metrics.rs +++ b/engine/packages/api-builder/src/metrics.rs @@ -1,29 +1,33 @@ -use rivet_metrics::{ - BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-api-builder"); - // TODO: Request body size // TODO: Response body size - // Keys "method", "path" - pub static ref API_REQUEST_PENDING: UpDownCounter = METER.i64_up_down_counter("rivet_api_request_pending") - .with_description("Total number of requests in progress.") - .build(); - - /// Expected attributes: "method", "path" - pub static ref API_REQUEST_TOTAL: Counter = METER.u64_counter("rivet_api_request_total") - .with_description("Total number of requests.") - .build(); - /// Expected attributes: "method", "path", "status", "error_code" - pub static ref API_REQUEST_DURATION: Histogram = METER.f64_histogram("rivet_api_request_duration") - .with_description("Duration of API requests.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "method", "path", "status", "error_code" - pub static ref API_REQUEST_ERRORS: Counter = METER.u64_counter("rivet_api_request_errors") - .with_description("All errors made to this request.") - .build(); + pub static ref API_REQUEST_PENDING: IntGaugeVec = + register_int_gauge_vec_with_registry!( + "api_request_pending", + "Total number of requests in progress.", + &["method", "path"], + *REGISTRY + ).unwrap(); + pub static ref API_REQUEST_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "api_request_total", + "Total number of requests.", + &["method", "path"], + *REGISTRY + ).unwrap(); + pub static ref API_REQUEST_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "api_request_duration", + "Duration of API requests.", + &["method", "path", "status", "error_code"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref API_REQUEST_ERRORS: IntCounterVec = register_int_counter_vec_with_registry!( + "api_request_errors", + "All errors made to this request.", + &["method", "path", "status", "error_code"], + *REGISTRY, + ) + .unwrap(); } diff --git a/engine/packages/api-builder/src/middleware.rs b/engine/packages/api-builder/src/middleware.rs index df834fe88d..d587e8212b 100644 --- a/engine/packages/api-builder/src/middleware.rs +++ b/engine/packages/api-builder/src/middleware.rs @@ -10,7 +10,6 @@ use axum::{ }; use hyper::header::HeaderName; use opentelemetry::trace::TraceContextExt; -use rivet_metrics::KeyValue; use tower_http::trace::TraceLayer; use tracing::Instrument; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -99,20 +98,12 @@ pub async fn http_logging_middleware( ); // Metrics - metrics::API_REQUEST_PENDING.add( - 1, - &[ - KeyValue::new("method", method.to_string()), - KeyValue::new("path", path.clone()), - ], - ); - metrics::API_REQUEST_TOTAL.add( - 1, - &[ - KeyValue::new("method", method.to_string()), - KeyValue::new("path", path.clone()), - ], - ); + metrics::API_REQUEST_PENDING + .with_label_values(&[method.as_str(), path.as_str()]) + .inc(); + metrics::API_REQUEST_TOTAL + .with_label_values(&[method.as_str(), path.as_str()]) + .inc(); // Clone values for the async block let method_clone = method.clone(); @@ -186,7 +177,7 @@ pub async fn http_logging_middleware( ); // Update metrics - metrics::API_REQUEST_PENDING.add(-1, &[KeyValue::new("method", method_clone.to_string()), KeyValue::new("path", path_clone.clone())]); + metrics::API_REQUEST_PENDING.with_label_values(&[method_clone.as_str(), path_clone.as_str()]).dec(); let error_code: String = if status.is_success() { String::new() @@ -195,26 +186,14 @@ pub async fn http_logging_middleware( } else { String::new() }; - metrics::API_REQUEST_DURATION.record( - duration, - &[ - KeyValue::new("method", method_clone.to_string()), - KeyValue::new("path", path_clone.clone()), - KeyValue::new("status", status.to_string()), - KeyValue::new("error_code", error_code.clone()), - ], - ); + metrics::API_REQUEST_DURATION + .with_label_values(&[method_clone.as_str(), path_clone.as_str(), status.as_str(), error_code.as_str()]) + .observe(duration); if !status.is_success() { - metrics::API_REQUEST_ERRORS.add( - 1, - &[ - KeyValue::new("method", method_clone.to_string()), - KeyValue::new("path", path_clone.clone()), - KeyValue::new("status", status.to_string()), - KeyValue::new("error_code", error_code), - ], - ); + metrics::API_REQUEST_ERRORS + .with_label_values(&[method_clone.as_str(), path_clone.as_str(), status.as_str(), error_code.as_str()]) + .inc(); } response diff --git a/engine/packages/cache/src/driver.rs b/engine/packages/cache/src/driver.rs index 1f169fd5b0..66d1e06980 100644 --- a/engine/packages/cache/src/driver.rs +++ b/engine/packages/cache/src/driver.rs @@ -7,8 +7,6 @@ use moka::future::{Cache, CacheBuilder}; use serde::{Serialize, de::DeserializeOwned}; use tracing::Instrument; -use rivet_metrics::KeyValue; - use crate::{errors::Error, metrics}; /// Type alias for cache values stored as bytes @@ -268,11 +266,13 @@ impl InMemoryDriver { keys: Vec, ) -> Result<(), Error> { let cache = self.cache.clone(); - let base_key = base_key.to_string(); - metrics::CACHE_PURGE_REQUEST_TOTAL.add(1, &[KeyValue::new("key", base_key.clone())]); + metrics::CACHE_PURGE_REQUEST_TOTAL + .with_label_values(&[base_key]) + .inc(); metrics::CACHE_PURGE_VALUE_TOTAL - .add(keys.len() as u64, &[KeyValue::new("key", base_key.clone())]); + .with_label_values(&[base_key]) + .inc_by(keys.len() as u64); // Async block for metrics async { diff --git a/engine/packages/cache/src/metrics.rs b/engine/packages/cache/src/metrics.rs index 79f6b99bac..5a41e43cc9 100644 --- a/engine/packages/cache/src/metrics.rs +++ b/engine/packages/cache/src/metrics.rs @@ -1,34 +1,46 @@ -use rivet_metrics::otel::{global::*, metrics::*}; +use rivet_metrics::{REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-cache"); - - /// Expected attributes: "key" - pub static ref CACHE_REQUEST_TOTAL: Counter = METER.u64_counter("rivet_cache_request_total") - .with_description("Total number of cache requests.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_REQUEST_ERRORS: Counter = METER.u64_counter("rivet_cache_request_errors") - .with_description("Total number of cache request errors.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_PURGE_REQUEST_TOTAL: Counter = METER.u64_counter("rivet_cache_purge_request_total") - .with_description("Total number of cache purge requests.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_VALUE_TOTAL: Counter = METER.u64_counter("rivet_cache_value_total") - .with_description("Total number of cache values requested.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_VALUE_MISS_TOTAL: Counter = METER.u64_counter("rivet_cache_value_miss_total") - .with_description("Total number of cache value misses.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_VALUE_EMPTY_TOTAL: Counter = METER.u64_counter("rivet_cache_value_empty_total") - .with_description("Total number of cache values that were requested but not resolved by cache nor getter.") - .build(); - /// Expected attributes: "key" - pub static ref CACHE_PURGE_VALUE_TOTAL: Counter = METER.u64_counter("rivet_cache_purge_value_total") - .with_description("Total number of cache values purged.") - .build(); + pub static ref CACHE_REQUEST_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_request_total", + "Total number of cache requests.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_REQUEST_ERRORS: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_request_errors", + "Total number of cache request errors.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_PURGE_REQUEST_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_purge_request_total", + "Total number of cache purge requests.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_VALUE_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_value_total", + "Total number of cache values requested.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_VALUE_MISS_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_value_miss_total", + "Total number of cache value misses.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_VALUE_EMPTY_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_value_empty_total", + "Total number of cache values that were requested but not resolved by cache nor getter.", + &["key"], + *REGISTRY + ).unwrap(); + pub static ref CACHE_PURGE_VALUE_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "cache_purge_value_total", + "Total number of cache values purged.", + &["key"], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/cache/src/req_config.rs b/engine/packages/cache/src/req_config.rs index ff42babff9..16640466e6 100644 --- a/engine/packages/cache/src/req_config.rs +++ b/engine/packages/cache/src/req_config.rs @@ -6,7 +6,6 @@ use tracing::Instrument; use super::*; use crate::{errors::Error, metrics}; -use rivet_metrics::KeyValue; /// Config specifying how cached values will behave. #[derive(Clone)] @@ -127,9 +126,12 @@ impl RequestConfig { return Ok(Vec::new()); } - metrics::CACHE_REQUEST_TOTAL.add(1, &[KeyValue::new("key", base_key.clone())]); + metrics::CACHE_REQUEST_TOTAL + .with_label_values(&[base_key.as_str()]) + .inc(); metrics::CACHE_VALUE_TOTAL - .add(keys.len() as u64, &[KeyValue::new("key", base_key.clone())]); + .with_label_values(&[base_key.as_str()]) + .inc_by(keys.len() as u64); // Build context. // @@ -179,10 +181,9 @@ impl RequestConfig { let remaining_keys = ctx.unresolved_keys(); let unresolved_len = remaining_keys.len(); - metrics::CACHE_VALUE_MISS_TOTAL.add( - unresolved_len as u64, - &[KeyValue::new("key", base_key.clone())], - ); + metrics::CACHE_VALUE_MISS_TOTAL + .with_label_values(&[base_key.as_str()]) + .inc_by(unresolved_len as u64); ctx = getter(ctx, remaining_keys).await.map_err(Error::Getter)?; @@ -245,10 +246,9 @@ impl RequestConfig { } } - metrics::CACHE_VALUE_EMPTY_TOTAL.add( - ctx.unresolved_keys().len() as u64, - &[KeyValue::new("key", base_key.clone())], - ); + metrics::CACHE_VALUE_EMPTY_TOTAL + .with_label_values(&[base_key]) + .inc_by(ctx.unresolved_keys().len() as u64); Ok(ctx.into_values()) } @@ -258,7 +258,9 @@ impl RequestConfig { "failed to read batch keys from cache, falling back to getter" ); - metrics::CACHE_REQUEST_ERRORS.add(1, &[KeyValue::new("key", base_key.clone())]); + metrics::CACHE_REQUEST_ERRORS + .with_label_values(&[base_key]) + .inc(); // Fall back to the getter since we can't fetch the value from // the cache diff --git a/engine/packages/config/src/config/metrics.rs b/engine/packages/config/src/config/metrics.rs new file mode 100644 index 0000000000..fba9cb0271 --- /dev/null +++ b/engine/packages/config/src/config/metrics.rs @@ -0,0 +1,22 @@ +use std::net::IpAddr; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Configuration for the metrics service. +#[derive(Debug, Serialize, Deserialize, Clone, Default, JsonSchema)] +#[serde(rename_all = "snake_case", deny_unknown_fields)] +pub struct Metrics { + pub host: Option, + pub port: Option, +} + +impl Metrics { + pub fn host(&self) -> IpAddr { + self.host.unwrap_or(crate::defaults::hosts::METRICS) + } + + pub fn port(&self) -> u16 { + self.port.unwrap_or(crate::defaults::ports::METRICS) + } +} diff --git a/engine/packages/config/src/config/mod.rs b/engine/packages/config/src/config/mod.rs index 439eb45b95..9a9d530764 100644 --- a/engine/packages/config/src/config/mod.rs +++ b/engine/packages/config/src/config/mod.rs @@ -11,6 +11,7 @@ pub mod clickhouse; pub mod db; pub mod guard; pub mod logs; +pub mod metrics; pub mod pegboard; pub mod pubsub; pub mod runtime; @@ -26,6 +27,7 @@ pub use clickhouse::*; pub use db::Database; pub use guard::*; pub use logs::*; +pub use metrics::*; pub use pegboard::*; pub use pubsub::PubSub; pub use runtime::*; @@ -105,6 +107,9 @@ pub struct Root { #[serde(default)] pub runtime: Runtime, + + #[serde(default)] + pub metrics: Metrics, } impl Default for Root { @@ -124,6 +129,7 @@ impl Default for Root { vector_http: None, telemetry: Default::default(), runtime: Default::default(), + metrics: Default::default(), } } } diff --git a/engine/packages/config/src/defaults.rs b/engine/packages/config/src/defaults.rs index ec2fa52c02..da575397e8 100644 --- a/engine/packages/config/src/defaults.rs +++ b/engine/packages/config/src/defaults.rs @@ -3,9 +3,12 @@ pub mod hosts { pub const GUARD: IpAddr = IpAddr::V6(Ipv6Addr::UNSPECIFIED); pub const API_PEER: IpAddr = IpAddr::V6(Ipv6Addr::UNSPECIFIED); + pub const METRICS: IpAddr = IpAddr::V6(Ipv6Addr::UNSPECIFIED); } pub mod ports { pub const GUARD: u16 = 6420; pub const API_PEER: u16 = 6421; + + pub const METRICS: u16 = 6430; } diff --git a/engine/packages/gasoline/src/builder/common/message.rs b/engine/packages/gasoline/src/builder/common/message.rs index 0044c5cee9..4d75165095 100644 --- a/engine/packages/gasoline/src/builder/common/message.rs +++ b/engine/packages/gasoline/src/builder/common/message.rs @@ -1,7 +1,6 @@ use std::{fmt::Display, time::Instant}; use anyhow::Result; -use rivet_metrics::KeyValue; use serde::Serialize; use crate::{builder::BuilderError, ctx::MessageCtx, message::Message, metrics}; @@ -84,20 +83,12 @@ impl MessageBuilder { } let dt = start_instant.elapsed().as_secs_f64(); - metrics::MESSAGE_SEND_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("message_name", M::NAME), - ], - ); - metrics::MESSAGE_PUBLISHED.add( - 1, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("message_name", M::NAME), - ], - ); + metrics::MESSAGE_SEND_DURATION + .with_label_values(&["", M::NAME]) + .observe(dt); + metrics::MESSAGE_PUBLISHED + .with_label_values(&["", M::NAME]) + .inc(); Ok(()) } diff --git a/engine/packages/gasoline/src/builder/common/signal.rs b/engine/packages/gasoline/src/builder/common/signal.rs index f3f8e89f18..020f995b05 100644 --- a/engine/packages/gasoline/src/builder/common/signal.rs +++ b/engine/packages/gasoline/src/builder/common/signal.rs @@ -1,7 +1,6 @@ use std::{fmt::Display, time::Instant}; use anyhow::Result; -use rivet_metrics::KeyValue; use rivet_util::Id; use serde::Serialize; @@ -198,20 +197,12 @@ impl SignalBuilder { } let dt = start_instant.elapsed().as_secs_f64(); - metrics::SIGNAL_SEND_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("signal_name", T::NAME), - ], - ); - metrics::SIGNAL_PUBLISHED.add( - 1, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("signal_name", T::NAME), - ], - ); + metrics::SIGNAL_SEND_DURATION + .with_label_values(&["", T::NAME]) + .observe(dt); + metrics::SIGNAL_PUBLISHED + .with_label_values(&["", T::NAME]) + .inc(); Ok(Some(signal_id)) } diff --git a/engine/packages/gasoline/src/builder/common/workflow.rs b/engine/packages/gasoline/src/builder/common/workflow.rs index 3ca002d418..f8404df9ac 100644 --- a/engine/packages/gasoline/src/builder/common/workflow.rs +++ b/engine/packages/gasoline/src/builder/common/workflow.rs @@ -1,7 +1,6 @@ use std::{fmt::Display, marker::PhantomData, time::Instant}; use anyhow::Result; -use rivet_metrics::KeyValue; use rivet_util::Id; use serde::Serialize; use tracing::Instrument; @@ -142,20 +141,12 @@ where if workflow_id == actual_workflow_id { let dt = start_instant.elapsed().as_secs_f64(); - metrics::WORKFLOW_DISPATCH_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("sub_workflow_name", workflow_name), - ], - ); - metrics::WORKFLOW_DISPATCHED.add( - 1, - &[ - KeyValue::new("workflow_name", ""), - KeyValue::new("sub_workflow_name", workflow_name), - ], - ); + metrics::WORKFLOW_DISPATCH_DURATION + .with_label_values(&["", workflow_name]) + .observe(dt); + metrics::WORKFLOW_DISPATCHED + .with_label_values(&["", workflow_name]) + .inc(); } Ok(actual_workflow_id) diff --git a/engine/packages/gasoline/src/builder/workflow/lupe.rs b/engine/packages/gasoline/src/builder/workflow/lupe.rs index 01b3278f2c..1d491633a7 100644 --- a/engine/packages/gasoline/src/builder/workflow/lupe.rs +++ b/engine/packages/gasoline/src/builder/workflow/lupe.rs @@ -1,7 +1,6 @@ use std::time::Instant; use anyhow::Result; -use rivet_metrics::KeyValue; use serde::{Serialize, de::DeserializeOwned}; use tracing::Instrument; @@ -264,10 +263,9 @@ impl<'a, S: Serialize + DeserializeOwned> LoopBuilder<'a, S> { iteration_branch.cursor().check_clear()?; let dt = start_instant.elapsed().as_secs_f64(); - metrics::LOOP_ITERATION_DURATION.record( - dt - dt2, - &[KeyValue::new("workflow_name", ctx.name().to_string())], - ); + metrics::LOOP_ITERATION_DURATION + .with_label_values(&[&ctx.name().to_string()]) + .observe(dt - dt2); if let Some(res) = res { break res; diff --git a/engine/packages/gasoline/src/builder/workflow/message.rs b/engine/packages/gasoline/src/builder/workflow/message.rs index 34cc21d5e9..5ee8ba9a2c 100644 --- a/engine/packages/gasoline/src/builder/workflow/message.rs +++ b/engine/packages/gasoline/src/builder/workflow/message.rs @@ -1,7 +1,6 @@ use std::{fmt::Display, time::Instant}; use anyhow::Result; -use rivet_metrics::KeyValue; use serde::Serialize; use crate::{ @@ -123,20 +122,12 @@ impl<'a, M: Message> MessageBuilder<'a, M> { } let dt = start_instant.elapsed().as_secs_f64(); - metrics::MESSAGE_SEND_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", self.ctx.name().to_string()), - KeyValue::new("message_name", M::NAME), - ], - ); - metrics::MESSAGE_PUBLISHED.add( - 1, - &[ - KeyValue::new("workflow_name", self.ctx.name().to_string()), - KeyValue::new("message_name", M::NAME), - ], - ); + metrics::MESSAGE_SEND_DURATION + .with_label_values(&[self.ctx.name(), M::NAME]) + .observe(dt); + metrics::MESSAGE_PUBLISHED + .with_label_values(&[self.ctx.name(), M::NAME]) + .inc(); } // Move to next event diff --git a/engine/packages/gasoline/src/builder/workflow/signal.rs b/engine/packages/gasoline/src/builder/workflow/signal.rs index bd476a4f4b..7f8fce7378 100644 --- a/engine/packages/gasoline/src/builder/workflow/signal.rs +++ b/engine/packages/gasoline/src/builder/workflow/signal.rs @@ -1,7 +1,6 @@ use std::{fmt::Display, time::Instant}; use anyhow::Result; -use rivet_metrics::KeyValue; use rivet_util::Id; use serde::Serialize; @@ -256,20 +255,12 @@ impl<'a, T: Signal + Serialize> SignalBuilder<'a, T> { } let dt = start_instant.elapsed().as_secs_f64(); - metrics::SIGNAL_SEND_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", self.ctx.name().to_string()), - KeyValue::new("signal_name", T::NAME), - ], - ); - metrics::SIGNAL_PUBLISHED.add( - 1, - &[ - KeyValue::new("workflow_name", self.ctx.name().to_string()), - KeyValue::new("signal_name", T::NAME), - ], - ); + metrics::SIGNAL_SEND_DURATION + .with_label_values(&[self.ctx.name(), T::NAME]) + .observe(dt); + metrics::SIGNAL_PUBLISHED + .with_label_values(&[self.ctx.name(), T::NAME]) + .inc(); signal_id }; diff --git a/engine/packages/gasoline/src/builder/workflow/sub_workflow.rs b/engine/packages/gasoline/src/builder/workflow/sub_workflow.rs index cca32f4319..31c5ae8f34 100644 --- a/engine/packages/gasoline/src/builder/workflow/sub_workflow.rs +++ b/engine/packages/gasoline/src/builder/workflow/sub_workflow.rs @@ -2,7 +2,6 @@ use std::{fmt::Display, marker::PhantomData, sync::Arc, time::Instant}; use anyhow::Result; use futures_util::StreamExt; -use rivet_metrics::KeyValue; use rivet_util::Id; use serde::Serialize; use tracing::Instrument; @@ -185,20 +184,12 @@ where if sub_workflow_id == actual_sub_workflow_id { let dt = start_instant.elapsed().as_secs_f64(); - metrics::WORKFLOW_DISPATCH_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", ctx.name().to_string()), - KeyValue::new("sub_workflow_name", sub_workflow_name), - ], - ); - metrics::WORKFLOW_DISPATCHED.add( - 1, - &[ - KeyValue::new("workflow_name", ctx.name().to_string()), - KeyValue::new("sub_workflow_name", sub_workflow_name), - ], - ); + metrics::WORKFLOW_DISPATCH_DURATION + .with_label_values(&[ctx.name(), sub_workflow_name]) + .observe(dt); + metrics::WORKFLOW_DISPATCHED + .with_label_values(&[ctx.name(), sub_workflow_name]) + .inc(); } actual_sub_workflow_id diff --git a/engine/packages/gasoline/src/ctx/common.rs b/engine/packages/gasoline/src/ctx/common.rs index b510bfcfa6..6fd373a7cf 100644 --- a/engine/packages/gasoline/src/ctx/common.rs +++ b/engine/packages/gasoline/src/ctx/common.rs @@ -2,7 +2,6 @@ use std::time::{Duration, Instant}; use anyhow::Result; use futures_util::StreamExt; -use rivet_metrics::KeyValue; use rivet_util::Id; /// Time to delay a workflow from retrying after an error @@ -92,8 +91,11 @@ where // Record metrics crate::metrics::OPERATION_PENDING - .add(1, &[KeyValue::new("operation_name", I::Operation::NAME)]); - crate::metrics::OPERATION_TOTAL.add(1, &[KeyValue::new("operation_name", I::Operation::NAME)]); + .with_label_values(&[I::Operation::NAME]) + .inc(); + crate::metrics::OPERATION_TOTAL + .with_label_values(&[I::Operation::NAME]) + .inc(); let start_instant = Instant::now(); @@ -118,13 +120,9 @@ where Ok(Err(err)) => { let error_code_str = err.to_string(); - crate::metrics::OPERATION_ERRORS.add( - 1, - &[ - KeyValue::new("operation_name", I::Operation::NAME), - KeyValue::new("error", error_code_str.clone()), - ], - ); + crate::metrics::OPERATION_ERRORS + .with_label_values(&[I::Operation::NAME, error_code_str.as_str()]) + .inc(); error_code_str } @@ -135,14 +133,11 @@ where // Other request metrics let dt = start_instant.elapsed().as_secs_f64(); crate::metrics::OPERATION_PENDING - .add(-1, &[KeyValue::new("operation_name", I::Operation::NAME)]); - crate::metrics::OPERATION_DURATION.record( - dt, - &[ - KeyValue::new("operation_name", I::Operation::NAME), - KeyValue::new("error", error_code_str.clone()), - ], - ); + .with_label_values(&[I::Operation::NAME]) + .dec(); + crate::metrics::OPERATION_DURATION + .with_label_values(&[I::Operation::NAME, error_code_str.as_str()]) + .observe(dt); } let res = res?; diff --git a/engine/packages/gasoline/src/ctx/listen.rs b/engine/packages/gasoline/src/ctx/listen.rs index b00c9d598f..1161072164 100644 --- a/engine/packages/gasoline/src/ctx/listen.rs +++ b/engine/packages/gasoline/src/ctx/listen.rs @@ -1,4 +1,3 @@ -use rivet_metrics::KeyValue; use std::ops::Deref; use crate::{ @@ -71,13 +70,9 @@ impl<'a> ListenCtx<'a> { let now = rivet_util::timestamp::now(); for signal in &signals { let recv_lag = (now as f64 - signal.create_ts as f64) / 1000.0; - metrics::SIGNAL_RECV_LAG.record( - recv_lag, - &[ - KeyValue::new("workflow_name", self.ctx.name().to_string()), - KeyValue::new("signal_name", signal.signal_name.clone()), - ], - ); + metrics::SIGNAL_RECV_LAG + .with_label_values(&[self.ctx.name(), signal.signal_name.as_str()]) + .observe(recv_lag); if recv_lag > 3.0 { // We print an error here so the trace of this workflow does not get dropped diff --git a/engine/packages/gasoline/src/ctx/workflow.rs b/engine/packages/gasoline/src/ctx/workflow.rs index 5f6147d864..30dafb7aa1 100644 --- a/engine/packages/gasoline/src/ctx/workflow.rs +++ b/engine/packages/gasoline/src/ctx/workflow.rs @@ -13,8 +13,6 @@ use tokio::sync::{Mutex, watch}; use tracing::Instrument; use tracing_opentelemetry::OpenTelemetrySpanExt; -use rivet_metrics::KeyValue; - use crate::{ activity::{Activity, ActivityInput}, builder::{WorkflowRepr, workflow as builder}, @@ -214,22 +212,18 @@ impl WorkflowCtx { // finish. This workflow will be retried when the sub workflow completes let wake_sub_workflow = err.sub_workflow(); + let err_str = err.to_string(); + if err.is_recoverable() && !err.is_retryable() { tracing::debug!(?err, "workflow sleeping"); } else { tracing::error!(?err, "workflow error"); - metrics::WORKFLOW_ERRORS.add( - 1, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("error", err.to_string()), - ], - ); + metrics::WORKFLOW_ERRORS + .with_label_values(&[self.name.as_str(), err_str.as_str()]) + .inc(); } - let err_str = err.to_string(); - let mut retries = 0; let mut interval = tokio::time::interval(DB_ACTION_RETRY); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -342,14 +336,9 @@ impl WorkflowCtx { }, )?; - metrics::ACTIVITY_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("activity_name", A::NAME), - KeyValue::new("error_code", ""), - ], - ); + metrics::ACTIVITY_DURATION + .with_label_values(&[self.name.as_str(), A::NAME, ""]) + .observe(dt); Ok(output) } @@ -381,23 +370,13 @@ impl WorkflowCtx { .unwrap_or_default(); if !is_recoverable { - metrics::ACTIVITY_ERRORS.add( - 1, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("activity_name", A::NAME), - KeyValue::new("error_code", err_str.clone()), - ], - ); + metrics::ACTIVITY_ERRORS + .with_label_values(&[self.name.as_str(), A::NAME, err_str.as_str()]) + .inc(); } - metrics::ACTIVITY_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("activity_name", A::NAME), - KeyValue::new("error_code", err_str.clone()), - ], - ); + metrics::ACTIVITY_DURATION + .with_label_values(&[self.name.as_str(), A::NAME, err_str.as_str()]) + .observe(dt); Err(WorkflowError::ActivityFailure(A::NAME, err, 0)) } @@ -421,22 +400,12 @@ impl WorkflowCtx { ) .await?; - metrics::ACTIVITY_ERRORS.add( - 1, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("activity_name", A::NAME), - KeyValue::new("error_code", err_str.clone()), - ], - ); - metrics::ACTIVITY_DURATION.record( - dt, - &[ - KeyValue::new("workflow_name", self.name.clone()), - KeyValue::new("activity_name", A::NAME), - KeyValue::new("error_code", err_str.clone()), - ], - ); + metrics::ACTIVITY_ERRORS + .with_label_values(&[self.name.as_str(), A::NAME, err_str.as_str()]) + .inc(); + metrics::ACTIVITY_DURATION + .with_label_values(&[self.name.as_str(), A::NAME, err_str.as_str()]) + .observe(dt); Err(err) } diff --git a/engine/packages/gasoline/src/db/kv/mod.rs b/engine/packages/gasoline/src/db/kv/mod.rs index bd195e921c..8cdb7c194e 100644 --- a/engine/packages/gasoline/src/db/kv/mod.rs +++ b/engine/packages/gasoline/src/db/kv/mod.rs @@ -23,8 +23,6 @@ use universaldb::{ value::Value, }; -use rivet_metrics::KeyValue; - use super::{BumpSubSubject, Database, PulledWorkflowData, SignalData, WorkflowData}; use crate::{ error::{WorkflowError, WorkflowResult}, @@ -686,10 +684,9 @@ impl Database for DatabaseKv { for (key, count) in entries { match key.metric { keys::metric::GaugeMetric::WorkflowActive(workflow_name) => { - metrics::WORKFLOW_ACTIVE.record( - count as u64, - &[KeyValue::new("workflow_name", workflow_name.clone())], - ); + metrics::WORKFLOW_ACTIVE + .with_label_values(&[workflow_name.as_str()]) + .set(count as i64); if let Some(entry) = total_workflow_counts .iter_mut() @@ -701,10 +698,9 @@ impl Database for DatabaseKv { } } keys::metric::GaugeMetric::WorkflowSleeping(workflow_name) => { - metrics::WORKFLOW_SLEEPING.record( - count as u64, - &[KeyValue::new("workflow_name", workflow_name.clone())], - ); + metrics::WORKFLOW_SLEEPING + .with_label_values(&[workflow_name.as_str()]) + .set(count as i64); if let Some(entry) = total_workflow_counts .iter_mut() @@ -716,13 +712,9 @@ impl Database for DatabaseKv { } } keys::metric::GaugeMetric::WorkflowDead(workflow_name, error) => { - metrics::WORKFLOW_DEAD.record( - count as u64, - &[ - KeyValue::new("workflow_name", workflow_name.clone()), - KeyValue::new("error", error), - ], - ); + metrics::WORKFLOW_DEAD + .with_label_values(&[workflow_name.as_str(), error.as_str()]) + .set(count as i64); if let Some(entry) = total_workflow_counts .iter_mut() @@ -745,16 +737,16 @@ impl Database for DatabaseKv { } keys::metric::GaugeMetric::SignalPending(signal_name) => { metrics::SIGNAL_PENDING - .record(count as u64, &[KeyValue::new("signal_name", signal_name)]); + .with_label_values(&[signal_name.as_str()]) + .set(count as i64); } } } for (workflow_name, count) in total_workflow_counts { - metrics::WORKFLOW_TOTAL.record( - count as u64, - &[KeyValue::new("workflow_name", workflow_name.clone())], - ); + metrics::WORKFLOW_TOTAL + .with_label_values(&[workflow_name.as_str()]) + .set(count as i64); } // Clear lock @@ -778,10 +770,9 @@ impl Database for DatabaseKv { #[tracing::instrument(skip_all)] async fn update_worker_ping(&self, worker_id: Id) -> WorkflowResult<()> { - metrics::WORKER_LAST_PING.record( - rivet_util::timestamp::now() as u64, - &[KeyValue::new("worker_id", worker_id.to_string())], - ); + metrics::WORKER_LAST_PING + .with_label_values(&[&worker_id.to_string()]) + .set(rivet_util::timestamp::now()); self.pools .udb() @@ -998,10 +989,9 @@ impl Database for DatabaseKv { .map_err(WorkflowError::Udb)?; let dt = start_instant.elapsed().as_secs_f64(); - metrics::FIND_WORKFLOWS_DURATION.record( - dt, - &[KeyValue::new("workflow_name", workflow_name.to_string())], - ); + metrics::FIND_WORKFLOWS_DURATION + .with_label_values(&[workflow_name]) + .observe(dt); Ok(workflow_id) } @@ -1055,7 +1045,7 @@ impl Database for DatabaseKv { calc_pull_ratio((cpu_usage_ratio * 1000.0) as u64, 500, 1000, 800, 50); // Record load shedding ratio metric - metrics::LOAD_SHEDDING_RATIO.record(load_shed_ratio_x1000 as f64 / 1000.0, &[]); + metrics::LOAD_SHEDDING_RATIO.observe(load_shed_ratio_x1000 as f64 / 1000.0); let active_worker_subspace_start = tx.pack( &keys::worker::ActiveWorkerIdxKey::subspace(active_workers_after), @@ -1331,9 +1321,11 @@ impl Database for DatabaseKv { let worker_id_str = worker_id.to_string(); let dt = start_instant.elapsed().as_secs_f64(); metrics::LAST_PULL_WORKFLOWS_DURATION - .record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]); + .with_label_values(&[worker_id_str.as_str()]) + .set(dt); metrics::PULL_WORKFLOWS_DURATION - .record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]); + .with_label_values(&[worker_id_str.as_str()]) + .observe(dt); if leased_workflows.is_empty() { return Ok(Vec::new()); @@ -1675,15 +1667,17 @@ impl Database for DatabaseKv { let dt2 = start_instant2.elapsed().as_secs_f64(); let dt = start_instant.elapsed().as_secs_f64(); metrics::LAST_PULL_WORKFLOWS_FULL_DURATION - .record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]); + .with_label_values(&[worker_id_str.as_str()]) + .set(dt); metrics::PULL_WORKFLOWS_FULL_DURATION - .record(dt, &[KeyValue::new("worker_id", worker_id_str.clone())]); - metrics::LAST_PULL_WORKFLOWS_HISTORY_DURATION.record( - dt2 as u64, - &[KeyValue::new("worker_id", worker_id_str.clone())], - ); + .with_label_values(&[worker_id_str.as_str()]) + .observe(dt); + metrics::LAST_PULL_WORKFLOWS_HISTORY_DURATION + .with_label_values(&[worker_id_str.as_str()]) + .set(dt2); metrics::PULL_WORKFLOWS_HISTORY_DURATION - .record(dt2, &[KeyValue::new("worker_id", worker_id_str.clone())]); + .with_label_values(&[worker_id_str.as_str()]) + .observe(dt2); Ok(pulled_workflows) } @@ -1853,10 +1847,9 @@ impl Database for DatabaseKv { } let dt = start_instant.elapsed().as_secs_f64(); - metrics::COMPLETE_WORKFLOW_DURATION.record( - dt, - &[KeyValue::new("workflow_name", workflow_name.to_string())], - ); + metrics::COMPLETE_WORKFLOW_DURATION + .with_label_values(&[workflow_name]) + .observe(dt); Ok(()) } @@ -2021,10 +2014,9 @@ impl Database for DatabaseKv { self.bump(BumpSubSubject::Worker); let dt = start_instant.elapsed().as_secs_f64(); - metrics::COMMIT_WORKFLOW_DURATION.record( - dt, - &[KeyValue::new("workflow_name", workflow_name.to_string())], - ); + metrics::COMMIT_WORKFLOW_DURATION + .with_label_values(&[workflow_name]) + .observe(dt); Ok(()) } diff --git a/engine/packages/gasoline/src/db/kv/system.rs b/engine/packages/gasoline/src/db/kv/system.rs index ab325be1d9..927c6bd433 100644 --- a/engine/packages/gasoline/src/db/kv/system.rs +++ b/engine/packages/gasoline/src/db/kv/system.rs @@ -95,7 +95,7 @@ impl SystemInfo { .unwrap_or(0.0) }; - crate::metrics::CPU_USAGE.record(total as f64, &[]); + crate::metrics::CPU_USAGE.observe(total as f64); total / cpu_max } diff --git a/engine/packages/gasoline/src/metrics.rs b/engine/packages/gasoline/src/metrics.rs index 3015ddccc9..49ec405fcb 100644 --- a/engine/packages/gasoline/src/metrics.rs +++ b/engine/packages/gasoline/src/metrics.rs @@ -1,161 +1,221 @@ -use rivet_metrics::{ - BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-gasoline"); - /// Expected attributes: "worker_id" - pub static ref WORKER_LAST_PING: Gauge = METER.u64_gauge("rivet_gasoline_worker_last_ping") - .with_description("Last ping of a worker as a unix ts.") - .build(); - /// Expected attributes: "worker_id" - pub static ref LAST_PULL_WORKFLOWS_DURATION: Gauge = METER.f64_gauge("rivet_gasoline_last_pull_workflows_duration") - .with_description("Last duration of pulling workflow data.") - .build(); - /// Expected attributes: "worker_id" - pub static ref LAST_PULL_WORKFLOWS_HISTORY_DURATION: Gauge = METER.u64_gauge("rivet_gasoline_last_pull_workflows_history_duration") - .with_description("Last duration of pulling workflow histories.") - .build(); - /// Expected attributes: "worker_id" - pub static ref LAST_PULL_WORKFLOWS_FULL_DURATION: Gauge = METER.f64_gauge("rivet_gasoline_last_pull_workflows_full_duration") - .with_description("Last duration of pulling workflow data and history.") - .build(); - /// Expected attributes: "worker_id" - pub static ref PULL_WORKFLOWS_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_pull_workflows_duration") - .with_description("Duration of pulling workflow data.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "worker_id" - pub static ref PULL_WORKFLOWS_HISTORY_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_pull_workflows_history_duration") - .with_description("Duration of pulling workflow histories.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "worker_id" - pub static ref PULL_WORKFLOWS_FULL_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_pull_workflows_full_duration") - .with_description("Duration of pulling workflow data and history.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref WORKER_LAST_PING: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_worker_last_ping", + "Last ping of a worker as a unix ts.", + &["worker_id"], + *REGISTRY + ).unwrap(); + pub static ref LAST_PULL_WORKFLOWS_DURATION: GaugeVec = register_gauge_vec_with_registry!( + "gasoline_last_pull_workflows_duration", + "Last duration of pulling workflow data.", + &["worker_id"], + *REGISTRY + ).unwrap(); + pub static ref LAST_PULL_WORKFLOWS_HISTORY_DURATION: GaugeVec = register_gauge_vec_with_registry!( + "gasoline_last_pull_workflows_history_duration", + "Last duration of pulling workflow histories.", + &["worker_id"], + *REGISTRY + ).unwrap(); + pub static ref LAST_PULL_WORKFLOWS_FULL_DURATION: GaugeVec = register_gauge_vec_with_registry!( + "gasoline_last_pull_workflows_full_duration", + "Last duration of pulling workflow data and history.", + &["worker_id"], + *REGISTRY + ).unwrap(); + pub static ref PULL_WORKFLOWS_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_pull_workflows_duration", + "Duration of pulling workflow data.", + &["worker_id"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref PULL_WORKFLOWS_HISTORY_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_pull_workflows_history_duration", + "Duration of pulling workflow histories.", + &["worker_id"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref PULL_WORKFLOWS_FULL_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_pull_workflows_full_duration", + "Duration of pulling workflow data and history.", + &["worker_id"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name" - pub static ref FIND_WORKFLOWS_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_find_workflows_duration") - .with_description("Duration to find a workflow with a given name and tags.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref FIND_WORKFLOWS_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_find_workflows_duration", + "Duration to find a workflow with a given name and tags.", + &["workflow_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name" - pub static ref WORKFLOW_TOTAL: Gauge = METER.u64_gauge("rivet_gasoline_workflow_total") - .with_description("Total workflows.") - .build(); - /// Expected attributes: "workflow_name" - pub static ref WORKFLOW_ACTIVE: Gauge = METER.u64_gauge("rivet_gasoline_workflow_active") - .with_description("Total active workflows.") - .build(); - /// Expected attributes: "workflow_name", "error" - pub static ref WORKFLOW_DEAD: Gauge = METER.u64_gauge("rivet_gasoline_workflow_dead") - .with_description("Total dead workflows.") - .build(); - /// Expected attributes: "workflow_name" - pub static ref WORKFLOW_SLEEPING: Gauge = METER.u64_gauge("rivet_gasoline_workflow_sleeping") - .with_description("Total sleeping workflows.") - .build(); - /// Expected attributes: "workflow_name", "error" - pub static ref WORKFLOW_ERRORS: Counter = METER.u64_counter("rivet_gasoline_workflow_errors") - .with_description("All errors made in a workflow.") - .build(); + pub static ref WORKFLOW_TOTAL: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_workflow_total", + "Total workflows.", + &["workflow_name"], + *REGISTRY + ).unwrap(); + pub static ref WORKFLOW_ACTIVE: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_workflow_active", + "Total active workflows.", + &["workflow_name"], + *REGISTRY + ).unwrap(); + pub static ref WORKFLOW_DEAD: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_workflow_dead", + "Total dead workflows.", + &["workflow_name", "error"], + *REGISTRY + ).unwrap(); + pub static ref WORKFLOW_SLEEPING: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_workflow_sleeping", + "Total sleeping workflows.", + &["workflow_name"], + *REGISTRY + ).unwrap(); + pub static ref WORKFLOW_ERRORS: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_workflow_errors", + "All errors made in a workflow.", + &["workflow_name", "error"], + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name" - pub static ref COMPLETE_WORKFLOW_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_complete_workflow_duration") - .with_description("Duration to complete a workflow with a given name.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "workflow_name" - pub static ref COMMIT_WORKFLOW_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_commit_workflow_duration") - .with_description("Duration to commit a workflow with a given name.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref COMPLETE_WORKFLOW_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_complete_workflow_duration", + "Duration to complete a workflow with a given name.", + &["workflow_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref COMMIT_WORKFLOW_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_commit_workflow_duration", + "Duration to commit a workflow with a given name.", + &["workflow_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name", "activity_name", "error_code" - pub static ref ACTIVITY_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_activity_duration") - .with_description("Total duration of an activity.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "workflow_name", "activity_name", "error_code" - pub static ref ACTIVITY_ERRORS: Counter = METER.u64_counter("rivet_gasoline_activity_errors") - .with_description("All errors made in an activity.") - .build(); + pub static ref ACTIVITY_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_activity_duration", + "Total duration of an activity.", + &["workflow_name", "activity_name", "error_code"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref ACTIVITY_ERRORS: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_activity_errors", + "All errors made in an activity.", + &["workflow_name", "activity_name", "error_code"], + *REGISTRY + ).unwrap(); - /// Expected attributes: "signal_name" - pub static ref SIGNAL_PENDING: Gauge = METER.u64_gauge("rivet_gasoline_signal_pending") - .with_description("Total pending signals.") - .build(); - /// Expected attributes: "workflow_name", "signal_name" - pub static ref SIGNAL_RECV_LAG: Histogram = METER.f64_histogram("rivet_gasoline_signal_recv_lag") - .with_description("Time between the publish timestamp and the timestamp the signal was received.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "workflow_name", "signal_name" - pub static ref SIGNAL_PUBLISHED: Counter = METER.u64_counter("rivet_gasoline_signal_published") - .with_description("Total published signals.") - .build(); - /// Expected attributes: "workflow_name", "signal_name" - pub static ref SIGNAL_SEND_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_signal_send_duration") - .with_description("Total duration of a signal send.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref SIGNAL_PENDING: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_signal_pending", + "Total pending signals.", + &["signal_name"], + *REGISTRY + ).unwrap(); + pub static ref SIGNAL_RECV_LAG: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_signal_recv_lag", + "Time between the publish timestamp and the timestamp the signal was received.", + &["workflow_name", "signal_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref SIGNAL_PUBLISHED: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_signal_published", + "Total published signals.", + &["workflow_name", "signal_name"], + *REGISTRY + ).unwrap(); + pub static ref SIGNAL_SEND_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_signal_send_duration", + "Total duration of a signal send.", + &["workflow_name", "signal_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name", "message_name" - pub static ref MESSAGE_PUBLISHED: Counter = METER.u64_counter("rivet_gasoline_message_published") - .with_description("Total published messages.") - .build(); - /// Expected attributes: "workflow_name", "message_name" - pub static ref MESSAGE_SEND_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_message_send_duration") - .with_description("Total duration of a message send.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref MESSAGE_PUBLISHED: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_message_published", + "Total published messages.", + &["workflow_name", "message_name"], + *REGISTRY + ).unwrap(); + pub static ref MESSAGE_SEND_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_message_send_duration", + "Total duration of a message send.", + &["workflow_name", "message_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name", "sub_workflow_name" - pub static ref WORKFLOW_DISPATCHED: Counter = METER.u64_counter("rivet_gasoline_workflow_dispatched") - .with_description("Total dispatched workflows.") - .build(); - /// Expected attributes: "workflow_name", "sub_workflow_name" - pub static ref WORKFLOW_DISPATCH_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_workflow_dispatch_duration") - .with_description("Total duration of a workflow dispatch.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref WORKFLOW_DISPATCHED: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_workflow_dispatched", + "Total dispatched workflows.", + &["workflow_name", "sub_workflow_name"], + *REGISTRY + ).unwrap(); + pub static ref WORKFLOW_DISPATCH_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_workflow_dispatch_duration", + "Total duration of a workflow dispatch.", + &["workflow_name", "sub_workflow_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); - /// Expected attributes: "workflow_name" - pub static ref LOOP_ITERATION_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_loop_iteration_duration") - .with_description("Total duration of a single loop iteration (excluding its body).") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref LOOP_ITERATION_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_loop_iteration_duration", + "Total duration of a single loop iteration (excluding its body).", + &["workflow_name"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); // MARK: Ops - /// Expected attributes: "operation_name" - pub static ref OPERATION_PENDING: UpDownCounter = METER.i64_up_down_counter("rivet_gasoline_operation_pending") - .with_description("Total number of operation calls in progress.") - .build(); - /// Expected attributes: "operation_name" - pub static ref OPERATION_TOTAL: Counter = METER.u64_counter("rivet_gasoline_operation_total") - .with_description("Total number of operation calls.") - .build(); - /// Expected attributes: "operation_name", "error" - pub static ref OPERATION_DURATION: Histogram = METER.f64_histogram("rivet_gasoline_operation_duration") - .with_description("Total duration of an op call.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "operation_name", "error" - pub static ref OPERATION_ERRORS: Counter = METER.u64_counter("rivet_gasoline_operation_errors") - .with_description("All errors made by this operation.") - .build(); + pub static ref OPERATION_PENDING: IntGaugeVec = register_int_gauge_vec_with_registry!( + "gasoline_operation_pending", + "Total number of operation calls in progress.", + &["operation_name"], + *REGISTRY + ).unwrap(); + pub static ref OPERATION_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_operation_total", + "Total number of operation calls.", + &["operation_name"], + *REGISTRY + ).unwrap(); + pub static ref OPERATION_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "gasoline_operation_duration", + "Total duration of an op call.", + &["operation_name", "error"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref OPERATION_ERRORS: IntCounterVec = register_int_counter_vec_with_registry!( + "gasoline_operation_errors", + "All errors made by this operation.", + &["operation_name", "error"], + *REGISTRY + ).unwrap(); // MARK: Load Shedding - pub static ref CPU_USAGE: Histogram = METER.f64_histogram("rivet_gasoline_cpu_usage") - .with_description("CPU usage (100 per core).") - .with_boundaries(vec![0.0, 0.1, 0.25, 0.5, 1.0, 1.5, 2.0, 4.0, 8.0, 16.0]) - .build(); - pub static ref LOAD_SHEDDING_RATIO: Histogram = METER.f64_histogram("rivet_gasoline_load_shedding_ratio") - .with_description("Load shedding ratio (0-1) based on CPU usage, determining the fraction of workflows to pull.") - .with_boundaries(vec![0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]) - .build(); + pub static ref CPU_USAGE: Histogram = register_histogram_with_registry!( + "gasoline_cpu_usage", + "CPU usage (100 per core).", + vec![0.1, 0.25, 0.5, 1.0, 1.5, 2.0, 4.0, 8.0, 16.0], + *REGISTRY + ).unwrap(); + pub static ref LOAD_SHEDDING_RATIO: Histogram = register_histogram_with_registry!( + "gasoline_load_shedding_ratio", + "Load shedding ratio (0-1) based on CPU usage, determining the fraction of workflows to pull.", + vec![0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/guard-core/src/metrics.rs b/engine/packages/guard-core/src/metrics.rs index a6e270b43a..a383b8adbe 100644 --- a/engine/packages/guard-core/src/metrics.rs +++ b/engine/packages/guard-core/src/metrics.rs @@ -1,64 +1,72 @@ use lazy_static::lazy_static; -use rivet_metrics::{ - BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; lazy_static! { - static ref METER: Meter = meter("rivet-guard"); - // MARK: Internal - /// Has no expected attributes - pub static ref ROUTE_CACHE_COUNT: Gauge = METER.u64_gauge("rivet_guard_route_cache_count") - .with_description("Number of entries in the route cache") - .build(); - /// Has no expected attributes - pub static ref RATE_LIMITER_COUNT: Gauge = METER.u64_gauge("rivet_guard_rate_limiter_count") - .with_description("Number of active rate limiters") - .build(); - /// Has no expected attributes - pub static ref IN_FLIGHT_COUNTER_COUNT: Gauge = METER.u64_gauge("rivet_guard_in_flight_counter_count") - .with_description("Number of active in-flight counters") - .build(); + pub static ref ROUTE_CACHE_COUNT: IntGauge = register_int_gauge_with_registry!( + "guard_route_cache_count", + "Number of entries in the route cache", + *REGISTRY + ).unwrap(); + pub static ref RATE_LIMITER_COUNT: IntGauge = register_int_gauge_with_registry!( + "guard_rate_limiter_count", + "Number of active rate limiters", + *REGISTRY + ).unwrap(); + pub static ref IN_FLIGHT_COUNTER_COUNT: IntGauge = register_int_gauge_with_registry!( + "guard_in_flight_counter_count", + "Number of active in-flight counters", + *REGISTRY + ).unwrap(); // MARK: TCP - /// Has no expected attributes - pub static ref TCP_CONNECTION_TOTAL: Counter = METER.u64_counter("rivet_guard_tcp_connection_total") - .with_description("Total number of TCP connections ever") - .build(); - /// Has no expected attributes - pub static ref TCP_CONNECTION_PENDING: UpDownCounter = METER.i64_up_down_counter("rivet_guard_tcp_connection_pending") - .with_description("Total number of open TCP connections") - .build(); - /// Has no expected attributes - pub static ref TCP_CONNECTION_DURATION: Histogram = METER.f64_histogram("rivet_guard_tcp_connection_duration") - .with_description("TCP connection duration in seconds") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref TCP_CONNECTION_TOTAL: IntCounter = register_int_counter_with_registry!( + "guard_tcp_connection_total", + "Total number of TCP connections ever", + *REGISTRY + ).unwrap(); + pub static ref TCP_CONNECTION_PENDING: IntGauge = register_int_gauge_with_registry!( + "guard_tcp_connection_pending", + "Total number of open TCP connections", + *REGISTRY + ).unwrap(); + pub static ref TCP_CONNECTION_DURATION: Histogram = register_histogram_with_registry!( + "guard_tcp_connection_duration", + "TCP connection duration in seconds", + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); // MARK: Pre-proxy - /// Has no expected attributes - pub static ref RESOLVE_ROUTE_DURATION: Histogram = METER.f64_histogram("rivet_guard_resolve_route_duration") - .with_description("Time to resolve request route in seconds") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref RESOLVE_ROUTE_DURATION: Histogram = register_histogram_with_registry!( + "guard_resolve_route_duration", + "Time to resolve request route in seconds", + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); // MARK: Proxy requests - /// Has no expected attributes - pub static ref PROXY_REQUEST_TOTAL: Counter = METER.u64_counter("rivet_guard_proxy_request_total") - .with_description("Total number of requests to actor") - .build(); - /// Has no expected attributes - pub static ref PROXY_REQUEST_PENDING: UpDownCounter = METER.i64_up_down_counter("rivet_guard_proxy_request_pending") - .with_description("Number of pending requests to actor") - .build(); - /// Expected attributes: "status" - pub static ref PROXY_REQUEST_DURATION: Histogram = METER.f64_histogram("rivet_guard_proxy_request_duration") - .with_description("Request duration in seconds") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "error_type" - pub static ref PROXY_REQUEST_ERROR: Counter = METER.u64_counter("rivet_guard_proxy_request_errors_total") - .with_description("Total number of errors when proxying requests to actor") - .build(); + pub static ref PROXY_REQUEST_TOTAL: IntCounter = register_int_counter_with_registry!( + "guard_proxy_request_total", + "Total number of requests to actor", + *REGISTRY + ).unwrap(); + pub static ref PROXY_REQUEST_PENDING: IntGauge = register_int_gauge_with_registry!( + "guard_proxy_request_pending", + "Number of pending requests to actor", + *REGISTRY + ).unwrap(); + pub static ref PROXY_REQUEST_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "guard_proxy_request_duration", + "Request duration in seconds", + &["status"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref PROXY_REQUEST_ERROR: IntCounterVec = register_int_counter_vec_with_registry!( + "guard_proxy_request_errors_total", + "Total number of errors when proxying requests to actor", + &["error_type"], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/guard-core/src/proxy_service.rs b/engine/packages/guard-core/src/proxy_service.rs index 40d4cfa629..b1fa472e26 100644 --- a/engine/packages/guard-core/src/proxy_service.rs +++ b/engine/packages/guard-core/src/proxy_service.rs @@ -10,7 +10,6 @@ use rand; use rivet_api_builder::{ErrorResponse, RawErrorResponse}; use rivet_api_builder::{RequestIds, X_RIVET_RAY_ID}; use rivet_error::{INTERNAL_ERROR, RivetError}; -use rivet_metrics::KeyValue; use rivet_util::Id; use serde_json; @@ -275,7 +274,7 @@ impl RouteCache { async fn insert(&self, key: u64, result: RouteConfig) { self.cache.insert(key, result).await; - metrics::ROUTE_CACHE_COUNT.record(self.cache.entry_count(), &[]); + metrics::ROUTE_CACHE_COUNT.set(self.cache.entry_count() as i64); } } @@ -584,7 +583,7 @@ impl ProxyState { self.rate_limiters .insert(cache_key, new_limiter.clone()) .await; - metrics::RATE_LIMITER_COUNT.record(self.rate_limiters.entry_count(), &[]); + metrics::RATE_LIMITER_COUNT.set(self.rate_limiters.entry_count() as i64); new_limiter }; @@ -623,7 +622,7 @@ impl ProxyState { self.in_flight_counters .insert(cache_key, new_counter.clone()) .await; - metrics::IN_FLIGHT_COUNTER_COUNT.record(self.in_flight_counters.entry_count(), &[]); + metrics::IN_FLIGHT_COUNTER_COUNT.set(self.in_flight_counters.entry_count() as i64); new_counter }; @@ -766,7 +765,7 @@ impl ProxyService { .await; let duration_secs = start_time.elapsed().as_secs_f64(); - metrics::RESOLVE_ROUTE_DURATION.record(duration_secs, &[]); + metrics::RESOLVE_ROUTE_DURATION.observe(duration_secs); // Resolve target let target = target_res?; @@ -827,8 +826,8 @@ impl ProxyService { }; // Increment metrics - metrics::PROXY_REQUEST_PENDING.add(1, &[]); - metrics::PROXY_REQUEST_TOTAL.add(1, &[]); + metrics::PROXY_REQUEST_PENDING.inc(); + metrics::PROXY_REQUEST_TOTAL.inc(); // Update request context with target info if let Some(actor_id) = actor_id { @@ -851,9 +850,10 @@ impl ProxyService { // Record metrics let duration_secs = start_time.elapsed().as_secs_f64(); metrics::PROXY_REQUEST_DURATION - .record(duration_secs, &[KeyValue::new("status", status.clone())]); + .with_label_values(&[status]) + .observe(duration_secs); - metrics::PROXY_REQUEST_PENDING.add(-1, &[]); + metrics::PROXY_REQUEST_PENDING.dec(); // Release in-flight counter and request ID when done let state_clone = self.state.clone(); @@ -2299,7 +2299,8 @@ impl ProxyService { tracing::error!(?err, "Request failed"); metrics::PROXY_REQUEST_ERROR - .add(1, &[KeyValue::new("error_type", err.to_string())]); + .with_label_values(&[&err.to_string()]) + .inc(); // If we receive an error during a websocket request, we attempt to open the websocket anyway // so we can send the error via websocket instead of http. Most websocket clients don't handle @@ -2688,7 +2689,7 @@ fn err_to_close_frame(err: anyhow::Error, ray_id: Option) -> CloseFrame { }; match code { - CloseCode::Normal => tracing::info!("websocket closed"), + CloseCode::Normal => tracing::debug!("websocket closed"), _ => tracing::error!(?err, "websocket failed"), } diff --git a/engine/packages/guard-core/src/server.rs b/engine/packages/guard-core/src/server.rs index 1f9decbfeb..ef285cca7f 100644 --- a/engine/packages/guard-core/src/server.rs +++ b/engine/packages/guard-core/src/server.rs @@ -98,8 +98,8 @@ pub async fn run_server( port_type_str: String, ) { let connection_start = Instant::now(); - metrics::TCP_CONNECTION_PENDING.add(1, &[]); - metrics::TCP_CONNECTION_TOTAL.add(1, &[]); + metrics::TCP_CONNECTION_PENDING.inc(); + metrics::TCP_CONNECTION_TOTAL.inc(); let io = hyper_util::rt::TokioIo::new(tcp_stream); @@ -124,8 +124,8 @@ pub async fn run_server( tracing::debug!("{} connection dropped: {}", port_type_str, remote_addr); let connection_duration = connection_start.elapsed().as_secs_f64(); - metrics::TCP_CONNECTION_DURATION.record(connection_duration, &[]); - metrics::TCP_CONNECTION_PENDING.add(-1, &[]); + metrics::TCP_CONNECTION_DURATION.observe(connection_duration); + metrics::TCP_CONNECTION_PENDING.dec(); } .instrument(tracing::info_span!(parent: None, "process_connection_task")), ); @@ -175,8 +175,8 @@ pub async fn run_server( // Accept TLS connection in a separate task to avoid ownership issues tokio::spawn(async move { let connection_start = Instant::now(); - metrics::TCP_CONNECTION_PENDING.add(1, &[]); - metrics::TCP_CONNECTION_TOTAL.add(1, &[]); + metrics::TCP_CONNECTION_PENDING.inc(); + metrics::TCP_CONNECTION_TOTAL.inc(); match acceptor_clone .accept(tcp_stream) @@ -215,8 +215,8 @@ pub async fn run_server( } let connection_duration = connection_start.elapsed().as_secs_f64(); - metrics::TCP_CONNECTION_DURATION.record(connection_duration, &[]); - metrics::TCP_CONNECTION_PENDING.add(-1, &[]); + metrics::TCP_CONNECTION_DURATION.observe(connection_duration); + metrics::TCP_CONNECTION_PENDING.dec(); }.instrument(tracing::info_span!(parent: None, "process_tls_connection_task"))); } else { // Fallback to non-TLS handling (useful for testing) diff --git a/engine/packages/metrics/Cargo.toml b/engine/packages/metrics/Cargo.toml index cf04d2a322..da3515473f 100644 --- a/engine/packages/metrics/Cargo.toml +++ b/engine/packages/metrics/Cargo.toml @@ -10,10 +10,11 @@ anyhow.workspace = true console-subscriber.workspace = true hyper.workspace = true lazy_static.workspace = true -opentelemetry = { workspace = true, features = ["metrics"] } +opentelemetry.workspace = true opentelemetry-otlp.workspace = true opentelemetry-semantic-conventions.workspace = true -opentelemetry_sdk = { workspace = true, features = ["metrics"] } +opentelemetry_sdk.workspace = true +prometheus.workspace = true rivet-config.workspace = true rivet-env.workspace = true tokio.workspace = true diff --git a/engine/packages/metrics/src/buckets.rs b/engine/packages/metrics/src/buckets.rs index 720e459ff8..2d589aea2b 100644 --- a/engine/packages/metrics/src/buckets.rs +++ b/engine/packages/metrics/src/buckets.rs @@ -1,22 +1,16 @@ pub const BUCKETS: &[f64] = &[ - // For otel - 0.0, // Added 0.001, 0.0025, // Copied from https://docs.rs/prometheus/latest/src/prometheus/histogram.rs.html#25-27 - 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, // Added - 25.0, 50.0, 100.0, 250.0, 500.0, + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, ]; pub const MICRO_BUCKETS: &[f64] = &[ - // For otel - 0.0, 0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.10, 0.25, 0.5, 1.0, - 2.5, 5.0, 10.0, 25.0, 50.0, + 0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.10, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0, 25.0, 50.0, ]; // Calculated based on the LogHistogram configuration in `packages/common/runtime/src/lib.rs` pub const TASK_POLL_BUCKETS: &[f64] = &[ - // For otel - 0.0, 0.00002, 0.000032768, 0.000065536, diff --git a/engine/packages/metrics/src/lib.rs b/engine/packages/metrics/src/lib.rs index 0095bec63e..97db1989df 100644 --- a/engine/packages/metrics/src/lib.rs +++ b/engine/packages/metrics/src/lib.rs @@ -1,8 +1,11 @@ mod providers; mod buckets; +mod registry; +mod server; -pub use buckets::*; -pub use opentelemetry as otel; -pub use opentelemetry::KeyValue; +pub use buckets::{BUCKETS, MICRO_BUCKETS, TASK_POLL_BUCKETS}; +pub use prometheus; pub use providers::{OtelProviderGuard, init_otel_providers, set_sampler_ratio}; +pub use registry::REGISTRY; +pub use server::run_standalone; diff --git a/engine/packages/metrics/src/providers.rs b/engine/packages/metrics/src/providers.rs index d54da249dc..23d4fbf342 100644 --- a/engine/packages/metrics/src/providers.rs +++ b/engine/packages/metrics/src/providers.rs @@ -1,12 +1,11 @@ // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs // Based off of https://github.com/tokio-rs/tracing-opentelemetry/blob/v0.1.x/examples/opentelemetry-otlp.rs +use opentelemetry::KeyValue; use opentelemetry::trace::{SamplingResult, SpanKind}; -use opentelemetry::{KeyValue, global}; use opentelemetry_otlp::WithExportConfig; use opentelemetry_sdk::{ Resource, - metrics::{MeterProviderBuilder, PeriodicReader, SdkMeterProvider}, trace::{RandomIdGenerator, Sampler, SdkTracerProvider}, }; use opentelemetry_semantic_conventions::{SCHEMA_URL, attribute::SERVICE_VERSION}; @@ -116,34 +115,6 @@ fn init_tracer_provider() -> SdkTracerProvider { .build() } -fn init_meter_provider() -> SdkMeterProvider { - let exporter = opentelemetry_otlp::MetricExporter::builder() - .with_tonic() - .with_temporality(opentelemetry_sdk::metrics::Temporality::Cumulative) - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_endpoint(otel_grpc_endpoint()) - .build() - .unwrap(); - - let reader = PeriodicReader::builder(exporter) - .with_interval(std::time::Duration::from_secs(30)) - .build(); - - // // For debugging in development - // let stdout_reader = - // PeriodicReader::builder(opentelemetry_stdout::MetricExporter::default()).build(); - - let meter_provider = MeterProviderBuilder::default() - .with_resource(resource()) - .with_reader(reader) - // .with_reader(stdout_reader) - .build(); - - global::set_meter_provider(meter_provider.clone()); - - meter_provider -} - /// Initialize OtelProviderGuard for opentelemetry-related termination processing. pub fn init_otel_providers() -> Option { // Check if otel is enabled @@ -151,16 +122,9 @@ pub fn init_otel_providers() -> Option { if enable_otel { let tracer_provider = init_tracer_provider(); - let meter_provider = init_meter_provider(); - Some(OtelProviderGuard { - tracer_provider, - meter_provider, - }) + Some(OtelProviderGuard { tracer_provider }) } else { - // NOTE: OTEL's global::meters are no-op without - // a meter provider configured, so its safe to - // not set any meter provider None } } @@ -168,7 +132,6 @@ pub fn init_otel_providers() -> Option { /// Guard opentelemetry-related providers termination processing. pub struct OtelProviderGuard { pub tracer_provider: SdkTracerProvider, - pub meter_provider: SdkMeterProvider, } impl Drop for OtelProviderGuard { @@ -176,8 +139,5 @@ impl Drop for OtelProviderGuard { if let Err(err) = self.tracer_provider.shutdown() { eprintln!("{err:?}"); } - if let Err(err) = self.meter_provider.shutdown() { - eprintln!("{err:?}"); - } } } diff --git a/engine/packages/metrics/src/registry.rs b/engine/packages/metrics/src/registry.rs new file mode 100644 index 0000000000..1dadc1c612 --- /dev/null +++ b/engine/packages/metrics/src/registry.rs @@ -0,0 +1,11 @@ +use prometheus::*; + +lazy_static::lazy_static! { + pub static ref REGISTRY: Registry = Registry::new_custom( + Some("rivet".to_string()), + Some(labels! { + "service".to_owned() => rivet_env::service_name().to_string(), + "kubernetes_pod_id".to_owned() => std::env::var("KUBERNETES_POD_ID").unwrap_or_default(), + "worker_source_hash".to_owned() => rivet_env::source_hash().to_string(), + })).unwrap(); +} diff --git a/engine/packages/metrics/src/server.rs b/engine/packages/metrics/src/server.rs new file mode 100644 index 0000000000..0e52c45bfe --- /dev/null +++ b/engine/packages/metrics/src/server.rs @@ -0,0 +1,57 @@ +use std::{net::SocketAddr, result::Result::Ok}; + +use anyhow::*; +use hyper::{ + Body, Request, Response, Server, + header::CONTENT_TYPE, + service::{make_service_fn, service_fn}, +}; +use prometheus::{Encoder, TextEncoder}; + +// TODO: Record extra labels + +#[tracing::instrument(skip_all)] +pub async fn run_standalone(config: rivet_config::Config) -> Result<()> { + let host = config.metrics.host(); + let port = config.metrics.port(); + let addr = SocketAddr::from((host, port)); + + let server = match Server::try_bind(&addr) { + Ok(x) => x, + Err(err) => { + tracing::error!(?host, ?port, ?err, "failed to bind metrics server"); + + // TODO: Find cleaner way of crashing entire program + // Hard crash program since a server failing to bind is critical + std::process::exit(1); + } + }; + + let server = server.serve(make_service_fn(|_| async { + Ok::<_, hyper::Error>(service_fn(serve_req)) + })); + + tracing::info!(?host, ?port, "started metrics server"); + server.await?; + + Ok(()) +} + +#[tracing::instrument(level = "debug", skip_all)] +async fn serve_req(_req: Request) -> Result, hyper::Error> { + let encoder = TextEncoder::new(); + + let metric_families = crate::registry::REGISTRY.gather(); + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .expect("encode"); + + let response = Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + .expect("response"); + + Ok(response) +} diff --git a/engine/packages/pegboard-gateway/src/metrics.rs b/engine/packages/pegboard-gateway/src/metrics.rs index 7fce45acba..f60990529e 100644 --- a/engine/packages/pegboard-gateway/src/metrics.rs +++ b/engine/packages/pegboard-gateway/src/metrics.rs @@ -1,14 +1,10 @@ -use rivet_metrics::{ - BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-gateway"); - - /// Has no expected attributes - pub static ref TUNNEL_PING_DURATION: Histogram = METER.f64_histogram("rivet_gateway_tunnel_ping_duration") - .with_description("RTT of messages from gateway to pegboard.") - .with_boundaries(BUCKETS.to_vec()) - .build(); + pub static ref TUNNEL_PING_DURATION: Histogram = register_histogram_with_registry!( + "gateway_tunnel_ping_duration", + "RTT of messages from gateway to pegboard.", + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/pegboard-gateway/src/shared_state.rs b/engine/packages/pegboard-gateway/src/shared_state.rs index 85bd85ee74..c2f2124c86 100644 --- a/engine/packages/pegboard-gateway/src/shared_state.rs +++ b/engine/packages/pegboard-gateway/src/shared_state.rs @@ -325,7 +325,7 @@ impl SharedState { in_flight.last_pong = now; let rtt = now.saturating_sub(pong.ts); - metrics::TUNNEL_PING_DURATION.record(rtt as f64 * 0.001, &[]); + metrics::TUNNEL_PING_DURATION.observe(rtt as f64 * 0.001); } Ok(protocol::mk2::ToGateway::ToServerTunnelMessage(msg)) => { let message_id = msg.message_id; diff --git a/engine/packages/pegboard/src/metrics.rs b/engine/packages/pegboard/src/metrics.rs index 30324e7d31..cc7754331c 100644 --- a/engine/packages/pegboard/src/metrics.rs +++ b/engine/packages/pegboard/src/metrics.rs @@ -1,19 +1,18 @@ -use rivet_metrics::{ - MICRO_BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{MICRO_BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-pegboard"); + pub static ref ACTOR_PENDING_ALLOCATION: IntGaugeVec = register_int_gauge_vec_with_registry!( + "pegboard_actor_pending_allocation", + "Total actors waiting for availability.", + &["namespace_id", "runner_name"], + *REGISTRY + ).unwrap(); - /// Expected attributes: "namespace_id", "runner_name" - pub static ref ACTOR_PENDING_ALLOCATION: Gauge = METER.f64_gauge("rivet_pegboard_actor_pending_allocation") - .with_description("Total actors waiting for availability.") - .build(); - - /// Expected attributes: "did_reserve" - pub static ref ACTOR_ALLOCATE_DURATION: Histogram = METER.f64_histogram("rivet_pegboard_actor_allocate_duration") - .with_description("Total duration to reserve resources for an actor.") - .with_boundaries(MICRO_BUCKETS.to_vec()) - .build(); + pub static ref ACTOR_ALLOCATE_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "pegboard_actor_allocate_duration", + "Total duration to reserve resources for an actor.", + &["did_reserve"], + MICRO_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/pegboard/src/workflows/actor/runtime.rs b/engine/packages/pegboard/src/workflows/actor/runtime.rs index d8c632ac2b..9429657292 100644 --- a/engine/packages/pegboard/src/workflows/actor/runtime.rs +++ b/engine/packages/pegboard/src/workflows/actor/runtime.rs @@ -4,7 +4,6 @@ use base64::prelude::BASE64_STANDARD; use futures_util::StreamExt; use futures_util::TryStreamExt; use gas::prelude::*; -use rivet_metrics::KeyValue; use rivet_runner_protocol::{ self as protocol, PROTOCOL_MK1_VERSION, PROTOCOL_MK2_VERSION, versioned, }; @@ -454,13 +453,11 @@ async fn allocate_actor_v2( .await?; let dt = start_instant.elapsed().as_secs_f64(); - metrics::ACTOR_ALLOCATE_DURATION.record( - dt, - &[KeyValue::new( - "did_reserve", - matches!(res.status, AllocateActorStatus::Allocated { .. }).to_string(), - )], - ); + metrics::ACTOR_ALLOCATE_DURATION + .with_label_values(&[ + matches!(res.status, AllocateActorStatus::Allocated { .. }).to_string() + ]) + .observe(dt); state.for_serverless = res.serverless; state.allocated_serverless_slot = res.serverless; diff --git a/engine/packages/pegboard/src/workflows/runner.rs b/engine/packages/pegboard/src/workflows/runner.rs index ce4fc2be6b..7ce1bf7d8f 100644 --- a/engine/packages/pegboard/src/workflows/runner.rs +++ b/engine/packages/pegboard/src/workflows/runner.rs @@ -3,7 +3,6 @@ use std::time::{Duration, Instant}; use futures_util::{FutureExt, StreamExt, TryStreamExt}; use gas::prelude::*; use rivet_data::converted::{ActorNameKeyData, MetadataKeyData, RunnerByKeyKeyData}; -use rivet_metrics::KeyValue; use rivet_runner_protocol::{self as protocol, PROTOCOL_MK1_VERSION, versioned}; use universaldb::{ options::{ConflictRangeType, StreamingMode}, @@ -1127,13 +1126,9 @@ pub(crate) async fn allocate_pending_actors( .custom_instrument(tracing::info_span!("runner_allocate_pending_actors_tx")) .await?; - metrics::ACTOR_PENDING_ALLOCATION.record( - pending_actor_count as f64, - &[ - KeyValue::new("namespace_id", input.namespace_id.to_string()), - KeyValue::new("runner_name", input.name.to_string()), - ], - ); + metrics::ACTOR_PENDING_ALLOCATION + .with_label_values(&[&input.namespace_id.to_string(), &input.name.to_string()]) + .set(pending_actor_count as i64); Ok(AllocatePendingActorsOutput { allocations }) } diff --git a/engine/packages/pegboard/src/workflows/runner2.rs b/engine/packages/pegboard/src/workflows/runner2.rs index 174bfd66f9..a28b3f108d 100644 --- a/engine/packages/pegboard/src/workflows/runner2.rs +++ b/engine/packages/pegboard/src/workflows/runner2.rs @@ -3,7 +3,6 @@ use std::time::{Duration, Instant}; use futures_util::{FutureExt, StreamExt, TryStreamExt}; use gas::prelude::*; use rivet_data::converted::RunnerByKeyKeyData; -use rivet_metrics::KeyValue; use rivet_runner_protocol::{self as protocol, PROTOCOL_MK2_VERSION, versioned}; use universaldb::{ options::{ConflictRangeType, StreamingMode}, @@ -774,13 +773,9 @@ pub(crate) async fn allocate_pending_actors( .custom_instrument(tracing::info_span!("runner_allocate_pending_actors_tx")) .await?; - metrics::ACTOR_PENDING_ALLOCATION.record( - pending_actor_count as f64, - &[ - KeyValue::new("namespace_id", input.namespace_id.to_string()), - KeyValue::new("runner_name", input.name.to_string()), - ], - ); + metrics::ACTOR_PENDING_ALLOCATION + .with_label_values(&[&input.namespace_id.to_string(), &input.name.to_string()]) + .set(pending_actor_count as i64); Ok(AllocatePendingActorsOutput { allocations }) } diff --git a/engine/packages/pools/src/metrics.rs b/engine/packages/pools/src/metrics.rs index d2c7773f17..5aa37981f7 100644 --- a/engine/packages/pools/src/metrics.rs +++ b/engine/packages/pools/src/metrics.rs @@ -1,32 +1,37 @@ -use rivet_metrics::{ - BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-pools"); - // MARK: SQL - /// Expected attributes: "action", "context_name", "location" - pub static ref SQL_QUERY_TOTAL: Counter = METER.u64_counter("rivet_sql_query_total") - .with_description("Total number of queries.") - .build(); - /// Expected attributes: "action", "context_name", "location" - pub static ref SQL_QUERY_DURATION: Histogram = METER.f64_histogram("rivet_sql_query_duration") - .with_description("Total duration of sql query.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "action", "context_name", "location" - pub static ref SQL_ACQUIRE_DURATION: Histogram = METER.f64_histogram("rivet_sql_acquire_duration") - .with_description("Total duration to acquire an sql connection.") - .with_boundaries(BUCKETS.to_vec()) - .build(); - /// Expected attributes: "action", "context_name", "location", "acquire_result" - pub static ref SQL_ACQUIRE_TOTAL: Counter = METER.u64_counter("rivet_sql_acquire_total") - .with_description("Amount times a pool connection was acquired.") - .build(); - /// Expected attributes: "action", "context_name", "location", "acquire_result" - pub static ref SQL_ACQUIRE_TRIES: Counter = METER.u64_counter("rivet_sql_acquire_tries") - .with_description("Amount of tries required to get a pool connection.") - .build(); + pub static ref SQL_QUERY_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "sql_query_total", + "Total number of queries.", + &["action", "context_name", "location"], + *REGISTRY + ).unwrap(); + pub static ref SQL_QUERY_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "sql_query_duration", + "Total duration of sql query.", + &["action", "context_name", "location"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref SQL_ACQUIRE_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "sql_acquire_duration", + "Total duration to acquire an sql connection.", + &["action", "context_name", "location"], + BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref SQL_ACQUIRE_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "sql_acquire_total", + "Amount times a pool connection was acquired.", + &["action", "context_name", "location", "acquire_result"], + *REGISTRY + ).unwrap(); + pub static ref SQL_ACQUIRE_TRIES: IntCounterVec = register_int_counter_vec_with_registry!( + "sql_acquire_tries", + "Amount of tries required to get a pool connection.", + &["action", "context_name", "location", "acquire_result"], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/runtime/src/lib.rs b/engine/packages/runtime/src/lib.rs index 40113d50d8..b9c17d3cd7 100644 --- a/engine/packages/runtime/src/lib.rs +++ b/engine/packages/runtime/src/lib.rs @@ -1,6 +1,6 @@ use std::{env, future::Future, sync::Arc, time::Duration}; -use rivet_metrics::{KeyValue, init_otel_providers}; +use rivet_metrics::init_otel_providers; use tokio::sync::{Notify, OnceCell}; mod metrics; mod traces; @@ -53,60 +53,54 @@ fn build_tokio_runtime_builder() -> tokio::runtime::Builder { rt_builder.enable_all(); rt_builder.on_thread_start(move || { - metrics::TOKIO_THREAD_COUNT.add(1, &[]); + metrics::TOKIO_THREAD_COUNT.inc(); }); rt_builder.on_thread_stop(move || { - metrics::TOKIO_THREAD_COUNT.add(-1, &[]); + metrics::TOKIO_THREAD_COUNT.dec(); }); - rt_builder.on_task_spawn(move |_| metrics::TOKIO_TASK_TOTAL.add(1, &[])); + rt_builder.on_task_spawn(move |_| metrics::TOKIO_TASK_TOTAL.inc()); if env::var("TOKIO_RUNTIME_METRICS").is_ok() { rt_builder.on_before_task_poll(|_| { let metrics = tokio::runtime::Handle::current().metrics(); - // let buckets = metrics.poll_time_histogram_num_buckets(); + let buckets = metrics.poll_time_histogram_num_buckets(); - metrics::TOKIO_GLOBAL_QUEUE_DEPTH.record(metrics.global_queue_depth() as u64, &[]); - metrics::TOKIO_ACTIVE_TASK_COUNT.record(metrics.num_alive_tasks() as u64, &[]); + metrics::TOKIO_GLOBAL_QUEUE_DEPTH.set(metrics.global_queue_depth() as i64); + metrics::TOKIO_ACTIVE_TASK_COUNT.set(metrics.num_alive_tasks() as i64); for worker in 0..metrics.num_workers() { - metrics::TOKIO_WORKER_OVERFLOW_COUNT.record( - metrics.worker_overflow_count(worker), - &[KeyValue::new("worker", worker.to_string())], - ); - metrics::TOKIO_WORKER_LOCAL_QUEUE_DEPTH.record( - metrics.worker_local_queue_depth(worker) as u64, - &[KeyValue::new("worker", worker.to_string())], - ); - - // TODO: Internal histogram data no longer accessable in OTEL - // needs to be turned into: 1. 20 gauges, or 2. store internal - // bucket counts manually - - // use rivet_metrics::prometheus::core::Metric; - // // Has some sort of internal lock, must read data before loop - // let prom_buckets = { - // metrics::TOKIO_TASK_POLL_DURATION - // .metric() - // .get_histogram() - // .get_bucket() - // .iter() - // .map(|bucket| bucket.get_cumulative_count()) - // .collect::>() - // }; - - // for (bucket, prom_bucket_count) in (0..buckets).zip(prom_buckets) { - // let range = metrics.poll_time_histogram_bucket_range(bucket); - // let count = metrics.poll_time_histogram_bucket_count(worker, bucket); - // // Calculate difference in tokio's internal bucket counts and - // // prom's internal count - // let diff = count.saturating_sub(prom_bucket_count); - - // // Match prom's count with tokio's for given bucket - // for _ in 0..diff { - // metrics::TOKIO_TASK_POLL_DURATION.record(range.start.as_secs_f64(), &[]); - // } - // } + metrics::TOKIO_WORKER_OVERFLOW_COUNT + .with_label_values(&[&worker.to_string()]) + .set(metrics.worker_overflow_count(worker) as i64); + metrics::TOKIO_WORKER_LOCAL_QUEUE_DEPTH + .with_label_values(&[&worker.to_string()]) + .set(metrics.worker_local_queue_depth(worker) as i64); + + use rivet_metrics::prometheus::core::Metric; + // Has some sort of internal lock, must read data before loop + let prom_buckets = { + metrics::TOKIO_TASK_POLL_DURATION + .metric() + .get_histogram() + .get_bucket() + .iter() + .map(|bucket| bucket.cumulative_count()) + .collect::>() + }; + + for (bucket, prom_bucket_count) in (0..buckets).zip(prom_buckets) { + let range = metrics.poll_time_histogram_bucket_range(bucket); + let count = metrics.poll_time_histogram_bucket_count(worker, bucket); + // Calculate difference in tokio's internal bucket counts and + // prom's internal count + let diff = count.saturating_sub(prom_bucket_count); + + // Match prom's count with tokio's for given bucket + for _ in 0..diff { + metrics::TOKIO_TASK_POLL_DURATION.observe(range.start.as_secs_f64()); + } + } } }); diff --git a/engine/packages/runtime/src/metrics.rs b/engine/packages/runtime/src/metrics.rs index 8d5a4f6e8b..0fd97cfaa9 100644 --- a/engine/packages/runtime/src/metrics.rs +++ b/engine/packages/runtime/src/metrics.rs @@ -1,34 +1,43 @@ -use rivet_metrics::{ - TASK_POLL_BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{REGISTRY, TASK_POLL_BUCKETS, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-runtime"); - // MARK: Tokio - pub static ref TOKIO_THREAD_COUNT: UpDownCounter = METER.i64_up_down_counter("rivet_tokio_thread_count") - .with_description("Total number of Tokio threads.") - .build(); - pub static ref TOKIO_GLOBAL_QUEUE_DEPTH: Gauge = METER.u64_gauge("rivet_tokio_global_queue_depth") - .with_description("Number of pending tasks in the global queue.") - .build(); - pub static ref TOKIO_TASK_TOTAL: Counter = METER.u64_counter("rivet_tokio_task_total") - .with_description("Total number of spawned tasks.") - .build(); - pub static ref TOKIO_ACTIVE_TASK_COUNT: Gauge = METER.u64_gauge("rivet_tokio_active_task_count") - .with_description("Total number of active (running or sleeping) tasks.") - .build(); - /// Expected attributes: "worker" - pub static ref TOKIO_WORKER_OVERFLOW_COUNT: Gauge = METER.u64_gauge("rivet_tokio_worker_overflow_count") - .with_description("Number of times the given worker thread saturated its local queue.") - .build(); - /// Expected attributes: "worker" - pub static ref TOKIO_WORKER_LOCAL_QUEUE_DEPTH: Gauge = METER.u64_gauge("rivet_tokio_worker_local_queue_depth") - .with_description("Number of pending tasks in a worker's queue.") - .build(); - pub static ref TOKIO_TASK_POLL_DURATION: Histogram = METER.f64_histogram("rivet_tokio_task_poll_duration") - .with_description("Duration to poll a task.") - .with_boundaries(TASK_POLL_BUCKETS.to_vec()) - .build(); + pub static ref TOKIO_THREAD_COUNT: IntGauge = register_int_gauge_with_registry!( + "tokio_thread_count", + "Total number of Tokio threads.", + *REGISTRY + ).unwrap(); + pub static ref TOKIO_GLOBAL_QUEUE_DEPTH: IntGauge = register_int_gauge_with_registry!( + "tokio_global_queue_depth", + "Number of pending tasks in the global queue.", + *REGISTRY + ).unwrap(); + pub static ref TOKIO_TASK_TOTAL: IntCounter = register_int_counter_with_registry!( + "tokio_task_total", + "Total number of spawned tasks.", + *REGISTRY + ).unwrap(); + pub static ref TOKIO_ACTIVE_TASK_COUNT: IntGauge = register_int_gauge_with_registry!( + "tokio_active_task_count", + "Total number of active (running or sleeping) tasks.", + *REGISTRY + ).unwrap(); + pub static ref TOKIO_WORKER_OVERFLOW_COUNT: IntGaugeVec = register_int_gauge_vec_with_registry!( + "tokio_worker_overflow_count", + "Number of times the given worker thread saturated its local queue.", + &["worker"], + *REGISTRY + ).unwrap(); + pub static ref TOKIO_WORKER_LOCAL_QUEUE_DEPTH: IntGaugeVec = register_int_gauge_vec_with_registry!( + "tokio_worker_local_queue_depth", + "Number of pending tasks in a worker's queue.", + &["worker"], + *REGISTRY + ).unwrap(); + pub static ref TOKIO_TASK_POLL_DURATION: Histogram = register_histogram_with_registry!( + "tokio_task_poll_duration", + "Duration to poll a task.", + TASK_POLL_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/runtime/src/traces.rs b/engine/packages/runtime/src/traces.rs index ebe015d4cb..d07c4155cb 100644 --- a/engine/packages/runtime/src/traces.rs +++ b/engine/packages/runtime/src/traces.rs @@ -4,7 +4,7 @@ use console_subscriber; use opentelemetry::trace::{TraceContextExt, TracerProvider}; use rivet_metrics::OtelProviderGuard; use std::sync::OnceLock; -use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; +use tracing_opentelemetry::OpenTelemetryLayer; use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt, reload, util::SubscriberInitExt}; type ReloadHandle = reload::Handle; @@ -22,24 +22,20 @@ pub fn init_tracing_subscriber(otel_providers: &Option) { let registry = tracing_subscriber::registry(); // Build and apply otel layers to the registry if otel is enabled - let (otel_trace_layer, otel_metric_layer) = match otel_providers { + let otel_trace_layer = match otel_providers { Some(providers) => { let tracer = providers.tracer_provider.tracer("tracing-otel-subscriber"); let otel_trace_layer = OpenTelemetryLayer::new(tracer) .with_filter(build_filter_from_env_var("RUST_TRACE")); - let otel_metric_layer = MetricsLayer::new(providers.meter_provider.clone()) - .with_filter(build_filter_from_env_var("RUST_TRACE")); - - (Some(otel_trace_layer), Some(otel_metric_layer)) + Some(otel_trace_layer) } - None => (None, None), + None => None, }; let registry = registry .with(reload_layer) - .with(otel_metric_layer) .with(otel_trace_layer) .with(sentry::integrations::tracing::layer()) .with(SentryOtelLayer); diff --git a/engine/packages/service-manager/src/lib.rs b/engine/packages/service-manager/src/lib.rs index bfc7a9a77c..3b1883f63a 100644 --- a/engine/packages/service-manager/src/lib.rs +++ b/engine/packages/service-manager/src/lib.rs @@ -135,8 +135,15 @@ impl RunConfigData { pub async fn start( config: rivet_config::Config, pools: rivet_pools::Pools, - services: Vec, + mut services: Vec, ) -> Result<()> { + services.push(Service::new( + "metrics", + ServiceKind::Core, + |config, _pools| rivet_metrics::run_standalone(config), + false, + )); + // Spawn services tracing::info!(services=?services.len(), "starting services"); let mut running_services = Vec::new(); diff --git a/engine/packages/tracing-utils/src/lib.rs b/engine/packages/tracing-utils/src/lib.rs index f9597573e1..f926a9959f 100644 --- a/engine/packages/tracing-utils/src/lib.rs +++ b/engine/packages/tracing-utils/src/lib.rs @@ -8,8 +8,6 @@ use std::{ use futures_util::future; use tracing::{Instrument, instrument::Instrumented}; -use rivet_metrics::KeyValue; - /// Attempts to create a new future to select over a list of futures. /// Non-panicking version of [futures_util::future::select_all](https://docs.rs/futures/0.3.15/futures/future/fn.select_all.html). /// @@ -57,13 +55,10 @@ impl Future for CustomInstrumented { Poll::Ready(val) => { if let Some(metadata) = metadata { if let (Some(file), Some(line)) = (metadata.file(), metadata.line()) { - metrics::INSTRUMENTED_FUTURE_DURATION.record( - this.start.elapsed().as_secs_f64(), - &[ - KeyValue::new("location", format!("{file}:{line}")), - KeyValue::new("name", metadata.name()), - ], - ); + let location = format!("{file}:{line}"); + metrics::INSTRUMENTED_FUTURE_DURATION + .with_label_values(&[location.as_str(), metadata.name()]) + .observe(this.start.elapsed().as_secs_f64()); } } Poll::Ready(val) @@ -74,18 +69,15 @@ impl Future for CustomInstrumented { } mod metrics { - use rivet_metrics::{ - MICRO_BUCKETS, - otel::{global::*, metrics::*}, - }; + use rivet_metrics::{MICRO_BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-util-core"); - - /// Expected attributes: "location", "name" - pub static ref INSTRUMENTED_FUTURE_DURATION: Histogram = METER.f64_histogram("rivet_instrumented_future_duration") - .with_description("Duration of a future.") - .with_boundaries(MICRO_BUCKETS.to_vec()) - .build(); + pub static ref INSTRUMENTED_FUTURE_DURATION: HistogramVec = register_histogram_vec_with_registry!( + "instrumented_future_duration", + "Duration of a future.", + &["location", "name"], + MICRO_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); } } diff --git a/engine/packages/universaldb/src/metrics.rs b/engine/packages/universaldb/src/metrics.rs index fd9aa61251..497693c7a6 100644 --- a/engine/packages/universaldb/src/metrics.rs +++ b/engine/packages/universaldb/src/metrics.rs @@ -1,27 +1,28 @@ -use rivet_metrics::{ - MICRO_BUCKETS, - otel::{global::*, metrics::*}, -}; +use rivet_metrics::{MICRO_BUCKETS, REGISTRY, prometheus::*}; lazy_static::lazy_static! { - static ref METER: Meter = meter("rivet-universaldb"); + pub static ref PING_DURATION: Histogram = register_histogram_with_registry!( + "udb_ping_duration", + "Total duration to retrieve a single value from udb.", + MICRO_BUCKETS.to_vec(), + *REGISTRY + ).unwrap(); + pub static ref MISSED_PING: IntGauge = register_int_gauge_with_registry!( + "udb_missed_ping", + "1 if udb missed the last ping.", + *REGISTRY + ).unwrap(); - /// Has no expected attributes - pub static ref PING_DURATION: Histogram = METER.f64_histogram("rivet_udb_ping_duration") - .with_description("Total duration to retrieve a single value from udb.") - .with_boundaries(MICRO_BUCKETS.to_vec()) - .build(); - /// Has no expected attributes - pub static ref MISSED_PING: Gauge = METER.u64_gauge("rivet_udb_missed_ping") - .with_description("1 if udb missed the last ping.") - .build(); - - /// Expected attributes: "type" - pub static ref KEY_PACK_COUNT: Counter = METER.u64_counter("rivet_udb_key_pack_count") - .with_description("How many times a key has been packed.") - .build(); - /// Expected attributes: "type" - pub static ref KEY_UNPACK_COUNT: Counter = METER.u64_counter("rivet_udb_key_unpack_count") - .with_description("How many times a key has been unpacked.") - .build(); + pub static ref KEY_PACK_COUNT: IntCounterVec = register_int_counter_vec_with_registry!( + "udb_key_pack_count", + "How many times a key has been packed.", + &["type"], + *REGISTRY + ).unwrap(); + pub static ref KEY_UNPACK_COUNT: IntCounterVec = register_int_counter_vec_with_registry!( + "udb_key_unpack_count", + "How many times a key has been unpacked.", + &["type"], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/universaldb/src/utils/subspace.rs b/engine/packages/universaldb/src/utils/subspace.rs index c29e64fbc0..bc42177828 100644 --- a/engine/packages/universaldb/src/utils/subspace.rs +++ b/engine/packages/universaldb/src/utils/subspace.rs @@ -5,7 +5,6 @@ use crate::{ range_option::RangeOption, tuple::{self, PackResult, TuplePack, TupleUnpack}, }; -use rivet_metrics::KeyValue; use crate::metrics; @@ -45,7 +44,9 @@ impl Subspace { /// Returns the key encoding the specified Tuple with the prefix of this Subspace /// prepended. pub fn pack(&self, t: &T) -> Vec { - metrics::KEY_PACK_COUNT.add(1, &[KeyValue::new("type", std::any::type_name::())]); + metrics::KEY_PACK_COUNT + .with_label_values(&[std::any::type_name::()]) + .inc(); self.inner.pack(t) } @@ -53,7 +54,9 @@ impl Subspace { /// Returns the key encoding the specified Tuple with the prefix of this Subspace /// prepended, with a versionstamp. pub fn pack_with_versionstamp(&self, t: &T) -> Vec { - metrics::KEY_PACK_COUNT.add(1, &[KeyValue::new("type", std::any::type_name::())]); + metrics::KEY_PACK_COUNT + .with_label_values(&[std::any::type_name::()]) + .inc(); self.inner.pack_with_versionstamp(t) } @@ -62,7 +65,9 @@ impl Subspace { /// removed. `unpack` will return an error if the key is not in this Subspace or does not /// encode a well-formed Tuple. pub fn unpack<'de, T: TupleUnpack<'de>>(&self, key: &'de [u8]) -> PackResult { - metrics::KEY_UNPACK_COUNT.add(1, &[KeyValue::new("type", std::any::type_name::())]); + metrics::KEY_UNPACK_COUNT + .with_label_values(&[std::any::type_name::()]) + .inc(); self.inner.unpack(key) }