From 38377d0ea429be42e5e311034ba88a5cda44299a Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 27 Jan 2022 14:01:59 +0000 Subject: [PATCH 1/5] STAC-0 Update dev container --- .devcontainer/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9990b159..96a2a256 100755 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -96,8 +96,8 @@ RUN apt-get update -y \ && apt-get -y install --no-install-recommends apt-utils 2>&1 \ # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed. && apt-get -y install git iproute2 procps lsb-release \ - # Install Python2.7 - && apt-get install -y python2.7 python-pip unzip \ + # Install Python + && apt-get install -y python3 python3-pip unzip \ && apt-get install -y protobuf-compiler \ && apt-get autoremove -y \ && apt-get clean -y \ From 72409695d5ef5e33b6f536dafee6e92522e9697e Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 27 Jan 2022 14:02:24 +0000 Subject: [PATCH 2/5] STAC-15634 Add optional openmetrics endpoint --- cmd/agent/main_common.go | 10 +++++++++- config/config.go | 14 ++++++++++++++ config/yaml_config.go | 11 +++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/cmd/agent/main_common.go b/cmd/agent/main_common.go index a09d5a5d..e31141d7 100644 --- a/cmd/agent/main_common.go +++ b/cmd/agent/main_common.go @@ -4,16 +4,18 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/StackVista/stackstate-process-agent/cmd/agent/features" "net/http" _ "net/http/pprof" "os" "time" + "github.com/StackVista/stackstate-process-agent/cmd/agent/features" + "github.com/StackVista/stackstate-agent/pkg/pidfile" log "github.com/cihub/seelog" "github.com/StackVista/stackstate-agent/pkg/tagger" + "github.com/StackVista/stackstate-agent/pkg/telemetry" "github.com/StackVista/stackstate-process-agent/checks" "github.com/StackVista/stackstate-process-agent/config" "github.com/StackVista/stackstate-process-agent/statsd" @@ -172,6 +174,12 @@ func runAgent(exit chan bool) { return } + if cfg.OpenMetricsEnabled { + // Expose the registered metrics via HTTP. + http.Handle("/metrics", telemetry.Handler()) + go http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", cfg.OpenMetricsPort), nil) //nolint:errcheck + } + if opts.info { // using the debug port to get info to work url := "http://localhost:6062/debug/vars" diff --git a/config/config.go b/config/config.go index 2837ea88..a5bae6e7 100644 --- a/config/config.go +++ b/config/config.go @@ -103,6 +103,10 @@ type AgentConfig struct { EnableShortLivedProcessFilter bool ShortLivedProcessQualifierSecs time.Duration + // Expose agent metrics via openmetrics endpoint + OpenMetricsEnabled bool + OpenMetricsPort int + // Relation Cache Expiration, In Minutes NetworkRelationCacheDurationMin time.Duration @@ -232,6 +236,9 @@ func NewDefaultAgentConfig() *AgentConfig { DDAgentPy: defaultDDAgentPy, DDAgentPyEnv: []string{defaultDDAgentPyEnv}, + OpenMetricsEnabled: false, + OpenMetricsPort: 5001, + EnableIncrementalPublishing: true, IncrementalPublishingRefreshInterval: 1 * time.Minute, @@ -764,6 +771,13 @@ func mergeEnvironmentVariables(c *AgentConfig) *AgentConfig { setNetworkRelationFilters(c, true, v) } + if ok, _ := isAffirmative(os.Getenv("STS_PROCESS_AGENT_OPEN_METRICS_ENABLED")); ok { + c.OpenMetricsEnabled = true + } + + if v, err := strconv.Atoi(os.Getenv("STS_PROCESS_AGENT_OPEN_METRICS_PORT")); err == nil { + c.OpenMetricsPort = v + } return c } diff --git a/config/yaml_config.go b/config/yaml_config.go index c17467ee..0b5c7c26 100644 --- a/config/yaml_config.go +++ b/config/yaml_config.go @@ -23,6 +23,10 @@ type YamlAgentConfig struct { StsURL string `yaml:"sts_url"` // Whether or not the process-agent should output logs to console LogToConsole bool `yaml:"log_to_console"` + + OpenMetricsEnabled bool `yaml:"open_metrics_enabled"` + OpenMetricsPort int `yaml:"open_metrics_port"` + // Incremental publishing: send only changes to server, instead of snapshots IncrementalPublishingEnabled string `yaml:"incremental_publishing_enabled"` // Periodically resend all data to allow downstream to recover from any lost data @@ -188,6 +192,13 @@ func mergeYamlConfig(agentConf *AgentConfig, yc *YamlAgentConfig) (*AgentConfig, agentConf.LogFile = yc.Process.LogFile } + if yc.OpenMetricsEnabled { + agentConf.OpenMetricsEnabled = true + } + if yc.OpenMetricsPort != 0 { + agentConf.OpenMetricsPort = yc.OpenMetricsPort + } + // (Re)configure the logging from our configuration if err := NewLoggerLevel(agentConf.LogLevel, agentConf.LogFile, agentConf.LogToConsole); err != nil { return nil, err From 9a9d37ff975239c9ab6f427390e538e1298389a7 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Thu, 27 Jan 2022 16:33:47 +0000 Subject: [PATCH 3/5] STAC-15634 Temporarily copying code in to avoid having to fix all dependency issues with stackstate-agent version bump --- Gopkg.lock | 3 ++ cmd/agent/main_common.go | 2 +- telemetry/counter.go | 65 +++++++++++++++++++++++++++++++++++++ telemetry/gauge.go | 59 +++++++++++++++++++++++++++++++++ telemetry/options.go | 17 ++++++++++ telemetry/prom_counter.go | 51 +++++++++++++++++++++++++++++ telemetry/prom_gauge.go | 45 +++++++++++++++++++++++++ telemetry/prom_telemetry.go | 28 ++++++++++++++++ telemetry/raw.go | 52 +++++++++++++++++++++++++++++ telemetry/utils.go | 31 ++++++++++++++++++ telemetry/utils_test.go | 48 +++++++++++++++++++++++++++ 11 files changed, 400 insertions(+), 1 deletion(-) create mode 100644 telemetry/counter.go create mode 100644 telemetry/gauge.go create mode 100644 telemetry/options.go create mode 100644 telemetry/prom_counter.go create mode 100644 telemetry/prom_gauge.go create mode 100644 telemetry/prom_telemetry.go create mode 100644 telemetry/raw.go create mode 100644 telemetry/utils.go create mode 100644 telemetry/utils_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 7ca2e857..af4080d6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -595,6 +595,7 @@ packages = [ "prometheus", "prometheus/internal", + "prometheus/promhttp", ] pruneopts = "" revision = "505eaef017263e299324067d40ca2c48f6a2cf50" @@ -1212,6 +1213,8 @@ "github.com/gogo/protobuf/proto", "github.com/mailru/easyjson", "github.com/patrickmn/go-cache", + "github.com/prometheus/client_golang/prometheus", + "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/shirou/w32", "github.com/stretchr/testify/assert", "golang.org/x/sys/windows/svc", diff --git a/cmd/agent/main_common.go b/cmd/agent/main_common.go index e31141d7..ad2e98fc 100644 --- a/cmd/agent/main_common.go +++ b/cmd/agent/main_common.go @@ -10,12 +10,12 @@ import ( "time" "github.com/StackVista/stackstate-process-agent/cmd/agent/features" + "github.com/StackVista/stackstate-process-agent/telemetry" "github.com/StackVista/stackstate-agent/pkg/pidfile" log "github.com/cihub/seelog" "github.com/StackVista/stackstate-agent/pkg/tagger" - "github.com/StackVista/stackstate-agent/pkg/telemetry" "github.com/StackVista/stackstate-process-agent/checks" "github.com/StackVista/stackstate-process-agent/config" "github.com/StackVista/stackstate-process-agent/statsd" diff --git a/telemetry/counter.go b/telemetry/counter.go new file mode 100644 index 00000000..f1ee6430 --- /dev/null +++ b/telemetry/counter.go @@ -0,0 +1,65 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2020 Datadog, Inc. + +package telemetry + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +// Counter tracks how many times something is happening. +type Counter interface { + // Inc increments the counter with the given tags value. + Inc(tagsValue ...string) + // Add adds the given value to the counter with the given tags value. + Add(value float64, tagsValue ...string) + // Delete deletes the value for the counter with the given tags value. + Delete(tagsValue ...string) + // IncWithTags increments the counter with the given tags. + // Even if less convenient, this signature could be used in hot path + // instead of Inc(...string) to avoid escaping the parameters on the heap. + IncWithTags(tags map[string]string) + // AddWithTags adds the given value to the counter with the given tags. + // Even if less convenient, this signature could be used in hot path + // instead of Add(float64, ...string) to avoid escaping the parameters on the heap. + AddWithTags(value float64, tags map[string]string) + // DeleteWithTags deletes the value for the counter with the given tags. + // Even if less convenient, this signature could be used in hot path + // instead of Delete(...string) to avoid escaping the parameters on the heap. + DeleteWithTags(tags map[string]string) +} + +// NewCounter creates a Counter with default options for telemetry purpose. +// Current implementation used: Prometheus Counter +func NewCounter(subsystem, name string, tags []string, help string) Counter { + return NewCounterWithOpts(subsystem, name, tags, help, DefaultOptions) +} + +// NewCounterWithOpts creates a Counter with the given options for telemetry purpose. +// See NewCounter() +func NewCounterWithOpts(subsystem, name string, tags []string, help string, opts Options) Counter { + // subsystem is optional + if subsystem != "" && !opts.NoDoubleUnderscoreSep { + // Prefix metrics with a _, prometheus will add a second _ + // It will create metrics with a custom separator and + // will let us replace it to a dot later in the process. + name = fmt.Sprintf("_%s", name) + } + + c := &promCounter{ + pc: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: subsystem, + Name: name, + Help: help, + }, + tags, + ), + } + telemetryRegistry.MustRegister(c.pc) + return c +} diff --git a/telemetry/gauge.go b/telemetry/gauge.go new file mode 100644 index 00000000..d3324305 --- /dev/null +++ b/telemetry/gauge.go @@ -0,0 +1,59 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2020 Datadog, Inc. + +package telemetry + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +// Gauge tracks the value of one health metric of the Agent. +type Gauge interface { + // Set stores the value for the given tags. + Set(value float64, tagsValue ...string) + // Inc increments the Gauge value. + Inc(tagsValue ...string) + // Dec decrements the Gauge value. + Dec(tagsValue ...string) + // Add adds the value to the Gauge value. + Add(value float64, tagsValue ...string) + // Sub subtracts the value to the Gauge value. + Sub(value float64, tagsValue ...string) + // Delete deletes the value for the Gauge with the given tags. + Delete(tagsValue ...string) +} + +// NewGauge creates a Gauge with default options for telemetry purpose. +// Current implementation used: Prometheus Gauge +func NewGauge(subsystem, name string, tags []string, help string) Gauge { + return NewGaugeWithOpts(subsystem, name, tags, help, DefaultOptions) +} + +// NewGaugeWithOpts creates a Gauge with the given options for telemetry purpose. +// See NewGauge() +func NewGaugeWithOpts(subsystem, name string, tags []string, help string, opts Options) Gauge { + // subsystem is optional + if subsystem != "" && !opts.NoDoubleUnderscoreSep { + // Prefix metrics with a _, prometheus will add a second _ + // It will create metrics with a custom separator and + // will let us replace it to a dot later in the process. + name = fmt.Sprintf("_%s", name) + } + + g := &promGauge{ + pg: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Subsystem: subsystem, + Name: name, + Help: help, + }, + tags, + ), + } + telemetryRegistry.MustRegister(g.pg) + return g +} diff --git a/telemetry/options.go b/telemetry/options.go new file mode 100644 index 00000000..2b9dc26a --- /dev/null +++ b/telemetry/options.go @@ -0,0 +1,17 @@ +package telemetry + +// Options for telemetry metrics. +// Creating an Options struct without specifying any of its fields should be the +// equivalent of using the DefaultOptions var. +type Options struct { + // NoDoubleUnderscoreSep is set to true when you don't want to + // separate the subsystem and the name with a double underscore separator. + NoDoubleUnderscoreSep bool +} + +// DefaultOptions for telemetry metrics which don't need to specify any option. +var DefaultOptions = Options{ + // By default, we want to separate the subsystem and the metric name with a + // double underscore to be able to replace it later in the process. + NoDoubleUnderscoreSep: false, +} diff --git a/telemetry/prom_counter.go b/telemetry/prom_counter.go new file mode 100644 index 00000000..bbddb077 --- /dev/null +++ b/telemetry/prom_counter.go @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2020 Datadog, Inc. + +package telemetry + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// Counter implementation using Prometheus. +type promCounter struct { + pc *prometheus.CounterVec +} + +// Add adds the given value to the counter with the given tags value. +func (c *promCounter) Add(value float64, tagsValue ...string) { + c.pc.WithLabelValues(tagsValue...).Add(value) +} + +// AddWithTags adds the given value to the counter with the given tags. +// Even if less convenient, this signature could be used in hot path +// instead of Add(float64, ...string) to avoid escaping the parameters on the heap. +func (c *promCounter) AddWithTags(value float64, tags map[string]string) { + c.pc.With(tags).Add(value) +} + +// Inc increments the counter with the given tags value. +func (c *promCounter) Inc(tagsValue ...string) { + c.pc.WithLabelValues(tagsValue...).Inc() +} + +// IncWithTags increments the counter with the given tags. +// Even if less convenient, this signature could be used in hot path +// instead of Inc(...string) to avoid escaping the parameters on the heap. +func (c *promCounter) IncWithTags(tags map[string]string) { + c.pc.With(tags).Inc() +} + +// Delete deletes the value for the counter with the given tags value. +func (c *promCounter) Delete(tagsValue ...string) { + c.pc.DeleteLabelValues(tagsValue...) +} + +// DeleteWithTags deletes the value for the counter with the given tags. +// Even if less convenient, this signature could be used in hot path +// instead of Delete(...string) to avoid escaping the parameters on the heap. +func (c *promCounter) DeleteWithTags(tags map[string]string) { + c.pc.Delete(tags) +} diff --git a/telemetry/prom_gauge.go b/telemetry/prom_gauge.go new file mode 100644 index 00000000..c27434b8 --- /dev/null +++ b/telemetry/prom_gauge.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2020 Datadog, Inc. + +package telemetry + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +// Gauge implementation using Prometheus. +type promGauge struct { + pg *prometheus.GaugeVec +} + +// Set stores the value for the given tags. +func (g *promGauge) Set(value float64, tagsValue ...string) { + g.pg.WithLabelValues(tagsValue...).Set(value) +} + +// Inc increments the Gauge value. +func (g *promGauge) Inc(tagsValue ...string) { + g.pg.WithLabelValues(tagsValue...).Inc() +} + +// Dec decrements the Gauge value. +func (g *promGauge) Dec(tagsValue ...string) { + g.pg.WithLabelValues(tagsValue...).Dec() +} + +// Delete deletes the value for the Gauge with the given tags. +func (g *promGauge) Delete(tagsValue ...string) { + g.pg.DeleteLabelValues(tagsValue...) +} + +// Add adds the value to the Gauge value. +func (g *promGauge) Add(value float64, tagsValue ...string) { + g.pg.WithLabelValues(tagsValue...).Add(value) +} + +// Sub subtracts the value to the Gauge value. +func (g *promGauge) Sub(value float64, tagsValue ...string) { + g.pg.WithLabelValues(tagsValue...).Sub(value) +} diff --git a/telemetry/prom_telemetry.go b/telemetry/prom_telemetry.go new file mode 100644 index 00000000..359add5f --- /dev/null +++ b/telemetry/prom_telemetry.go @@ -0,0 +1,28 @@ +package telemetry + +import ( + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var ( + telemetryRegistry = prometheus.NewRegistry() +) + +func init() { + telemetryRegistry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + telemetryRegistry.MustRegister(prometheus.NewGoCollector()) +} + +// Handler serves the HTTP route containing the prometheus metrics. +func Handler() http.Handler { + return promhttp.HandlerFor(telemetryRegistry, promhttp.HandlerOpts{}) +} + +// Reset resets the global telemetry registry, stopping the collection of every previously registered metrics. +// Mainly used for unit tests and integration tests. +func Reset() { + telemetryRegistry = prometheus.NewRegistry() +} diff --git a/telemetry/raw.go b/telemetry/raw.go new file mode 100644 index 00000000..11474f75 --- /dev/null +++ b/telemetry/raw.go @@ -0,0 +1,52 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2020 Datadog, Inc. + +package telemetry + +import ( + "encoding/json" + "fmt" +) + +// RawMetrics single payload structure +type RawMetrics struct { + Name string `json:"name,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` + HostName string `json:"host_name,omitempty"` + Value float64 `json:"value,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// RawMetricsMetaData payload containing meta data for the metric +type RawMetricsMetaData struct { + Hostname string `json:"hostname,omitempty"` + Tags []string `json:"tags,omitempty"` + Type string `json:"type,omitempty"` +} + +// ConvertToIntakeMetric Converts RawMetricsCheckData struct to an older v1 metrics structure +func (r RawMetrics) ConvertToIntakeMetric() []interface{} { + data := []interface{}{ + r.Name, + r.Timestamp, + r.Value, + RawMetricsMetaData{ + Hostname: r.HostName, + Type: "raw", + Tags: r.Tags, + }, + } + return data +} + +// JSONString returns a JSON string of the Component +func (r RawMetrics) JSONString() string { + b, err := json.Marshal(r) + if err != nil { + fmt.Println(err) + return fmt.Sprintf("{\"error\": \"%s\"}", err.Error()) + } + return string(b) +} diff --git a/telemetry/utils.go b/telemetry/utils.go new file mode 100644 index 00000000..6e520302 --- /dev/null +++ b/telemetry/utils.go @@ -0,0 +1,31 @@ +package telemetry + +import ( + "github.com/StackVista/stackstate-agent/pkg/config" +) + +// IsCheckEnabled returns if we want telemetry for the given check. +// Returns true if a * is present in the telemetry.checks list. +func IsCheckEnabled(checkName string) bool { + // false if telemetry is disabled + if !IsEnabled() { + return false + } + + // by default, we don't enable telemetry for every checks stats + if config.Datadog.IsSet("telemetry.checks") { + for _, check := range config.Datadog.GetStringSlice("telemetry.checks") { + if check == "*" { + return true + } else if check == checkName { + return true + } + } + } + return false +} + +// IsEnabled returns whether or not telemetry is enabled +func IsEnabled() bool { + return config.Datadog.IsSet("telemetry.enabled") && config.Datadog.GetBool("telemetry.enabled") +} diff --git a/telemetry/utils_test.go b/telemetry/utils_test.go new file mode 100644 index 00000000..d5a195db --- /dev/null +++ b/telemetry/utils_test.go @@ -0,0 +1,48 @@ +package telemetry + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/StackVista/stackstate-agent/pkg/config" +) + +func TestIsCheckEnabled(t *testing.T) { + assert := assert.New(t) + + mockConfig := config.Mock() + mockConfig.Set("telemetry.enabled", false) + + assert.False(IsCheckEnabled("cpu")) + assert.False(IsCheckEnabled("disk")) + + mockConfig.Set("telemetry.enabled", true) + + assert.False(IsCheckEnabled("cpu")) + assert.False(IsCheckEnabled("disk")) + + mockConfig.Set("telemetry.enabled", true) + mockConfig.Set("telemetry.checks", []string{"*"}) + + assert.True(IsCheckEnabled("cpu")) + assert.True(IsCheckEnabled("disk")) + + mockConfig.Set("telemetry.enabled", true) + mockConfig.Set("telemetry.checks", []string{"cpu"}) + + assert.True(IsCheckEnabled("cpu")) + assert.False(IsCheckEnabled("disk")) + + mockConfig.Set("telemetry.enabled", false) + mockConfig.Set("telemetry.checks", []string{"cpu"}) + + assert.False(IsCheckEnabled("cpu")) + assert.False(IsCheckEnabled("disk")) + + mockConfig.Set("telemetry.enabled", true) + mockConfig.Set("telemetry.checks", []string{"cpu", "disk"}) + + assert.True(IsCheckEnabled("cpu")) + assert.True(IsCheckEnabled("disk")) +} From 4761552f7cbac11a7ed4fcd3df097ae2c61e662b Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 28 Jan 2022 07:39:03 +0000 Subject: [PATCH 4/5] STAC-0 Fix docker authentication --- packaging/publish_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/publish_image.sh b/packaging/publish_image.sh index acf56b4c..cde8f16b 100755 --- a/packaging/publish_image.sh +++ b/packaging/publish_image.sh @@ -17,7 +17,7 @@ cp $ARTIFACT_PATH DockerFiles/agent docker build -t stackstate/${IMAGE_REPO}:${IMAGE_TAG} DockerFiles/agent -docker login -u $DOCKER_USER -p $DOCKER_PASS +docker login -u $docker_user -p $docker_password docker push stackstate/${IMAGE_REPO}:${IMAGE_TAG} if [ "$PUSH_LATEST" = "true" ]; then From a495c5982461d4f650ed38bd4d75609f3e20bd2c Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Fri, 28 Jan 2022 14:05:09 +0000 Subject: [PATCH 5/5] STAC-15634 Metrics port that does not collide --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index a5bae6e7..ec187979 100644 --- a/config/config.go +++ b/config/config.go @@ -237,7 +237,7 @@ func NewDefaultAgentConfig() *AgentConfig { DDAgentPyEnv: []string{defaultDDAgentPyEnv}, OpenMetricsEnabled: false, - OpenMetricsPort: 5001, + OpenMetricsPort: 5123, EnableIncrementalPublishing: true, IncrementalPublishingRefreshInterval: 1 * time.Minute,