Skip to content

cluster: support retention by size for prometheus #2612

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AstroProfundis wants to merge 2 commits into
base: master
Choose a base branch
from
+85 −16
Open

cluster: support retention by size for prometheus #2612

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e9036ee
cluster: support retention by size for prometheus
AstroProfundis Oct 10, 2025
b2b2026
cluster: log warning when retention values are invalid
AstroProfundis Nov 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion embed/templates/scripts/run_prometheus.sh.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,5 +55,10 @@ exec bin/prometheus/prometheus \
{{- end}}
{{- end}}
{{- if not .EnablePromAgentMode}}
--storage.tsdb.retention="{{.Retention}}"
{{- if .RetentionSize}}
--storage.tsdb.retention.size="{{.RetentionSize}}"
{{- end}}
{{- if .RetentionTime}}
--storage.tsdb.retention.time="{{.RetentionTime}}"
{{- end}}
{{- end}}
32 changes: 29 additions & 3 deletions pkg/cluster/spec/monitoring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/pingcap/tiup/pkg/cluster/ctxt"
"github.com/pingcap/tiup/pkg/cluster/template/config"
"github.com/pingcap/tiup/pkg/cluster/template/scripts"
logprinter "github.com/pingcap/tiup/pkg/logger/printer"
"github.com/pingcap/tiup/pkg/meta"
"github.com/pingcap/tiup/pkg/set"
"github.com/pingcap/tiup/pkg/utils"
Expand All @@ -54,7 +55,9 @@ type PrometheusSpec struct {
RemoteConfig Remote `yaml:"remote_config,omitempty" validate:"remote_config:ignore"`
ExternalAlertmanagers []ExternalAlertmanager `yaml:"external_alertmanagers" validate:"external_alertmanagers:ignore"`
PushgatewayAddrs []string `yaml:"pushgateway_addrs,omitempty" validate:"pushgateway_addrs:ignore"`
Retention string `yaml:"storage_retention,omitempty" validate:"storage_retention:editable"`
Retention string `yaml:"storage_retention,omitempty" validate:"storage_retention:editable"` // deprecated
RetentionSize string `yaml:"storage_retention_size,omitempty" validate:"storage_retention_size:editable"`
RetentionTime string `yaml:"storage_retention_time,omitempty" validate:"storage_retention_time:editable"`
ResourceControl meta.ResourceControl `yaml:"resource_control,omitempty" validate:"resource_control:editable"`
Arch string `yaml:"arch,omitempty"`
OS string `yaml:"os,omitempty"`
Expand Down Expand Up @@ -270,7 +273,6 @@ func (i *MonitorInstance) InitConfig(
cfg := &scripts.PrometheusScript{
Port: spec.Port,
WebExternalURL: fmt.Sprintf("http://%s", utils.JoinHostPort(spec.Host, spec.Port)),
Retention: getRetention(spec.Retention),
EnableNG: spec.NgPort > 0,
EnablePromAgentMode: spec.EnablePromAgentMode, // Get from spec directly

Expand All @@ -282,6 +284,14 @@ func (i *MonitorInstance) InitConfig(

AdditionalArgs: spec.AdditionalArgs,
}
// Set retention policy
logPtr := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
if spec.RetentionTime == "" { // keep backward compatiability
cfg.RetentionTime = getRetentionTime(logPtr, spec.Retention)
} else {
cfg.RetentionTime = getRetentionTime(logPtr, spec.RetentionTime)
}
cfg.RetentionSize = getRetentionSize(logPtr, spec.RetentionSize)

// Check if agent mode is enabled in additional arguments
if !cfg.EnablePromAgentMode {
Expand Down Expand Up @@ -675,9 +685,25 @@ func mergeAdditionalScrapeConf(source string, addition map[string]any) error {
return utils.WriteFile(source, bytes, 0644)
}

func getRetention(retention string) string {
func getRetentionSize(l *logprinter.Logger, retention string) string {
retention = strings.ToUpper(strings.TrimSpace(retention))
valid, _ := regexp.MatchString("^[1-9]\\d*(B|KB|MB|GB|TB|PB|EB)$", retention)
Copy link
Contributor

@dveeden dveeden Oct 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What about KiB etc?

Copy link
Contributor Author

@AstroProfundis AstroProfundis Oct 31, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn't see that in Prometheus doc though...

Copy link
Contributor

@dveeden dveeden Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then this should be fine.

if retention == "" || !valid {
if !valid && l != nil {
l.Warnf("invalid retention size %s, ignored.", retention)
}
return ""
}
return retention
}

func getRetentionTime(l *logprinter.Logger, retention string) string {
retention = strings.TrimSpace(retention)
valid, _ := regexp.MatchString("^[1-9]\\d*d$", retention)
if retention == "" || !valid {
if !valid && l != nil {
l.Warnf("invalid retention time %s, using 30d as default", retention)
}
return "30d"
}
return retention
Expand Down
56 changes: 46 additions & 10 deletions pkg/cluster/spec/monitoring_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,32 +229,68 @@ scrape_configs:

func TestGetRetention(t *testing.T) {
var val string
val = getRetention("-1d")
val = getRetentionTime(nil, "-1d")
assert.EqualValues(t, "30d", val)

val = getRetention("0d")
val = getRetentionTime(nil, "0d")
assert.EqualValues(t, "30d", val)

val = getRetention("01d")
val = getRetentionTime(nil, "01d")
assert.EqualValues(t, "30d", val)

val = getRetention("1dd")
val = getRetentionTime(nil, "1dd")
assert.EqualValues(t, "30d", val)

val = getRetention("*1d")
val = getRetentionTime(nil, "*1d")
assert.EqualValues(t, "30d", val)

val = getRetention("1d ")
assert.EqualValues(t, "30d", val)
val = getRetentionTime(nil, "1d ")
assert.EqualValues(t, "1d", val)

val = getRetentionTime(nil, " 1d")
assert.EqualValues(t, "1d", val)

val = getRetention("ddd")
val = getRetentionTime(nil, "ddd")
assert.EqualValues(t, "30d", val)

val = getRetention("60d")
val = getRetentionTime(nil, "60d")
assert.EqualValues(t, "60d", val)

val = getRetention("999d")
val = getRetentionTime(nil, "999d")
assert.EqualValues(t, "999d", val)

val = getRetentionSize(nil, "-1MB")
assert.EqualValues(t, "", val)

val = getRetentionSize(nil, "30d")
assert.EqualValues(t, "", val)

val = getRetentionSize(nil, "1k")
assert.EqualValues(t, "", val)

val = getRetentionSize(nil, "01G")
assert.EqualValues(t, "", val)

val = getRetentionSize(nil, "233mb")
assert.EqualValues(t, "233MB", val)

val = getRetentionSize(nil, "*1GB")
assert.EqualValues(t, "", val)

val = getRetentionSize(nil, "20GB ")
assert.EqualValues(t, "20GB", val)

val = getRetentionSize(nil, " 20GB")
assert.EqualValues(t, "20GB", val)

val = getRetentionSize(nil, "3TB")
assert.EqualValues(t, "3TB", val)

val = getRetentionSize(nil, "30GB")
assert.EqualValues(t, "30GB", val)

val = getRetentionSize(nil, "1EB")
assert.EqualValues(t, "1EB", val)
}

// TestHandleRemoteWrite verifies that remote write configurations are properly handled
Expand Down
3 changes: 2 additions & 1 deletion pkg/cluster/template/scripts/monitoring.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ import (
type PrometheusScript struct {
Port int
WebExternalURL string
Retention string
RetentionSize string
RetentionTime string
EnableNG bool
EnablePromAgentMode bool

Expand Down
3 changes: 2 additions & 1 deletion pkg/cluster/template/scripts/monitoring_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ func TestPrometheusScriptWithAgentMode(t *testing.T) {
script := &PrometheusScript{
Port: 9090,
WebExternalURL: "http://localhost:9090",
Retention: "30d",
RetentionTime: "30d",
RetentionSize: "100GB",
EnableNG: false,
EnablePromAgentMode: true,
DeployDir: "/deploy",
Expand Down
Loading