diff --git a/internal/pkg/database/database.go b/internal/pkg/database/database.go index f378db3..a932fa7 100644 --- a/internal/pkg/database/database.go +++ b/internal/pkg/database/database.go @@ -3,8 +3,10 @@ package database import ( "context" "database/sql" + "time" "github.com/babourine/x/pkg/set" + "github.com/hladush/go-telemetry/pkg/telemetry" _ "github.com/lib/pq" ) @@ -13,7 +15,8 @@ const ( ) var ( - ctx = context.Background() + ctx = context.Background() + newSessionMethod = telemetry.NewMethod("db_connection", "new_session") ) type Database struct { @@ -28,22 +31,42 @@ type Session struct { func (d *Database) NewSession(withTransaction bool) (*Session, error) { + // Track session creation metrics + transactionLabel := "false" + if withTransaction { + transactionLabel = "true" + } + + defer newSessionMethod.RecordLatency(time.Now(), "with_transaction", transactionLabel) + newSessionMethod.CountRequest("with_transaction", transactionLabel) + var err error s := &Session{} + // Track database connection creation + connectionStart := time.Now() + newSessionMethod.CountRequest("with_transaction", transactionLabel) + // open connection if s.db, err = sql.Open(dbDriverName, d.ConnectionString); err != nil { + newSessionMethod.LogAndCountError(err, "with_transaction", transactionLabel) return nil, err } + newSessionMethod.RecordLatency(connectionStart, "with_transaction", transactionLabel) + newSessionMethod.CountSuccess("with_transaction", transactionLabel) + // start transaction if withTransaction { if s.trx, err = s.db.BeginTx(ctx, nil); err != nil { + s.db.Close() // Close the connection before returning error + newSessionMethod.LogAndCountError(err, "with_transaction", transactionLabel) return nil, err } } + newSessionMethod.CountSuccess("with_transaction", transactionLabel) return s, nil } diff --git a/internal/pkg/database/slice.go b/internal/pkg/database/slice.go index d5b387c..8b59bfa 100644 --- a/internal/pkg/database/slice.go +++ b/internal/pkg/database/slice.go @@ -1,16 +1,32 @@ package database +import ( + "time" + + "github.com/hladush/go-telemetry/pkg/telemetry" +) + +var ( + getSliceMethod = telemetry.NewMethod("db_connection", "get_slice") +) + func GetSlice(db *Database, query string) (any, error) { + // Track DB connection for slice query operation + defer getSliceMethod.RecordLatency(time.Now()) + getSliceMethod.CountRequest() + // open connection sess, err := db.NewSession(false) if err != nil { + getSliceMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() rows, err := sess.Query(query) if err != nil { + getSliceMethod.LogAndCountError(err, "query") return nil, err } defer rows.Close() @@ -21,6 +37,7 @@ func GetSlice(db *Database, query string) (any, error) { var item any if err := rows.Scan(&item); err != nil { + getSliceMethod.LogAndCountError(err, "scan") return nil, err } @@ -28,6 +45,7 @@ func GetSlice(db *Database, query string) (any, error) { } + getSliceMethod.CountSuccess() return result, nil } diff --git a/internal/pkg/heimdall/cluster_dal.go b/internal/pkg/heimdall/cluster_dal.go index 99a31c6..188aabf 100644 --- a/internal/pkg/heimdall/cluster_dal.go +++ b/internal/pkg/heimdall/cluster_dal.go @@ -5,7 +5,9 @@ import ( _ "embed" "encoding/json" "fmt" + "time" + "github.com/hladush/go-telemetry/pkg/telemetry" _ "github.com/lib/pq" "github.com/patterninc/heimdall/internal/pkg/database" @@ -67,7 +69,12 @@ var ( ) var ( - ErrUnknownClusterID = fmt.Errorf(`unknown cluster_id`) + ErrUnknownClusterID = fmt.Errorf(`unknown cluster_id`) + upsertClusterMethod = telemetry.NewMethod("db_connection", "upsert_cluster") + getClusterMethod = telemetry.NewMethod("db_connection", "get_cluster") + getClusterStatusMethod = telemetry.NewMethod("db_connection", "get_cluster_status") + updateClusterStatusMethod = telemetry.NewMethod("db_connection", "update_cluster_status") + getClustersMethod = telemetry.NewMethod("db_connection", "get_clusters") ) func (h *Heimdall) submitCluster(c *cluster.Cluster) (any, error) { @@ -82,9 +89,14 @@ func (h *Heimdall) submitCluster(c *cluster.Cluster) (any, error) { func (h *Heimdall) clusterUpsert(c *cluster.Cluster) error { + // Track DB connection for cluster upsert operation + defer upsertClusterMethod.RecordLatency(time.Now()) + upsertClusterMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(true) if err != nil { + upsertClusterMethod.LogAndCountError(err, "new_session") return err } defer sess.Close() @@ -112,15 +124,26 @@ func (h *Heimdall) clusterUpsert(c *cluster.Cluster) error { } } - return sess.Commit() + if err := sess.Commit(); err != nil { + upsertClusterMethod.LogAndCountError(err, "commit") + return err + } + + upsertClusterMethod.CountSuccess() + return nil } func (h *Heimdall) getCluster(c *cluster.Cluster) (any, error) { + // Track DB connection for get cluster operation + defer getClusterMethod.RecordLatency(time.Now()) + getClusterMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getClusterMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -142,25 +165,32 @@ func (h *Heimdall) getCluster(c *cluster.Cluster) (any, error) { if err := row.Scan(&r.SystemID, &r.Status, &r.Name, &r.Version, &r.Description, &clusterContext, &r.User, &r.CreatedAt, &r.UpdatedAt); err != nil { if err == sql.ErrNoRows { - return nil, ErrUnknownCommandID + return nil, ErrUnknownClusterID } else { return nil, err } } if err := clusterParseContextAndTags(r, clusterContext, sess); err != nil { + getClusterMethod.LogAndCountError(err, "cluster_parse_context_and_tags") return nil, err } + getClusterMethod.CountSuccess() return r, nil } func (h *Heimdall) getClusterStatus(c *cluster.Cluster) (any, error) { + // Track DB connection for cluster status operation + defer getClusterStatusMethod.RecordLatency(time.Now()) + getClusterStatusMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getClusterStatusMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -181,15 +211,20 @@ func (h *Heimdall) getClusterStatus(c *cluster.Cluster) (any, error) { } } + getClusterStatusMethod.CountSuccess() return r, nil } func (h *Heimdall) updateClusterStatus(c *cluster.Cluster) (any, error) { + defer updateClusterStatusMethod.RecordLatency(time.Now()) + updateClusterStatusMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + updateClusterStatusMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -203,15 +238,21 @@ func (h *Heimdall) updateClusterStatus(c *cluster.Cluster) (any, error) { return nil, ErrUnknownClusterID } + updateClusterStatusMethod.CountSuccess() return h.getClusterStatus(c) } func (h *Heimdall) getClusters(f *database.Filter) (any, error) { + // Track DB connection for clusters list operation + defer getClustersMethod.RecordLatency(time.Now()) + getClustersMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getClustersMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -247,6 +288,7 @@ func (h *Heimdall) getClusters(f *database.Filter) (any, error) { } + getClustersMethod.CountSuccess() return &resultset{ Data: result, }, nil diff --git a/internal/pkg/heimdall/command_dal.go b/internal/pkg/heimdall/command_dal.go index c47262b..ad9cd99 100644 --- a/internal/pkg/heimdall/command_dal.go +++ b/internal/pkg/heimdall/command_dal.go @@ -5,7 +5,9 @@ import ( _ "embed" "encoding/json" "fmt" + "time" + "github.com/hladush/go-telemetry/pkg/telemetry" _ "github.com/lib/pq" "github.com/patterninc/heimdall/internal/pkg/database" @@ -79,7 +81,12 @@ var ( ) var ( - ErrUnknownCommandID = fmt.Errorf(`unknown command_id`) + ErrUnknownCommandID = fmt.Errorf(`unknown command_id`) + upsertCommandMethod = telemetry.NewMethod("db_connection", "upsert_command") + getCommandMethod = telemetry.NewMethod("db_connection", "get_command") + getCommandStatusMethod = telemetry.NewMethod("db_connection", "get_command_status") + updateCommandStatusMethod = telemetry.NewMethod("db_connection", "update_command_status") + getCommandsMethod = telemetry.NewMethod("db_connection", "get_commands") ) func (h *Heimdall) submitCommand(c *command.Command) (any, error) { @@ -94,9 +101,14 @@ func (h *Heimdall) submitCommand(c *command.Command) (any, error) { func (h *Heimdall) commandUpsert(c *command.Command) error { + // Track DB connection for command upsert operation + defer upsertCommandMethod.RecordLatency(time.Now()) + upsertCommandMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(true) if err != nil { + upsertCommandMethod.LogAndCountError(err, "new_session") return err } defer sess.Close() @@ -141,15 +153,26 @@ func (h *Heimdall) commandUpsert(c *command.Command) error { } } - return sess.Commit() + if err := sess.Commit(); err != nil { + upsertCommandMethod.LogAndCountError(err, "commit") + return err + } + + upsertCommandMethod.CountSuccess() + return nil } func (h *Heimdall) getCommand(c *command.Command) (any, error) { + // Track DB connection for get command operation + defer getCommandMethod.RecordLatency(time.Now()) + getCommandMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getCommandMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -178,18 +201,25 @@ func (h *Heimdall) getCommand(c *command.Command) (any, error) { } if err := commandParseContextAndTags(r, commandContext, sess); err != nil { + getCommandMethod.LogAndCountError(err, "command_parse_context_and_tags") return nil, err } + getCommandMethod.CountSuccess() return r, nil } func (h *Heimdall) getCommandStatus(c *command.Command) (any, error) { + // Track DB connection for command status operation + defer getCommandStatusMethod.RecordLatency(time.Now()) + getCommandStatusMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getCommandStatusMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -210,15 +240,21 @@ func (h *Heimdall) getCommandStatus(c *command.Command) (any, error) { } } + getCommandStatusMethod.CountSuccess() return r, nil } func (h *Heimdall) updateCommandStatus(c *command.Command) (any, error) { + // Track DB connection for command status update operation + defer updateCommandStatusMethod.RecordLatency(time.Now()) + updateCommandStatusMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + updateCommandStatusMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -232,15 +268,21 @@ func (h *Heimdall) updateCommandStatus(c *command.Command) (any, error) { return nil, ErrUnknownCommandID } + updateCommandStatusMethod.CountSuccess() return h.getCommandStatus(c) } func (h *Heimdall) getCommands(f *database.Filter) (any, error) { + // Track DB connection for commands list operation + defer getCommandsMethod.RecordLatency(time.Now()) + getCommandsMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getCommandsMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -276,6 +318,7 @@ func (h *Heimdall) getCommands(f *database.Filter) (any, error) { } + getCommandsMethod.CountSuccess() return &resultset{ Data: result, }, nil diff --git a/internal/pkg/heimdall/job_dal.go b/internal/pkg/heimdall/job_dal.go index aaa34cd..31f070a 100644 --- a/internal/pkg/heimdall/job_dal.go +++ b/internal/pkg/heimdall/job_dal.go @@ -5,7 +5,9 @@ import ( _ "embed" "encoding/json" "fmt" + "time" + "github.com/hladush/go-telemetry/pkg/telemetry" _ "github.com/lib/pq" "github.com/patterninc/heimdall/internal/pkg/database" @@ -50,7 +52,11 @@ var queryJobStatusSelect string var queryJobStatusesSelect string var ( - ErrUnknownJobID = fmt.Errorf(`unknown job_id`) + ErrUnknownJobID = fmt.Errorf(`unknown job_id`) + insertJobMethod = telemetry.NewMethod("db_connection", "insert_job") + getJobMethod = telemetry.NewMethod("db_connection", "get_job") + getJobsMethod = telemetry.NewMethod("db_connection", "get_jobs") + getJobStatusMethod = telemetry.NewMethod("db_connection", "get_job_status") ) var ( @@ -92,9 +98,14 @@ type jobRequest struct { func (h *Heimdall) insertJob(j *job.Job, clusterID, commandID string) (int64, error) { + // Track DB connection for job insert operation + defer insertJobMethod.RecordLatency(time.Now()) + insertJobMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(true) if err != nil { + insertJobMethod.LogAndCountError(err, "new_session") return 0, err } defer sess.Close() @@ -149,18 +160,25 @@ func (h *Heimdall) insertJob(j *job.Job, clusterID, commandID string) (int64, er } if err := sess.Commit(); err != nil { + insertJobMethod.LogAndCountError(err, "commit") return 0, err } + insertJobMethod.CountSuccess() return jobID, nil } func (h *Heimdall) getJob(j *jobRequest) (any, error) { + // Track DB connection for job get operation + defer getJobMethod.RecordLatency(time.Now()) + getJobMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getJobMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -189,29 +207,38 @@ func (h *Heimdall) getJob(j *jobRequest) (any, error) { } if err := jobParseContextAndTags(r, jobContext, sess); err != nil { + getJobMethod.LogAndCountError(err, "job_parse_context_and_tags") return nil, err } + getJobMethod.CountSuccess() return r, nil } func (h *Heimdall) getJobs(f *database.Filter) (any, error) { + // Track DB connection for jobs list operation + defer getJobsMethod.RecordLatency(time.Now()) + getJobsMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getJobsMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() query, args, err := f.Render(queryJobsSelect, jobsFilterConfig) if err != nil { + getJobsMethod.LogAndCountError(err, "query") return nil, err } rows, err := sess.Query(query, args...) if err != nil { + getJobsMethod.LogAndCountError(err, "query") return nil, err } defer rows.Close() @@ -225,10 +252,12 @@ func (h *Heimdall) getJobs(f *database.Filter) (any, error) { if err := rows.Scan(&r.SystemID, &r.ID, &r.Status, &r.Name, &r.Version, &r.Description, &jobContext, &r.Error, &r.User, &r.IsSync, &r.CreatedAt, &r.UpdatedAt, &r.CommandID, &r.CommandName, &r.CluserID, &r.ClusterName, &r.StoreResultSync); err != nil { + getJobsMethod.LogAndCountError(err, "scan") return nil, err } if err := jobParseContextAndTags(r, jobContext, sess); err != nil { + getJobsMethod.LogAndCountError(err, "job_parse_context_and_tags") return nil, err } @@ -236,6 +265,7 @@ func (h *Heimdall) getJobs(f *database.Filter) (any, error) { } + getJobsMethod.CountSuccess() return &resultset{ Data: result, }, nil @@ -244,9 +274,14 @@ func (h *Heimdall) getJobs(f *database.Filter) (any, error) { func (h *Heimdall) getJobStatus(j *jobRequest) (any, error) { + // Track DB connection for job status operation + defer getJobStatusMethod.RecordLatency(time.Now()) + getJobStatusMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(false) if err != nil { + getJobStatusMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -261,12 +296,15 @@ func (h *Heimdall) getJobStatus(j *jobRequest) (any, error) { if err := row.Scan(&r.Status, &r.Error, &r.UpdatedAt); err != nil { if err == sql.ErrNoRows { + getJobStatusMethod.LogAndCountError(ErrUnknownJobID, "query") return nil, ErrUnknownJobID } else { + getJobStatusMethod.LogAndCountError(err, "query") return nil, err } } + getJobStatusMethod.CountSuccess() return r, nil } diff --git a/internal/pkg/heimdall/jobs_async.go b/internal/pkg/heimdall/jobs_async.go index bdb0c71..49a23f4 100644 --- a/internal/pkg/heimdall/jobs_async.go +++ b/internal/pkg/heimdall/jobs_async.go @@ -3,7 +3,9 @@ package heimdall import ( _ "embed" "fmt" + "time" + "github.com/hladush/go-telemetry/pkg/telemetry" "github.com/patterninc/heimdall/internal/pkg/database" "github.com/patterninc/heimdall/pkg/object/job" "github.com/patterninc/heimdall/pkg/object/job/status" @@ -14,6 +16,12 @@ const ( formatErrUnknownCluster = "unknown cluster: %s" ) +var ( + getAsyncJobsMethod = telemetry.NewMethod("db_connection", "get_async_jobs") + runAsyncJobMethod = telemetry.NewMethod("db_connection", "run_async_job") + updateAsyncJobStatusMethod = telemetry.NewMethod("db_connection", "update_async_job_status") +) + //go:embed queries/job/active_select.sql var queryActiveJobSelect string @@ -28,9 +36,14 @@ var queryActiveJobDelete string func (h *Heimdall) getAsyncJobs(limit int) ([]*job.Job, error) { + // Track DB connection for async jobs retrieval + defer getAsyncJobsMethod.RecordLatency(time.Now()) + getAsyncJobsMethod.CountRequest() + // open connection sess, err := h.Database.NewSession(true) if err != nil { + getAsyncJobsMethod.LogAndCountError(err, "new_session") return nil, err } defer sess.Close() @@ -81,18 +94,25 @@ func (h *Heimdall) getAsyncJobs(limit int) ([]*job.Job, error) { // commit transaction if err := sess.Commit(); err != nil { + getAsyncJobsMethod.LogAndCountError(err, "commit") return nil, err } + getAsyncJobsMethod.CountSuccess() return result, nil } func (h *Heimdall) runAsyncJob(j *job.Job) error { + // Track DB connection for async job execution + defer runAsyncJobMethod.RecordLatency(time.Now()) + runAsyncJobMethod.CountRequest() + // let's updte job status that we're running it... sess, err := h.Database.NewSession(false) if err != nil { + runAsyncJobMethod.LogAndCountError(err, "new_session") return h.updateAsyncJobStatus(j, err) } defer sess.Close() @@ -113,12 +133,17 @@ func (h *Heimdall) runAsyncJob(j *job.Job) error { return h.updateAsyncJobStatus(j, fmt.Errorf(formatErrUnknownCluster, j.CluserID)) } + runAsyncJobMethod.CountSuccess() return h.updateAsyncJobStatus(j, h.runJob(j, command, cluster)) } func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { + // Track DB connection for async job status update + defer updateAsyncJobStatusMethod.RecordLatency(time.Now()) + updateAsyncJobStatusMethod.CountRequest() + // we updte the final job status based on presence of the error if jobError == nil { j.Status = status.Succeeded @@ -130,8 +155,9 @@ func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { // now we update that status in the database sess, err := h.Database.NewSession(true) if err != nil { - // TODO: implement proper logging + updateAsyncJobStatusMethod.LogAndCountError(err, "new_session") fmt.Println(`session error:`, err) + return jobError // Return early if session creation fails } defer sess.Close() @@ -150,6 +176,7 @@ func (h *Heimdall) updateAsyncJobStatus(j *job.Job, jobError error) error { fmt.Println(`session commit error:`, err) } + updateAsyncJobStatusMethod.CountSuccess() return jobError } diff --git a/internal/pkg/heimdall/keepalive.go b/internal/pkg/heimdall/keepalive.go index 60da93c..4a4e0c1 100644 --- a/internal/pkg/heimdall/keepalive.go +++ b/internal/pkg/heimdall/keepalive.go @@ -3,12 +3,18 @@ package heimdall import ( _ "embed" "time" + + "github.com/hladush/go-telemetry/pkg/telemetry" ) const ( defaultJanitorKeepalive = 5 // seconds ) +var ( + jobKeepaliveMethod = telemetry.NewMethod("db_connection", "job_keepalive") +) + //go:embed queries/job/active_keepalive.sql var queryActiveJobKeepalive string @@ -22,9 +28,14 @@ func (h *Heimdall) jobKeepalive(done <-chan struct{}, jobID int64, agentName str ticker := time.NewTicker(time.Duration(keepaliveSeconds) * time.Second) defer ticker.Stop() + // Track DB connection for job keepalive + defer jobKeepaliveMethod.RecordLatency(time.Now()) + jobKeepaliveMethod.CountRequest() + // set the db session sess, err := h.Database.NewSession(false) if err != nil { + jobKeepaliveMethod.LogAndCountError(err, "new_session") sess = nil } defer sess.Close() @@ -39,6 +50,7 @@ func (h *Heimdall) jobKeepalive(done <-chan struct{}, jobID int64, agentName str } case _, stillOpen := <-done: if !stillOpen { + jobKeepaliveMethod.CountSuccess() return } } diff --git a/internal/pkg/janitor/job.go b/internal/pkg/janitor/job.go index 2e4ea80..9c40421 100644 --- a/internal/pkg/janitor/job.go +++ b/internal/pkg/janitor/job.go @@ -2,10 +2,16 @@ package janitor import ( _ "embed" + "time" + "github.com/hladush/go-telemetry/pkg/telemetry" "github.com/patterninc/heimdall/internal/pkg/database" ) +var ( + cleanUpStaleJobsMethod = telemetry.NewMethod("db_connection", "cleanup_stale_jobs") +) + //go:embed queries/stale_jobs_select.sql var queryStaleJobsSelect string @@ -17,15 +23,21 @@ var queryStaleJobsDelete string func (j *Janitor) cleanupStaleJobs() error { + // Track DB connection for stale jobs cleanup operations + defer cleanUpStaleJobsMethod.RecordLatency(time.Now()) + cleanUpStaleJobsMethod.CountRequest() + // let's find the jobs we'll be cleaning up... sess, err := j.db.NewSession(false) if err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "new_session") return err } defer sess.Close() rows, err := sess.Query(queryStaleJobsSelect, j.StaleJob) if err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "query") return err } defer rows.Close() @@ -37,6 +49,7 @@ func (j *Janitor) cleanupStaleJobs() error { var jobID int if err := rows.Scan(&jobID); err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "scan") return err } @@ -54,23 +67,28 @@ func (j *Janitor) cleanupStaleJobs() error { // prepare query to update job statuses updateStaleJobs, jobSystemIDs, err := database.PrepareSliceQuery(queryFailStaleJobs, `$%d`, staleJobIDs) if err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "prepare_slice_query") return err } if _, err := sess.Exec(updateStaleJobs, jobSystemIDs...); err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "exec") return err } // delete stale jobs from active jobs deleteStaleJobs, jobSystemIDs, err := database.PrepareSliceQuery(queryStaleJobsDelete, `$%d`, staleJobIDs) if err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "prepare_slice_query") return err } if _, err := sess.Exec(deleteStaleJobs, jobSystemIDs...); err != nil { + cleanUpStaleJobsMethod.LogAndCountError(err, "exec") return err } + cleanUpStaleJobsMethod.CountSuccess() return nil }