From b7825b609ebf3d086b9a68f8fa4beb5ea9f84b39 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Sun, 4 Jan 2026 03:32:07 +0000 Subject: [PATCH 1/9] cache infoschema --- pkg/scdb/storage/storage.go | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pkg/scdb/storage/storage.go b/pkg/scdb/storage/storage.go index 0179aae2..41bb5839 100644 --- a/pkg/scdb/storage/storage.go +++ b/pkg/scdb/storage/storage.go @@ -21,6 +21,7 @@ import ( "reflect" "sort" "strings" + "sync" "github.com/sethvargo/go-password/password" "gorm.io/gorm" @@ -53,6 +54,14 @@ var EnablePasswordCheck = false // keep creating order var allTables = []interface{}{&User{}, &Database{}, &Table{}, &Column{}, &DatabasePriv{}, &TablePriv{}, &ColumnPriv{}} +// InfoSchema cache to avoid frequent database queries +var ( + infoSchemaCache sync.Map // map[string]infoschema.InfoSchema, key is database name + infoCacheHitCount int64 + infoCacheMissCount int64 + infoCacheMutex sync.RWMutex +) + // NeedBootstrap checks if the store is empty func NeedBootstrap(store *gorm.DB) bool { for _, tn := range allTables { @@ -173,6 +182,33 @@ func FindUserByParty(store *gorm.DB, partyCode string) (*User, error) { return &user, nil } +// InvalidateInfoSchemaCache invalidates the InfoSchema cache for a specific database +// If dbName is empty, it invalidates all cached InfoSchemas +func InvalidateInfoSchemaCache(dbName string) { + if dbName == "" { + // Clear all cache + infoSchemaCache = sync.Map{} + } else { + // Clear specific database cache + infoSchemaCache.Delete(dbName) + } +} + +// ResetInfoSchemaCacheStats resets cache statistics (for testing) +func ResetInfoSchemaCacheStats() { + infoCacheMutex.Lock() + defer infoCacheMutex.Unlock() + infoCacheHitCount = 0 + infoCacheMissCount = 0 +} + +// GetInfoSchemaCacheStats returns cache hit and miss statistics +func GetInfoSchemaCacheStats() (hits, misses int64) { + infoCacheMutex.RLock() + defer infoCacheMutex.RUnlock() + return infoCacheHitCount, infoCacheMissCount +} + func QueryInfoSchema(store *gorm.DB) (result infoschema.InfoSchema, err error) { callFc := func(tx *gorm.DB) error { result, err = queryInfoSchema(tx) @@ -244,6 +280,19 @@ func queryInfoSchema(store *gorm.DB) (infoschema.InfoSchema, error) { } func QueryDBInfoSchema(store *gorm.DB, dbName string) (result infoschema.InfoSchema, err error) { + // Try to get from cache first + if cached, ok := infoSchemaCache.Load(dbName); ok { + infoCacheMutex.Lock() + infoCacheHitCount++ + infoCacheMutex.Unlock() + return cached.(infoschema.InfoSchema), nil + } + + // Cache miss, query from database + infoCacheMutex.Lock() + infoCacheMissCount++ + infoCacheMutex.Unlock() + callFc := func(tx *gorm.DB) error { result, err = queryDBInfoSchema(tx, dbName) return err @@ -251,6 +300,12 @@ func QueryDBInfoSchema(store *gorm.DB, dbName string) (result infoschema.InfoSch if err := store.Transaction(callFc, &sql.TxOptions{ReadOnly: true}); err != nil { return nil, fmt.Errorf("queryDBInfoSchema: %v", err) } + + // Store in cache + if result != nil { + infoSchemaCache.Store(dbName, result) + } + return result, nil } From 92690281260c76d48ebf87da1b8be07da8bb300f Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Sun, 4 Jan 2026 03:32:48 +0000 Subject: [PATCH 2/9] drop cache when schema changed --- pkg/scdb/executor/ddl.go | 14 ++++++++++++++ pkg/scdb/storage/schema_tables.go | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/pkg/scdb/executor/ddl.go b/pkg/scdb/executor/ddl.go index 8d8c9246..b8936e8a 100644 --- a/pkg/scdb/executor/ddl.go +++ b/pkg/scdb/executor/ddl.go @@ -113,6 +113,8 @@ func (e *DDLExec) executeCreateDatabase(s *ast.CreateDatabaseStmt) (err error) { if result.Error != nil { return fmt.Errorf("ddl.executeCreateDatabase: %v", result.Error) } + // Invalidate cache after successful DDL operation + storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -212,6 +214,8 @@ func (e *DDLExec) executeCreateTable(s *ast.CreateTableStmt) (err error) { } } + // Invalidate cache after successful DDL operation + storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -321,6 +325,8 @@ func (e *DDLExec) executeCreateView(s *ast.CreateViewStmt) (err error) { return fmt.Errorf("ddl.executeCreateView: %v", result.Error) } } + // Invalidate cache after successful DDL operation + storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -346,6 +352,10 @@ func (e *DDLExec) executeDropDatabase(s *ast.DropDatabaseStmt) (err error) { return fmt.Errorf("database %v not exists", dbName) } err = storage.NewDDLHandler(tx).DropSchema(model.NewCIStr(dbName)) + if err == nil { + // Invalidate cache after successful DDL operation (in case not done in DropSchema) + storage.InvalidateInfoSchemaCache(dbName) + } return err } @@ -404,5 +414,9 @@ func (e *DDLExec) executeDropTableOrView(s *ast.DropTableStmt) (err error) { return err } err = storage.NewDDLHandler(tx).DropTable(model.NewCIStr(dbName), model.NewCIStr(tblName)) + if err == nil { + // Invalidate cache after successful DDL operation (in case not done in DropTable) + storage.InvalidateInfoSchemaCache(dbName) + } return err } diff --git a/pkg/scdb/storage/schema_tables.go b/pkg/scdb/storage/schema_tables.go index f81dace8..b37ffe45 100644 --- a/pkg/scdb/storage/schema_tables.go +++ b/pkg/scdb/storage/schema_tables.go @@ -92,6 +92,8 @@ func (h *DDLHandler) DropSchema(schema model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropSchema: %v", err) } + // Invalidate cache after successful DDL operation + InvalidateInfoSchemaCache(schema.String()) return nil } @@ -102,6 +104,8 @@ func (h *DDLHandler) DropTable(schema, tblName model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropTable: %v", err) } + // Invalidate cache after successful DDL operation + InvalidateInfoSchemaCache(schema.String()) return nil } From 3503a6e43e818aa50feca7428056b619fea0e310 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Sun, 4 Jan 2026 03:46:27 +0000 Subject: [PATCH 3/9] add ut & remove unnecessary comments --- pkg/scdb/executor/ddl.go | 7 ++-- pkg/scdb/storage/schema_tables.go | 4 +-- pkg/scdb/storage/storage.go | 8 ++--- pkg/scdb/storage/storage_test.go | 57 +++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 11 deletions(-) diff --git a/pkg/scdb/executor/ddl.go b/pkg/scdb/executor/ddl.go index b8936e8a..013ed064 100644 --- a/pkg/scdb/executor/ddl.go +++ b/pkg/scdb/executor/ddl.go @@ -113,7 +113,7 @@ func (e *DDLExec) executeCreateDatabase(s *ast.CreateDatabaseStmt) (err error) { if result.Error != nil { return fmt.Errorf("ddl.executeCreateDatabase: %v", result.Error) } - // Invalidate cache after successful DDL operation + storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -214,7 +214,6 @@ func (e *DDLExec) executeCreateTable(s *ast.CreateTableStmt) (err error) { } } - // Invalidate cache after successful DDL operation storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -325,7 +324,7 @@ func (e *DDLExec) executeCreateView(s *ast.CreateViewStmt) (err error) { return fmt.Errorf("ddl.executeCreateView: %v", result.Error) } } - // Invalidate cache after successful DDL operation + storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -353,7 +352,6 @@ func (e *DDLExec) executeDropDatabase(s *ast.DropDatabaseStmt) (err error) { } err = storage.NewDDLHandler(tx).DropSchema(model.NewCIStr(dbName)) if err == nil { - // Invalidate cache after successful DDL operation (in case not done in DropSchema) storage.InvalidateInfoSchemaCache(dbName) } return err @@ -415,7 +413,6 @@ func (e *DDLExec) executeDropTableOrView(s *ast.DropTableStmt) (err error) { } err = storage.NewDDLHandler(tx).DropTable(model.NewCIStr(dbName), model.NewCIStr(tblName)) if err == nil { - // Invalidate cache after successful DDL operation (in case not done in DropTable) storage.InvalidateInfoSchemaCache(dbName) } return err diff --git a/pkg/scdb/storage/schema_tables.go b/pkg/scdb/storage/schema_tables.go index b37ffe45..3ffafbfc 100644 --- a/pkg/scdb/storage/schema_tables.go +++ b/pkg/scdb/storage/schema_tables.go @@ -92,7 +92,7 @@ func (h *DDLHandler) DropSchema(schema model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropSchema: %v", err) } - // Invalidate cache after successful DDL operation + InvalidateInfoSchemaCache(schema.String()) return nil } @@ -104,7 +104,7 @@ func (h *DDLHandler) DropTable(schema, tblName model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropTable: %v", err) } - // Invalidate cache after successful DDL operation + InvalidateInfoSchemaCache(schema.String()) return nil } diff --git a/pkg/scdb/storage/storage.go b/pkg/scdb/storage/storage.go index 41bb5839..f69d0cd4 100644 --- a/pkg/scdb/storage/storage.go +++ b/pkg/scdb/storage/storage.go @@ -56,10 +56,10 @@ var allTables = []interface{}{&User{}, &Database{}, &Table{}, &Column{}, &Databa // InfoSchema cache to avoid frequent database queries var ( - infoSchemaCache sync.Map // map[string]infoschema.InfoSchema, key is database name - infoCacheHitCount int64 - infoCacheMissCount int64 - infoCacheMutex sync.RWMutex + infoSchemaCache sync.Map // map[string]infoschema.InfoSchema, key is database name + infoCacheHitCount int64 + infoCacheMissCount int64 + infoCacheMutex sync.RWMutex ) // NeedBootstrap checks if the store is empty diff --git a/pkg/scdb/storage/storage_test.go b/pkg/scdb/storage/storage_test.go index c330bec5..eb036640 100644 --- a/pkg/scdb/storage/storage_test.go +++ b/pkg/scdb/storage/storage_test.go @@ -107,3 +107,60 @@ func batchInsert(db *gorm.DB, records []interface{}) error { } return nil } + +func createTestDB(t *testing.T, db *gorm.DB, dbName string) { + r := require.New(t) + records := []interface{}{ + &Database{Db: dbName}, + &Table{Db: dbName, Table: "t1", Owner: "root", Host: "%", RefDb: "ref", RefTable: "ref", DBType: 0}, + &Column{Db: dbName, TableName: "t1", ColumnName: "c1", Type: "int"}, + } + r.NoError(batchInsert(db, records)) +} + +func TestInfoSchemaCache(t *testing.T) { + r := require.New(t) + db, err := newDbStore() + r.NoError(err) + + InvalidateInfoSchemaCache("") + ResetInfoSchemaCacheStats() + + createTestDB(t, db, "db1") + createTestDB(t, db, "db2") + + // Test cache miss and hit + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses := GetInfoSchemaCacheStats() + r.Equal(int64(0), hits) + r.Equal(int64(1), misses) + + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(1), hits) + r.Equal(int64(1), misses) + + // Test cache invalidation + InvalidateInfoSchemaCache("db1") + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(1), hits) + r.Equal(int64(2), misses) + + // Test multiple databases cache isolation + _, err = QueryDBInfoSchema(db, "db2") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(1), hits) + r.Equal(int64(3), misses) + + InvalidateInfoSchemaCache("db2") + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(2), hits) // db1 still cached + r.Equal(int64(3), misses) +} From 20b6b18309923efdf3d8585c0747c0c6afad7e23 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Sun, 4 Jan 2026 05:55:09 +0000 Subject: [PATCH 4/9] opt lock --- pkg/scdb/executor/ddl.go | 6 ------ pkg/scdb/storage/storage.go | 31 +++++++++++++------------------ 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/pkg/scdb/executor/ddl.go b/pkg/scdb/executor/ddl.go index 013ed064..2ea99e8f 100644 --- a/pkg/scdb/executor/ddl.go +++ b/pkg/scdb/executor/ddl.go @@ -351,9 +351,6 @@ func (e *DDLExec) executeDropDatabase(s *ast.DropDatabaseStmt) (err error) { return fmt.Errorf("database %v not exists", dbName) } err = storage.NewDDLHandler(tx).DropSchema(model.NewCIStr(dbName)) - if err == nil { - storage.InvalidateInfoSchemaCache(dbName) - } return err } @@ -412,8 +409,5 @@ func (e *DDLExec) executeDropTableOrView(s *ast.DropTableStmt) (err error) { return err } err = storage.NewDDLHandler(tx).DropTable(model.NewCIStr(dbName), model.NewCIStr(tblName)) - if err == nil { - storage.InvalidateInfoSchemaCache(dbName) - } return err } diff --git a/pkg/scdb/storage/storage.go b/pkg/scdb/storage/storage.go index f69d0cd4..3a361824 100644 --- a/pkg/scdb/storage/storage.go +++ b/pkg/scdb/storage/storage.go @@ -22,6 +22,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "github.com/sethvargo/go-password/password" "gorm.io/gorm" @@ -57,9 +58,8 @@ var allTables = []interface{}{&User{}, &Database{}, &Table{}, &Column{}, &Databa // InfoSchema cache to avoid frequent database queries var ( infoSchemaCache sync.Map // map[string]infoschema.InfoSchema, key is database name - infoCacheHitCount int64 - infoCacheMissCount int64 - infoCacheMutex sync.RWMutex + infoCacheHitCount int64 // accessed via atomic operations + infoCacheMissCount int64 // accessed via atomic operations ) // NeedBootstrap checks if the store is empty @@ -186,8 +186,11 @@ func FindUserByParty(store *gorm.DB, partyCode string) (*User, error) { // If dbName is empty, it invalidates all cached InfoSchemas func InvalidateInfoSchemaCache(dbName string) { if dbName == "" { - // Clear all cache - infoSchemaCache = sync.Map{} + // Clear all cache by iterating and deleting each key to avoid race conditions + infoSchemaCache.Range(func(key, value interface{}) bool { + infoSchemaCache.Delete(key) + return true + }) } else { // Clear specific database cache infoSchemaCache.Delete(dbName) @@ -196,17 +199,13 @@ func InvalidateInfoSchemaCache(dbName string) { // ResetInfoSchemaCacheStats resets cache statistics (for testing) func ResetInfoSchemaCacheStats() { - infoCacheMutex.Lock() - defer infoCacheMutex.Unlock() - infoCacheHitCount = 0 - infoCacheMissCount = 0 + atomic.StoreInt64(&infoCacheHitCount, 0) + atomic.StoreInt64(&infoCacheMissCount, 0) } // GetInfoSchemaCacheStats returns cache hit and miss statistics func GetInfoSchemaCacheStats() (hits, misses int64) { - infoCacheMutex.RLock() - defer infoCacheMutex.RUnlock() - return infoCacheHitCount, infoCacheMissCount + return atomic.LoadInt64(&infoCacheHitCount), atomic.LoadInt64(&infoCacheMissCount) } func QueryInfoSchema(store *gorm.DB) (result infoschema.InfoSchema, err error) { @@ -282,16 +281,12 @@ func queryInfoSchema(store *gorm.DB) (infoschema.InfoSchema, error) { func QueryDBInfoSchema(store *gorm.DB, dbName string) (result infoschema.InfoSchema, err error) { // Try to get from cache first if cached, ok := infoSchemaCache.Load(dbName); ok { - infoCacheMutex.Lock() - infoCacheHitCount++ - infoCacheMutex.Unlock() + atomic.AddInt64(&infoCacheHitCount, 1) return cached.(infoschema.InfoSchema), nil } // Cache miss, query from database - infoCacheMutex.Lock() - infoCacheMissCount++ - infoCacheMutex.Unlock() + atomic.AddInt64(&infoCacheMissCount, 1) callFc := func(tx *gorm.DB) error { result, err = queryDBInfoSchema(tx, dbName) From 518a6186771d70da5541d87da1ce8295a9233520 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 5 Jan 2026 09:37:15 +0000 Subject: [PATCH 5/9] concentrate invalidate --- pkg/scdb/executor/ddl.go | 32 ++++++++++++++++++++++++++----- pkg/scdb/storage/schema_tables.go | 4 ---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/scdb/executor/ddl.go b/pkg/scdb/executor/ddl.go index 2ea99e8f..a41b41c1 100644 --- a/pkg/scdb/executor/ddl.go +++ b/pkg/scdb/executor/ddl.go @@ -53,14 +53,41 @@ func (e *DDLExec) Next(ctx context.Context, req *chunk.Chunk) (err error) { switch x := e.stmt.(type) { case *ast.CreateDatabaseStmt: err = e.executeCreateDatabase(x) + if err == nil { + storage.InvalidateInfoSchemaCache(strings.ToLower(x.Name)) + } case *ast.CreateTableStmt: err = e.executeCreateTable(x) + if err == nil { + dbName := x.Table.Schema.O + if dbName == "" { + dbName = e.ctx.GetSessionVars().CurrentDB + } + storage.InvalidateInfoSchemaCache(dbName) + } case *ast.DropDatabaseStmt: err = e.executeDropDatabase(x) + if err == nil { + storage.InvalidateInfoSchemaCache(strings.ToLower(x.Name)) + } case *ast.DropTableStmt: err = e.executeDropTableOrView(x) + if err == nil && len(x.Tables) > 0 { + dbName := x.Tables[0].Schema.L + if dbName == "" { + dbName = e.ctx.GetSessionVars().CurrentDB + } + storage.InvalidateInfoSchemaCache(dbName) + } case *ast.CreateViewStmt: err = e.executeCreateView(x) + if err == nil { + dbName := x.ViewName.Schema.L + if dbName == "" { + dbName = e.ctx.GetSessionVars().CurrentDB + } + storage.InvalidateInfoSchemaCache(dbName) + } default: err = fmt.Errorf("ddl.Next: Unsupported statement %v", x) @@ -113,8 +140,6 @@ func (e *DDLExec) executeCreateDatabase(s *ast.CreateDatabaseStmt) (err error) { if result.Error != nil { return fmt.Errorf("ddl.executeCreateDatabase: %v", result.Error) } - - storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -214,7 +239,6 @@ func (e *DDLExec) executeCreateTable(s *ast.CreateTableStmt) (err error) { } } - storage.InvalidateInfoSchemaCache(dbName) return nil } @@ -324,8 +348,6 @@ func (e *DDLExec) executeCreateView(s *ast.CreateViewStmt) (err error) { return fmt.Errorf("ddl.executeCreateView: %v", result.Error) } } - - storage.InvalidateInfoSchemaCache(dbName) return nil } diff --git a/pkg/scdb/storage/schema_tables.go b/pkg/scdb/storage/schema_tables.go index 3ffafbfc..f81dace8 100644 --- a/pkg/scdb/storage/schema_tables.go +++ b/pkg/scdb/storage/schema_tables.go @@ -92,8 +92,6 @@ func (h *DDLHandler) DropSchema(schema model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropSchema: %v", err) } - - InvalidateInfoSchemaCache(schema.String()) return nil } @@ -104,8 +102,6 @@ func (h *DDLHandler) DropTable(schema, tblName model.CIStr) error { if err := h.store.Transaction(callFc); err != nil { return fmt.Errorf("dropTable: %v", err) } - - InvalidateInfoSchemaCache(schema.String()) return nil } From 7e4d13f9b1ef09ee1a001fdfd313fbc4fa239b28 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 5 Jan 2026 11:57:24 +0000 Subject: [PATCH 6/9] infoschema cache configurable --- pkg/scdb/config/config.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/scdb/config/config.go b/pkg/scdb/config/config.go index 251788fb..ec4a443a 100644 --- a/pkg/scdb/config/config.go +++ b/pkg/scdb/config/config.go @@ -38,6 +38,7 @@ const ( DefaultProtocol = "https" DefaultLogLevel = "info" DefaultEngineClientMode = "HTTP" + DefaultInfoSchemaCacheTTL = 10 * time.Minute // 10 minutes ) type EngineConfig struct { @@ -62,6 +63,11 @@ type SecurityCompromiseConf struct { RevealGroupCount bool `yaml:"reveal_group_count"` } +type InfoSchemaCacheConf struct { + Enabled bool `yaml:"enabled"` + TTL time.Duration `yaml:"ttl"` +} + // Config contains bootstrap configuration for SCDB type Config struct { // SCDBHost is used as callback url for engine worked in async mode @@ -79,6 +85,7 @@ type Config struct { Engine EngineConfig `yaml:"engine"` SecurityCompromise SecurityCompromiseConf `yaml:"security_compromise"` PartyAuth PartyAuthConf `yaml:"party_auth"` + InfoSchemaCache InfoSchemaCacheConf `yaml:"infoschema_cache"` } const ( @@ -164,6 +171,10 @@ func NewDefaultConfig() *Config { EnableTimestampCheck: true, ValidityPeriod: 30 * time.Second, // 30s } + config.InfoSchemaCache = InfoSchemaCacheConf{ + Enabled: true, + TTL: DefaultInfoSchemaCacheTTL, + } return &config } From a4d9d9ea55f57be680b842fcfc222ee30392e9be Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 5 Jan 2026 11:57:59 +0000 Subject: [PATCH 7/9] enable infoschema cache --- cmd/scdbserver/config.yml | 3 +++ cmd/scdbserver/main.go | 1 + pkg/scdb/storage/storage.go | 47 ++++++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/cmd/scdbserver/config.yml b/cmd/scdbserver/config.yml index 61000cd8..db7421d1 100644 --- a/cmd/scdbserver/config.yml +++ b/cmd/scdbserver/config.yml @@ -21,3 +21,6 @@ engine: "protocol": "SEMI2K", "field": "FM64" } + infoschema_cache: + enabled: true + ttl: 10m diff --git a/cmd/scdbserver/main.go b/cmd/scdbserver/main.go index 2d1d651c..47f6d0db 100644 --- a/cmd/scdbserver/main.go +++ b/cmd/scdbserver/main.go @@ -87,6 +87,7 @@ func main() { log.Info("Starting to connect to database and do bootstrap if necessary...") storage.InitPasswordValidation(cfg.PasswordCheck) + storage.InitInfoSchemaCache(cfg.InfoSchemaCache.Enabled, cfg.InfoSchemaCache.TTL) store, err := server.NewDbConnWithBootstrap(&cfg.Storage) if err != nil { log.Fatalf("Failed to connect to database and bootstrap it: %v", err) diff --git a/pkg/scdb/storage/storage.go b/pkg/scdb/storage/storage.go index 3a361824..77de02e5 100644 --- a/pkg/scdb/storage/storage.go +++ b/pkg/scdb/storage/storage.go @@ -23,6 +23,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "github.com/sethvargo/go-password/password" "gorm.io/gorm" @@ -55,13 +56,30 @@ var EnablePasswordCheck = false // keep creating order var allTables = []interface{}{&User{}, &Database{}, &Table{}, &Column{}, &DatabasePriv{}, &TablePriv{}, &ColumnPriv{}} +// InfoSchema cache entry with TTL support +type infoCacheEntry struct { + schema infoschema.InfoSchema + createTime time.Time +} + // InfoSchema cache to avoid frequent database queries var ( - infoSchemaCache sync.Map // map[string]infoschema.InfoSchema, key is database name + infoSchemaCache sync.Map // map[string]*infoCacheEntry, key is database name infoCacheHitCount int64 // accessed via atomic operations infoCacheMissCount int64 // accessed via atomic operations + cacheEnabled bool // whether cache is enabled + cacheTTL time.Duration ) +// InitInfoSchemaCache initializes the InfoSchema cache configuration +func InitInfoSchemaCache(enabled bool, ttl time.Duration) { + cacheEnabled = enabled + cacheTTL = ttl + if ttl <= 0 { + ttl = 10 * time.Minute // default to 10 minutes + } +} + // NeedBootstrap checks if the store is empty func NeedBootstrap(store *gorm.DB) bool { for _, tn := range allTables { @@ -279,13 +297,21 @@ func queryInfoSchema(store *gorm.DB) (infoschema.InfoSchema, error) { } func QueryDBInfoSchema(store *gorm.DB, dbName string) (result infoschema.InfoSchema, err error) { - // Try to get from cache first - if cached, ok := infoSchemaCache.Load(dbName); ok { - atomic.AddInt64(&infoCacheHitCount, 1) - return cached.(infoschema.InfoSchema), nil + // Try to get from cache first if enabled + if cacheEnabled { + if cached, ok := infoSchemaCache.Load(dbName); ok { + entry := cached.(*infoCacheEntry) + // Check if cache entry is still valid (not expired) + if time.Since(entry.createTime) < cacheTTL { + atomic.AddInt64(&infoCacheHitCount, 1) + return entry.schema, nil + } + // Cache expired, remove it + infoSchemaCache.Delete(dbName) + } } - // Cache miss, query from database + // Cache miss or disabled, query from database atomic.AddInt64(&infoCacheMissCount, 1) callFc := func(tx *gorm.DB) error { @@ -296,9 +322,12 @@ func QueryDBInfoSchema(store *gorm.DB, dbName string) (result infoschema.InfoSch return nil, fmt.Errorf("queryDBInfoSchema: %v", err) } - // Store in cache - if result != nil { - infoSchemaCache.Store(dbName, result) + // Store in cache if enabled + if cacheEnabled && result != nil { + infoSchemaCache.Store(dbName, &infoCacheEntry{ + schema: result, + createTime: time.Now(), + }) } return result, nil From 02832526a3b7ff3289ba85186378d13814afcc5d Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Mon, 5 Jan 2026 11:58:10 +0000 Subject: [PATCH 8/9] add ut --- pkg/scdb/storage/storage_test.go | 66 ++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/pkg/scdb/storage/storage_test.go b/pkg/scdb/storage/storage_test.go index eb036640..708b17c2 100644 --- a/pkg/scdb/storage/storage_test.go +++ b/pkg/scdb/storage/storage_test.go @@ -16,6 +16,7 @@ package storage import ( "testing" + "time" "github.com/stretchr/testify/require" "gorm.io/driver/sqlite" @@ -123,6 +124,8 @@ func TestInfoSchemaCache(t *testing.T) { db, err := newDbStore() r.NoError(err) + // Initialize cache with enabled=true and a reasonable TTL + InitInfoSchemaCache(true, 5*time.Minute) InvalidateInfoSchemaCache("") ResetInfoSchemaCacheStats() @@ -164,3 +167,66 @@ func TestInfoSchemaCache(t *testing.T) { r.Equal(int64(2), hits) // db1 still cached r.Equal(int64(3), misses) } + +func TestInfoSchemaCacheTTL(t *testing.T) { + r := require.New(t) + db, err := newDbStore() + r.NoError(err) + + // Initialize cache with a short TTL for testing + InitInfoSchemaCache(true, 100*time.Millisecond) + InvalidateInfoSchemaCache("") + ResetInfoSchemaCacheStats() + + createTestDB(t, db, "db1") + + // First query - cache miss + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses := GetInfoSchemaCacheStats() + r.Equal(int64(0), hits) + r.Equal(int64(1), misses) + + // Second query immediately - cache hit + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(1), hits) + r.Equal(int64(1), misses) + + // Wait for cache to expire + time.Sleep(150 * time.Millisecond) + + // Query after TTL expired - should be cache miss + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(1), hits) + r.Equal(int64(2), misses) +} + +func TestInfoSchemaCacheDisabled(t *testing.T) { + r := require.New(t) + db, err := newDbStore() + r.NoError(err) + + // Disable cache + InitInfoSchemaCache(false, 5*time.Minute) + InvalidateInfoSchemaCache("") + ResetInfoSchemaCacheStats() + + createTestDB(t, db, "db1") + + // All queries should be cache misses when disabled + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses := GetInfoSchemaCacheStats() + r.Equal(int64(0), hits) + r.Equal(int64(1), misses) + + _, err = QueryDBInfoSchema(db, "db1") + r.NoError(err) + hits, misses = GetInfoSchemaCacheStats() + r.Equal(int64(0), hits) + r.Equal(int64(2), misses) +} From 4b90c3133d61f35d7d932321e4389be9fd948140 Mon Sep 17 00:00:00 2001 From: Song-Quan Date: Tue, 6 Jan 2026 07:42:58 +0000 Subject: [PATCH 9/9] fix go lint & update config ut --- pkg/scdb/config/config.go | 2 +- pkg/scdb/config/config_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/scdb/config/config.go b/pkg/scdb/config/config.go index ec4a443a..1f144e82 100644 --- a/pkg/scdb/config/config.go +++ b/pkg/scdb/config/config.go @@ -38,7 +38,7 @@ const ( DefaultProtocol = "https" DefaultLogLevel = "info" DefaultEngineClientMode = "HTTP" - DefaultInfoSchemaCacheTTL = 10 * time.Minute // 10 minutes + DefaultInfoSchemaCacheTTL = 10 * time.Minute // 10 minutes ) type EngineConfig struct { diff --git a/pkg/scdb/config/config_test.go b/pkg/scdb/config/config_test.go index 32b89065..69a6f7a7 100644 --- a/pkg/scdb/config/config_test.go +++ b/pkg/scdb/config/config_test.go @@ -114,6 +114,10 @@ security_compromise: ValidityPeriod: time.Minute * 3, }, SecurityCompromise: SecurityCompromiseConf{GroupByThreshold: 3}, + InfoSchemaCache: InfoSchemaCacheConf{ + Enabled: true, + TTL: DefaultInfoSchemaCacheTTL, + }, } expectedCfg.Engine.SpuRuntimeCfg = strings.ReplaceAll(expectedCfg.Engine.SpuRuntimeCfg, "\t", " ") r.Equal(expectedCfg, cfg) @@ -206,6 +210,10 @@ party_auth: EnableTimestampCheck: true, ValidityPeriod: time.Minute * 3, }, + InfoSchemaCache: InfoSchemaCacheConf{ + Enabled: true, + TTL: DefaultInfoSchemaCacheTTL, + }, } expectedCfg.Engine.SpuRuntimeCfg = strings.ReplaceAll(expectedCfg.Engine.SpuRuntimeCfg, "\t", " ") r.Equal(expectedCfg, cfg)