diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ddf08f4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Exclude node_modules from Docker builds +ffm/node_modules +ffm/dist +ffm/.git +ffm/.DS_Store +ffm/*.log + +# Exclude other build artifacts +api/tmp +*.log +.DS_Store +.git +.gitignore diff --git a/Makefile b/Makefile index 568403b..f3707ed 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ standalone-down: ## Stop standalone production container standalone-build: ## Build standalone production container @echo "$(GREEN)Building standalone production container...$(NC)" - docker compose -p bfm-standalone -f $(COMPOSE_STANDALONE_FILE) build + docker compose -p bfm-standalone -f $(COMPOSE_STANDALONE_FILE) build --no-cache standalone-logs: ## Show logs from standalone container docker compose -p bfm-standalone -f $(COMPOSE_STANDALONE_FILE) logs -f diff --git a/api/cmd/server/main.go b/api/cmd/server/main.go index 2364a7d..b5f1eef 100644 --- a/api/cmd/server/main.go +++ b/api/cmd/server/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "net" "net/http" @@ -21,6 +22,7 @@ import ( "github.com/toolsascode/bfm/api/internal/logger" "github.com/toolsascode/bfm/api/internal/queuefactory" "github.com/toolsascode/bfm/api/internal/registry" + "github.com/toolsascode/bfm/api/internal/state" statepg "github.com/toolsascode/bfm/api/internal/state/postgresql" "github.com/gin-gonic/gin" @@ -114,7 +116,7 @@ func main() { if migrationCount == 0 { logger.Warnf("No migrations loaded from %s - ensure migration files exist in the expected directory structure", sfmPath) } else { - logger.Infof("✅ Successfully loaded %d migration(s) from %s", migrationCount, sfmPath) + logger.Infof("Successfully loaded %d migration(s) from %s", migrationCount, sfmPath) // Log migration breakdown by backend/connection for better visibility allMigrations := registry.GlobalRegistry.GetAll() @@ -137,15 +139,56 @@ func main() { loader.StartWatching() defer loader.StopWatching() + // Start background reindexer + reindexInterval := 5 * time.Minute + if intervalStr := os.Getenv("BFM_REINDEX_INTERVAL_MINUTES"); intervalStr != "" { + if intervalMinutes, err := time.ParseDuration(intervalStr + "m"); err == nil { + reindexInterval = intervalMinutes + } + } + reindexer := state.NewReindexer(stateTracker, registry.GlobalRegistry, reindexInterval) + reindexer.Start() + defer reindexer.Stop() + logger.Infof("Background reindexer started with interval: %v", reindexInterval) + + // Set Gin mode - use BFM_APP_MODE env var if set, otherwise default to release mode + if ginMode := os.Getenv("BFM_APP_MODE"); ginMode != "" { + gin.SetMode(ginMode) + } else { + gin.SetMode(gin.ReleaseMode) + } + // Initialize HTTP server router := gin.New() - // Custom logger middleware that skips health check endpoints + // Determine log format from environment variable (default to JSON) + logFormat := strings.ToLower(os.Getenv("BFM_LOG_FORMAT")) + useJSON := logFormat != "plaintext" && logFormat != "plain" && logFormat != "text" + + // Custom logger middleware that skips health check endpoints and supports JSON/plaintext router.Use(gin.LoggerWithFormatter(func(param gin.LogFormatterParams) string { // Skip logging for health check endpoints if param.Path == "/health" || param.Path == "/api/v1/health" { return "" } + + if useJSON { + // JSON format + logEntry := map[string]interface{}{ + "timestamp": param.TimeStamp.Format("2006-01-02T15:04:05.000Z07:00"), + "status": param.StatusCode, + "latency": param.Latency.String(), + "client_ip": param.ClientIP, + "method": param.Method, + "path": param.Path, + "error": param.ErrorMessage, + } + if jsonBytes, err := json.Marshal(logEntry); err == nil { + return string(jsonBytes) + "\n" + } + } + + // Plaintext format (fallback or when explicitly set) return fmt.Sprintf("[GIN] %s | %3d | %13v | %15s | %-7s %s\n", param.TimeStamp.Format("2006/01/02 - 15:04:05"), param.StatusCode, diff --git a/api/deploy/docker-compose.servers.yml b/api/deploy/docker-compose.servers.yml index 0e9c3ec..75b105d 100644 --- a/api/deploy/docker-compose.servers.yml +++ b/api/deploy/docker-compose.servers.yml @@ -15,6 +15,11 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres} - POSTGRES_DB=migration_state + command: > + postgres + -c max_connections=200 + -c shared_buffers=256MB + -c effective_cache_size=1GB ports: - "5433:5432" volumes: diff --git a/api/go.mod b/api/go.mod index 4288f3c..ec097f8 100644 --- a/api/go.mod +++ b/api/go.mod @@ -7,6 +7,7 @@ require ( github.com/gin-gonic/gin v1.10.1 github.com/lib/pq v1.10.9 github.com/segmentio/kafka-go v0.4.49 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.10.2 go.etcd.io/etcd/client/v3 v3.5.10 google.golang.org/grpc v1.77.0 @@ -65,7 +66,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/api/internal/api/http/dto/migrations.go b/api/internal/api/http/dto/migrations.go index be210cd..524e0d4 100644 --- a/api/internal/api/http/dto/migrations.go +++ b/api/internal/api/http/dto/migrations.go @@ -59,6 +59,11 @@ type MigrationDetailResponse struct { StructuredDependencies []DependencyResponse `json:"structured_dependencies,omitempty"` // Structured dependencies with validation requirements } +// RollbackRequest represents a request to rollback a migration +type RollbackRequest struct { + Schemas []string `json:"schemas,omitempty"` // Array for dynamic schemas +} + // RollbackResponse represents a rollback operation result type RollbackResponse struct { Success bool `json:"success"` @@ -82,6 +87,21 @@ type MigrateUpRequest struct { DryRun bool `json:"dry_run"` } +// MigrationExecutionResponse represents an execution record from migrations_executions +type MigrationExecutionResponse struct { + ID int `json:"id"` + MigrationID string `json:"migration_id"` + Schema string `json:"schema"` + Version string `json:"version"` + Connection string `json:"connection"` + Backend string `json:"backend"` + Status string `json:"status"` + Applied bool `json:"applied"` + AppliedAt string `json:"applied_at,omitempty"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + // MigrateDownRequest represents a request to execute down migrations type MigrateDownRequest struct { MigrationID string `json:"migration_id" binding:"required"` diff --git a/api/internal/api/http/handler.go b/api/internal/api/http/handler.go index 4e95c3a..cc75a33 100644 --- a/api/internal/api/http/handler.go +++ b/api/internal/api/http/handler.go @@ -5,6 +5,7 @@ import ( _ "embed" "net/http" "os" + "strconv" "strings" "time" @@ -44,6 +45,8 @@ func (h *Handler) RegisterRoutes(router *gin.Engine) { api.GET("/migrations/:id", h.authenticate, h.getMigration) api.GET("/migrations/:id/status", h.authenticate, h.getMigrationStatus) api.GET("/migrations/:id/history", h.authenticate, h.getMigrationHistory) + api.GET("/migrations/:id/executions", h.authenticate, h.getMigrationExecutions) + api.GET("/migrations/executions/recent", h.authenticate, h.getRecentExecutions) api.POST("/migrations/:id/rollback", h.authenticate, h.rollbackMigration) api.POST("/migrations/reindex", h.authenticate, h.reindexMigrations) api.GET("/health", h.Health) @@ -289,65 +292,140 @@ func (h *Handler) getMigration(c *gin.Context) { // Get migration from registry migration := h.executor.GetMigrationByID(migrationID) - if migration == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) - return + + // Get migration details from database (migrations_list table) + // This is the source of truth for dependencies and metadata + dbDetail, err := h.executor.GetMigrationDetail(c.Request.Context(), migrationID) + var schemaValue, tableValue, versionValue, nameValue, connectionValue, backendValue string + var foundMigrationID string + var dbDependencies []string + var dbStructuredDeps []dto.DependencyResponse + + if err == nil && dbDetail != nil { + schemaValue = dbDetail.Schema + versionValue = dbDetail.Version + nameValue = dbDetail.Name + connectionValue = dbDetail.Connection + backendValue = dbDetail.Backend + foundMigrationID = dbDetail.MigrationID + dbDependencies = dbDetail.Dependencies + // Convert structured dependencies from database + for _, dep := range dbDetail.StructuredDependencies { + dbStructuredDeps = append(dbStructuredDeps, dto.DependencyResponse{ + Connection: dep.Connection, + Schema: dep.Schema, + Target: dep.Target, + TargetType: dep.TargetType, + RequiresTable: dep.RequiresTable, + RequiresSchema: dep.RequiresSchema, + }) + } + } + + // Use foundMigrationID for status check if we found a schema-specific version, otherwise use original migrationID + statusCheckID := foundMigrationID + if statusCheckID == "" { + statusCheckID = migrationID } // Get status from state tracker - applied, err := h.executor.IsMigrationApplied(c.Request.Context(), migrationID) + applied, err := h.executor.IsMigrationApplied(c.Request.Context(), statusCheckID) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - // Get schema and table from state tracker (migrations_list table) - // These are populated when the migration is executed or registered - var schemaValue, tableValue string - migrationList, err := h.executor.GetMigrationList(c.Request.Context(), &state.MigrationFilters{}) - if err == nil { - for _, item := range migrationList { - if item.MigrationID == migrationID { - schemaValue = item.Schema - tableValue = item.Table - break + // If migration not found in registry, check if it exists in database + if migration == nil { + // If we found it in the database, construct response from database data + if versionValue != "" { + // Use the found migration ID (which might be schema-specific) or the requested one + responseMigrationID := foundMigrationID + if responseMigrationID == "" { + responseMigrationID = migrationID } + response := dto.MigrationDetailResponse{ + MigrationID: responseMigrationID, + Schema: schemaValue, + Table: tableValue, + Version: versionValue, + Name: nameValue, + Connection: connectionValue, + Backend: backendValue, + Applied: applied, + UpSQL: "", // Not available if not in registry + DownSQL: "", // Not available if not in registry + Dependencies: dbDependencies, + StructuredDependencies: dbStructuredDeps, + } + c.JSON(http.StatusOK, response) + return } + // Migration not found in registry or database + c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) + return } - // Fallback to registry values if not found in state tracker + // Migration found in registry, use database values for dependencies (source of truth) + // but use registry values for UpSQL/DownSQL (only available in registry) if tableValue == "" && migration.Table != nil { tableValue = *migration.Table } if schemaValue == "" { schemaValue = migration.Schema } + if versionValue == "" { + versionValue = migration.Version + } + if nameValue == "" { + nameValue = migration.Name + } + if connectionValue == "" { + connectionValue = migration.Connection + } + if backendValue == "" { + backendValue = migration.Backend + } + + // Use dependencies from database (migrations_list) as source of truth + // Fall back to registry if database doesn't have them + dependencies := dbDependencies + if len(dependencies) == 0 { + dependencies = migration.Dependencies + } + structuredDeps := dbStructuredDeps + if len(structuredDeps) == 0 { + // Convert structured dependencies from registry if database doesn't have them + for _, dep := range migration.StructuredDependencies { + structuredDeps = append(structuredDeps, dto.DependencyResponse{ + Connection: dep.Connection, + Schema: dep.Schema, + Target: dep.Target, + TargetType: dep.TargetType, + RequiresTable: dep.RequiresTable, + RequiresSchema: dep.RequiresSchema, + }) + } + } - // Convert structured dependencies to response format - structuredDeps := make([]dto.DependencyResponse, 0, len(migration.StructuredDependencies)) - for _, dep := range migration.StructuredDependencies { - structuredDeps = append(structuredDeps, dto.DependencyResponse{ - Connection: dep.Connection, - Schema: dep.Schema, - Target: dep.Target, - TargetType: dep.TargetType, - RequiresTable: dep.RequiresTable, - RequiresSchema: dep.RequiresSchema, - }) + // Use foundMigrationID if we found a schema-specific version, otherwise use the original migrationID + responseMigrationID := foundMigrationID + if responseMigrationID == "" { + responseMigrationID = migrationID } response := dto.MigrationDetailResponse{ - MigrationID: migrationID, + MigrationID: responseMigrationID, Schema: schemaValue, Table: tableValue, - Version: migration.Version, - Name: migration.Name, - Connection: migration.Connection, - Backend: migration.Backend, + Version: versionValue, + Name: nameValue, + Connection: connectionValue, + Backend: backendValue, Applied: applied, UpSQL: migration.UpSQL, DownSQL: migration.DownSQL, - Dependencies: migration.Dependencies, + Dependencies: dependencies, StructuredDependencies: structuredDeps, } @@ -464,11 +542,26 @@ func (h *Handler) getMigrationStatus(c *gin.Context) { func (h *Handler) getMigrationHistory(c *gin.Context) { migrationID := c.Param("id") - // Get migration from registry to verify it exists + // Check if migration exists in registry or database migration := h.executor.GetMigrationByID(migrationID) if migration == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) - return + // Check if migration exists in database + migrationList, err := h.executor.GetMigrationList(c.Request.Context(), &state.MigrationFilters{}) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + foundInDB := false + for _, item := range migrationList { + if item.MigrationID == migrationID { + foundInDB = true + break + } + } + if !foundInDB { + c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) + return + } } // Get all migration history @@ -516,42 +609,125 @@ func (h *Handler) getMigrationHistory(c *gin.Context) { }) } -// rollbackMigration rolls back a specific migration -func (h *Handler) rollbackMigration(c *gin.Context) { +// getMigrationExecutions gets all execution records for a specific migration +func (h *Handler) getMigrationExecutions(c *gin.Context) { migrationID := c.Param("id") - // Get migration from registry - migration := h.executor.GetMigrationByID(migrationID) - if migration == nil { - c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) + // Get executions from state tracker + executions, err := h.executor.GetMigrationExecutions(c.Request.Context(), migrationID) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - // Check if migration is applied - applied, err := h.executor.IsMigrationApplied(c.Request.Context(), migrationID) + // Convert to DTO format + executionDTOs := make([]dto.MigrationExecutionResponse, 0, len(executions)) + for _, exec := range executions { + executionDTOs = append(executionDTOs, dto.MigrationExecutionResponse{ + ID: exec.ID, + MigrationID: exec.MigrationID, + Schema: exec.Schema, + Version: exec.Version, + Connection: exec.Connection, + Backend: exec.Backend, + Status: exec.Status, + Applied: exec.Applied, + AppliedAt: exec.AppliedAt, + CreatedAt: exec.CreatedAt, + UpdatedAt: exec.UpdatedAt, + }) + } + + c.JSON(http.StatusOK, gin.H{ + "migration_id": migrationID, + "executions": executionDTOs, + }) +} + +// getRecentExecutions gets recent execution records across all migrations +func (h *Handler) getRecentExecutions(c *gin.Context) { + limit := 10 // Default limit + if limitParam := c.Query("limit"); limitParam != "" { + if parsedLimit, err := strconv.Atoi(limitParam); err == nil && parsedLimit > 0 { + limit = parsedLimit + } + } + + // Get recent executions from state tracker + executions, err := h.executor.GetRecentExecutions(c.Request.Context(), limit) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - if !applied { - c.JSON(http.StatusBadRequest, gin.H{"error": "migration is not applied"}) + // Convert to DTO format + executionDTOs := make([]dto.MigrationExecutionResponse, 0, len(executions)) + for _, exec := range executions { + executionDTOs = append(executionDTOs, dto.MigrationExecutionResponse{ + ID: exec.ID, + MigrationID: exec.MigrationID, + Schema: exec.Schema, + Version: exec.Version, + Connection: exec.Connection, + Backend: exec.Backend, + Status: exec.Status, + Applied: exec.Applied, + AppliedAt: exec.AppliedAt, + CreatedAt: exec.CreatedAt, + UpdatedAt: exec.UpdatedAt, + }) + } + + c.JSON(http.StatusOK, gin.H{ + "executions": executionDTOs, + }) +} + +// rollbackMigration rolls back a specific migration +func (h *Handler) rollbackMigration(c *gin.Context) { + migrationID := c.Param("id") + + var req dto.RollbackRequest + if err := c.ShouldBindJSON(&req); err != nil { + // If no body provided, use empty schemas (backward compatibility) + req = dto.RollbackRequest{Schemas: []string{}} + } + + // Get migration from registry + migration := h.executor.GetMigrationByID(migrationID) + if migration == nil { + c.JSON(http.StatusNotFound, gin.H{"error": "migration not found"}) return } // Set execution context ctx := h.setExecutionContext(c) - // Execute rollback - result, err := h.executor.Rollback(ctx, migrationID) + // Execute rollback with schemas + result, err := h.executor.Rollback(ctx, migrationID, req.Schemas) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } + // If no migrations were rolled back, return 400 Bad Request + if !result.Success && result.Message == "no migrations to rollback" { + c.JSON(http.StatusBadRequest, gin.H{ + "error": result.Message, + "success": result.Success, + "message": result.Message, + "applied": result.Applied, + "skipped": result.Skipped, + "errors": result.Errors, + }) + return + } + c.JSON(http.StatusOK, gin.H{ "success": result.Success, "message": result.Message, + "applied": result.Applied, + "skipped": result.Skipped, "errors": result.Errors, }) } diff --git a/api/internal/api/http/handler_test.go b/api/internal/api/http/handler_test.go index 5024fc4..bb988bf 100644 --- a/api/internal/api/http/handler_test.go +++ b/api/internal/api/http/handler_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "time" @@ -282,6 +283,85 @@ func (m *mockStateTracker) Initialize(ctx interface{}) error { return m.healthCheckError } +func (m *mockStateTracker) ReindexMigrations(ctx interface{}, registry interface{}) error { + return nil +} + +func (m *mockStateTracker) GetMigrationDetail(ctx interface{}, migrationID string) (*state.MigrationDetail, error) { + // Find migration in listItems + for _, item := range m.listItems { + if item.MigrationID == migrationID { + return &state.MigrationDetail{ + MigrationID: item.MigrationID, + Schema: item.Schema, + Version: item.Version, + Name: item.Name, + Connection: item.Connection, + Backend: item.Backend, + UpSQL: "", + DownSQL: "", + Dependencies: []string{}, + StructuredDependencies: []backends.Dependency{}, + Status: item.LastStatus, + }, nil + } + } + return nil, nil +} + +func (m *mockStateTracker) GetMigrationExecutions(ctx interface{}, migrationID string) ([]*state.MigrationExecution, error) { + // Check if this migration is applied + applied := m.appliedMigrations[migrationID] + if !applied { + return []*state.MigrationExecution{}, nil + } + + // Parse migration ID to extract details: {version}_{name}_{backend}_{connection} + parts := strings.Split(migrationID, "_") + if len(parts) < 4 { + return []*state.MigrationExecution{}, nil + } + + // Extract version, backend, and connection + version := parts[0] + backend := parts[len(parts)-2] + connection := parts[len(parts)-1] + + // Return execution records for both empty schema and "public" schema + // This allows the executor to find the execution regardless of which schema it's looking for + return []*state.MigrationExecution{ + { + ID: 1, + MigrationID: migrationID, + Schema: "", // Empty schema + Version: version, + Connection: connection, + Backend: backend, + Status: "applied", + Applied: true, + AppliedAt: time.Now().Format(time.RFC3339), + CreatedAt: time.Now().Format(time.RFC3339), + UpdatedAt: time.Now().Format(time.RFC3339), + }, + { + ID: 2, + MigrationID: migrationID, + Schema: "public", // Public schema + Version: version, + Connection: connection, + Backend: backend, + Status: "applied", + Applied: true, + AppliedAt: time.Now().Format(time.RFC3339), + CreatedAt: time.Now().Format(time.RFC3339), + UpdatedAt: time.Now().Format(time.RFC3339), + }, + }, nil +} +func (m *mockStateTracker) GetRecentExecutions(ctx interface{}, limit int) ([]*state.MigrationExecution, error) { + return []*state.MigrationExecution{}, nil +} + func setupTestRouter(reg *mockRegistry, tracker *mockStateTracker) (*gin.Engine, *executor.Executor) { gin.SetMode(gin.TestMode) router := gin.New() @@ -915,7 +995,10 @@ func TestHandler_rollbackMigration(t *testing.T) { } _ = reg.Register(migration) migrationID := "public_test_20240101120000_test_migration" - tracker.appliedMigrations[migrationID] = true + // Use the base migration ID format that executor expects: {version}_{name}_{backend}_{connection} + baseMigrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) + // Set applied status using base ID (executor uses base ID when checking via GetMigrationExecutions) + tracker.appliedMigrations[baseMigrationID] = true router, exec := setupTestRouter(reg, tracker) // Set up backend and connection for rollback @@ -998,8 +1081,11 @@ func TestHandler_rollbackMigration_NotApplied(t *testing.T) { DownSQL: "DROP TABLE test;", } _ = reg.Register(migration) + // Use the base migration ID format that executor expects: {version}_{name}_{backend}_{connection} + baseMigrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) migrationID := "public_test_20240101120000_test_migration" - tracker.appliedMigrations[migrationID] = false + // Set applied status to false using base ID (executor uses base ID when checking) + tracker.appliedMigrations[baseMigrationID] = false router, exec := setupTestRouter(reg, tracker) // Set up backend and connection for rollback diff --git a/api/internal/api/protobuf/handler.go b/api/internal/api/protobuf/handler.go index 7a72d6f..2a9d9f5 100644 --- a/api/internal/api/protobuf/handler.go +++ b/api/internal/api/protobuf/handler.go @@ -530,8 +530,8 @@ func (s *Server) RollbackMigration(ctx context.Context, req *RollbackMigrationRe return nil, status.Errorf(codes.FailedPrecondition, "migration is not applied: %s", req.MigrationId) } - // Execute rollback - result, err := s.executor.Rollback(ctx, req.MigrationId) + // Execute rollback with schemas + result, err := s.executor.Rollback(ctx, req.MigrationId, req.Schemas) if err != nil { return nil, status.Errorf(codes.Internal, "failed to rollback migration: %v", err) } diff --git a/api/internal/api/protobuf/migration.pb.go b/api/internal/api/protobuf/migration.pb.go index 32a2bbc..ae30b3f 100644 --- a/api/internal/api/protobuf/migration.pb.go +++ b/api/internal/api/protobuf/migration.pb.go @@ -7,11 +7,12 @@ package protobuf import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" sync "sync" unsafe "unsafe" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" ) const ( @@ -1257,6 +1258,7 @@ func (x *MigrationHistoryItem) GetExecutionContext() string { type RollbackMigrationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` MigrationId string `protobuf:"bytes,1,opt,name=migration_id,json=migrationId,proto3" json:"migration_id,omitempty"` // Required: ID of migration to rollback + Schemas []string `protobuf:"bytes,2,rep,name=schemas,proto3" json:"schemas,omitempty"` // Optional: Array for dynamic schemas unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1298,6 +1300,13 @@ func (x *RollbackMigrationRequest) GetMigrationId() string { return "" } +func (x *RollbackMigrationRequest) GetSchemas() []string { + if x != nil { + return x.Schemas + } + return nil +} + // RollbackResponse represents the result of a rollback operation type RollbackResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1686,9 +1695,10 @@ const file_migration_proto_rawDesc = "" + " \x01(\tR\n" + "executedBy\x12)\n" + "\x10execution_method\x18\v \x01(\tR\x0fexecutionMethod\x12+\n" + - "\x11execution_context\x18\f \x01(\tR\x10executionContext\"=\n" + + "\x11execution_context\x18\f \x01(\tR\x10executionContext\"W\n" + "\x18RollbackMigrationRequest\x12!\n" + - "\fmigration_id\x18\x01 \x01(\tR\vmigrationId\"^\n" + + "\fmigration_id\x18\x01 \x01(\tR\vmigrationId\x12\x18\n" + + "\aschemas\x18\x02 \x03(\tR\aschemas\"^\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12\x18\n" + "\amessage\x18\x02 \x01(\tR\amessage\x12\x16\n" + diff --git a/api/internal/api/protobuf/migration.proto b/api/internal/api/protobuf/migration.proto index 37dfa26..90bd1fd 100644 --- a/api/internal/api/protobuf/migration.proto +++ b/api/internal/api/protobuf/migration.proto @@ -184,6 +184,7 @@ message MigrationHistoryItem { // RollbackMigrationRequest represents a request to rollback a migration message RollbackMigrationRequest { string migration_id = 1; // Required: ID of migration to rollback + repeated string schemas = 2; // Optional: Array for dynamic schemas } // RollbackResponse represents the result of a rollback operation diff --git a/api/internal/api/protobuf/migration_grpc.pb.go b/api/internal/api/protobuf/migration_grpc.pb.go index 1fe0002..816bd6d 100644 --- a/api/internal/api/protobuf/migration_grpc.pb.go +++ b/api/internal/api/protobuf/migration_grpc.pb.go @@ -8,6 +8,7 @@ package protobuf import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/api/internal/backends/postgresql/backend.go b/api/internal/backends/postgresql/backend.go index f51afba..359bebd 100644 --- a/api/internal/backends/postgresql/backend.go +++ b/api/internal/backends/postgresql/backend.go @@ -4,7 +4,10 @@ import ( "context" "database/sql" "fmt" + "os" + "strconv" "strings" + "time" "github.com/toolsascode/bfm/api/internal/backends" @@ -47,6 +50,9 @@ func (b *Backend) Connect(config *backends.ConnectionConfig) error { return fmt.Errorf("failed to open PostgreSQL connection: %w", err) } + // Configure connection pool settings + configureConnectionPool(b.db) + // Test connection if err := b.db.Ping(); err != nil { return fmt.Errorf("failed to ping PostgreSQL: %w", err) @@ -165,3 +171,37 @@ func (b *Backend) HealthCheck(ctx context.Context) error { func quoteIdentifier(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// configureConnectionPool configures the database connection pool with reasonable defaults +// that can be overridden via environment variables +func configureConnectionPool(db *sql.DB) { + // Max open connections per pool (default: 5) + // This limits how many connections each sql.DB instance can open + maxOpenConns := getEnvInt("BFM_DB_MAX_OPEN_CONNS", 5) + db.SetMaxOpenConns(maxOpenConns) + + // Max idle connections per pool (default: 2) + // This keeps some connections ready for reuse + maxIdleConns := getEnvInt("BFM_DB_MAX_IDLE_CONNS", 2) + db.SetMaxIdleConns(maxIdleConns) + + // Connection max lifetime (default: 5 minutes) + // This prevents using stale connections + connMaxLifetime := time.Duration(getEnvInt("BFM_DB_CONN_MAX_LIFETIME_MINUTES", 5)) * time.Minute + db.SetConnMaxLifetime(connMaxLifetime) + + // Connection max idle time (default: 1 minute) + // This closes idle connections after this duration + connMaxIdleTime := time.Duration(getEnvInt("BFM_DB_CONN_MAX_IDLE_TIME_MINUTES", 1)) * time.Minute + db.SetConnMaxIdleTime(connMaxIdleTime) +} + +// getEnvInt gets an integer environment variable or returns the default value +func getEnvInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue +} diff --git a/api/internal/backends/postgresql/validator.go b/api/internal/backends/postgresql/validator.go index 3a8ff7a..de2c40a 100644 --- a/api/internal/backends/postgresql/validator.go +++ b/api/internal/backends/postgresql/validator.go @@ -28,11 +28,25 @@ func NewDependencyValidator(backend *Backend, tracker state.StateTracker, reg re // ValidateDependencies validates all dependencies for a migration func (v *DependencyValidator) ValidateDependencies(ctx context.Context, migration *backends.MigrationScript, schemaName string) []error { + return v.ValidateDependenciesWithExecutionSet(ctx, migration, schemaName, nil) +} + +// ValidateDependenciesWithExecutionSet validates all dependencies for a migration, +// considering migrations in the execution set as satisfied dependencies +func (v *DependencyValidator) ValidateDependenciesWithExecutionSet(ctx context.Context, migration *backends.MigrationScript, schemaName string, executionSet []*backends.MigrationScript) []error { var errors []error + // Build a map of migration IDs in the execution set for quick lookup + executionSetMap := make(map[string]bool) + for _, m := range executionSet { + // Generate migration ID using the same format as executor + migrationID := fmt.Sprintf("%s_%s_%s_%s", m.Version, m.Name, m.Backend, m.Connection) + executionSetMap[migrationID] = true + } + // Validate structured dependencies for _, dep := range migration.StructuredDependencies { - if err := v.validateDependency(ctx, dep, schemaName); err != nil { + if err := v.validateDependencyWithExecutionSet(ctx, dep, schemaName, executionSetMap); err != nil { errors = append(errors, fmt.Errorf("dependency validation failed for %s: %w", v.dependencyString(dep), err)) } } @@ -40,7 +54,7 @@ func (v *DependencyValidator) ValidateDependencies(ctx context.Context, migratio // Validate simple string dependencies (backward compatibility) // For simple dependencies, we only check if the migration exists and is applied for _, depName := range migration.Dependencies { - if err := v.validateSimpleDependency(ctx, depName, schemaName); err != nil { + if err := v.validateSimpleDependencyWithExecutionSet(ctx, depName, schemaName, executionSetMap); err != nil { errors = append(errors, fmt.Errorf("dependency validation failed for '%s': %w", depName, err)) } } @@ -48,8 +62,9 @@ func (v *DependencyValidator) ValidateDependencies(ctx context.Context, migratio return errors } -// validateDependency validates a single structured dependency -func (v *DependencyValidator) validateDependency(ctx context.Context, dep backends.Dependency, currentSchema string) error { +// validateDependencyWithExecutionSet validates a single structured dependency, +// considering migrations in the execution set as satisfied dependencies +func (v *DependencyValidator) validateDependencyWithExecutionSet(ctx context.Context, dep backends.Dependency, currentSchema string, executionSetMap map[string]bool) error { // Validate required schema exists if dep.RequiresSchema != "" { exists, err := v.backend.SchemaExists(ctx, dep.RequiresSchema) @@ -62,45 +77,78 @@ func (v *DependencyValidator) validateDependency(ctx context.Context, dep backen } // Validate required table exists + // Note: If the table is created by a dependency migration in the execution set, + // it won't exist yet, so we skip this check if the dependency is in the execution set if dep.RequiresTable != "" { - schemaToCheck := dep.RequiresSchema - if schemaToCheck == "" { - // Use current schema or default to public - schemaToCheck = currentSchema - if schemaToCheck == "" { - schemaToCheck = "public" + // First check if the dependency migration is in the execution set + targetMigrations, err := v.findMigrationByTarget(dep) + if err == nil { + // Check if any target migration is in the execution set + dependencyInExecutionSet := false + for _, targetMigration := range targetMigrations { + migrationID := fmt.Sprintf("%s_%s_%s_%s", targetMigration.Version, targetMigration.Name, targetMigration.Backend, targetMigration.Connection) + if executionSetMap != nil && executionSetMap[migrationID] { + dependencyInExecutionSet = true + break + } + } + + // If dependency is in execution set, skip table existence check (it will be created) + if !dependencyInExecutionSet { + schemaToCheck := dep.RequiresSchema + if schemaToCheck == "" { + // If RequiresSchema is not specified, use the dependency's Schema if available + // This handles cross-schema dependencies where the table is in the dependency's schema + if dep.Schema != "" { + schemaToCheck = dep.Schema + } else { + // Fall back to current schema or default to public + schemaToCheck = currentSchema + if schemaToCheck == "" { + schemaToCheck = "public" + } + } + } + exists, err := v.backend.TableExists(ctx, schemaToCheck, dep.RequiresTable) + if err != nil { + return fmt.Errorf("failed to check table existence: %w", err) + } + if !exists { + return fmt.Errorf("required table '%s.%s' does not exist", schemaToCheck, dep.RequiresTable) + } } - } - exists, err := v.backend.TableExists(ctx, schemaToCheck, dep.RequiresTable) - if err != nil { - return fmt.Errorf("failed to check table existence: %w", err) - } - if !exists { - return fmt.Errorf("required table '%s.%s' does not exist", schemaToCheck, dep.RequiresTable) } } - // Validate dependency migration is applied - if err := v.validateMigrationApplied(ctx, dep); err != nil { + // Validate dependency migration is applied or in execution set + if err := v.validateMigrationAppliedWithExecutionSet(ctx, dep, executionSetMap); err != nil { return err } return nil } -// validateSimpleDependency validates a simple string dependency -func (v *DependencyValidator) validateSimpleDependency(ctx context.Context, depName string, currentSchema string) error { +// validateSimpleDependencyWithExecutionSet validates a simple string dependency, +// considering migrations in the execution set as satisfied dependencies +func (v *DependencyValidator) validateSimpleDependencyWithExecutionSet(ctx context.Context, depName string, currentSchema string, executionSetMap map[string]bool) error { // Find migrations with this name targetMigrations := v.registry.GetMigrationByName(depName) if len(targetMigrations) == 0 { return fmt.Errorf("dependency migration '%s' not found", depName) } - // Check if at least one of the target migrations is applied - // We check all found migrations and see if any are applied + // Check if at least one of the target migrations is applied or in execution set + // We check all found migrations and see if any are applied or in execution set for _, targetMigration := range targetMigrations { - // Generate migration ID for the target - migrationID := v.getMigrationID(targetMigration, currentSchema) + // Generate migration ID using executor format: {version}_{name}_{backend}_{connection} + migrationID := fmt.Sprintf("%s_%s_%s_%s", targetMigration.Version, targetMigration.Name, targetMigration.Backend, targetMigration.Connection) + + // Check if in execution set + if executionSetMap != nil && executionSetMap[migrationID] { + return nil // Dependency is in execution set, will be executed + } + + // Check if already applied applied, err := v.stateTracker.IsMigrationApplied(ctx, migrationID) if err != nil { return fmt.Errorf("failed to check migration status: %w", err) @@ -113,23 +161,26 @@ func (v *DependencyValidator) validateSimpleDependency(ctx context.Context, depN return fmt.Errorf("dependency migration '%s' is not applied", depName) } -// validateMigrationApplied checks if a dependency migration is applied -func (v *DependencyValidator) validateMigrationApplied(ctx context.Context, dep backends.Dependency) error { +// validateMigrationAppliedWithExecutionSet checks if a dependency migration is applied or in the execution set +func (v *DependencyValidator) validateMigrationAppliedWithExecutionSet(ctx context.Context, dep backends.Dependency, executionSetMap map[string]bool) error { // Find the target migration targetMigrations, err := v.findMigrationByTarget(dep) if err != nil { return fmt.Errorf("dependency target not found: %w", err) } - // Check if at least one target migration is applied + // Check if at least one target migration is applied or in execution set for _, targetMigration := range targetMigrations { - // Determine schema to use for migration ID - schemaToUse := dep.Schema - if schemaToUse == "" { - schemaToUse = targetMigration.Schema + // Generate migration ID using the same format as executor: {version}_{name}_{backend}_{connection} + migrationID := fmt.Sprintf("%s_%s_%s_%s", targetMigration.Version, targetMigration.Name, targetMigration.Backend, targetMigration.Connection) + + // Check if in execution set + if executionSetMap != nil && executionSetMap[migrationID] { + return nil // Dependency is in execution set, will be executed } - migrationID := v.getMigrationID(targetMigration, schemaToUse) + // Check if already applied + // Use the same ID format as executor for state tracker applied, err := v.stateTracker.IsMigrationApplied(ctx, migrationID) if err != nil { return fmt.Errorf("failed to check migration status: %w", err) @@ -182,17 +233,6 @@ func (v *DependencyValidator) findMigrationByTarget(dep backends.Dependency) ([] return candidates, nil } -// getMigrationID generates a migration ID (helper method) -func (v *DependencyValidator) getMigrationID(migration *backends.MigrationScript, schema string) string { - // Migration ID format: {version}_{name} (base format) - baseID := fmt.Sprintf("%s_%s", migration.Version, migration.Name) - if schema != "" { - // For schema-specific checks, prefix with schema - return fmt.Sprintf("%s_%s", schema, baseID) - } - return baseID -} - // dependencyString returns a string representation of a dependency for error messages func (v *DependencyValidator) dependencyString(dep backends.Dependency) string { parts := []string{} diff --git a/api/internal/backends/postgresql/validator_test.go b/api/internal/backends/postgresql/validator_test.go index a56a76c..a960075 100644 --- a/api/internal/backends/postgresql/validator_test.go +++ b/api/internal/backends/postgresql/validator_test.go @@ -56,6 +56,21 @@ func (m *mockStateTrackerForValidator) Initialize(ctx interface{}) error { return nil } +func (m *mockStateTrackerForValidator) ReindexMigrations(ctx interface{}, registry interface{}) error { + return nil +} + +func (m *mockStateTrackerForValidator) GetMigrationDetail(ctx interface{}, migrationID string) (*state.MigrationDetail, error) { + return nil, nil +} + +func (m *mockStateTrackerForValidator) GetMigrationExecutions(ctx interface{}, migrationID string) ([]*state.MigrationExecution, error) { + return nil, nil +} +func (m *mockStateTrackerForValidator) GetRecentExecutions(ctx interface{}, limit int) ([]*state.MigrationExecution, error) { + return nil, nil +} + func TestDependencyValidator_ValidateDependencies(t *testing.T) { backend := &Backend{} // We'll need to use a real backend or mock differently // For now, we'll test the logic without actual database calls @@ -74,7 +89,8 @@ func TestDependencyValidator_ValidateDependencies(t *testing.T) { _ = reg.Register(depMigration) // Mark it as applied - tracker.appliedMigrations["core_20240101120000_base_migration"] = true + // Migration ID format: {version}_{name}_{backend}_{connection} + tracker.appliedMigrations["20240101120000_base_migration_postgresql_core"] = true validator := NewDependencyValidator(backend, tracker, reg) @@ -94,7 +110,7 @@ func TestDependencyValidator_ValidateDependencies(t *testing.T) { }) t.Run("validate simple dependency - not applied", func(t *testing.T) { - tracker.appliedMigrations["core_20240101120000_base_migration"] = false + tracker.appliedMigrations["20240101120000_base_migration_postgresql_core"] = false migration := &backends.MigrationScript{ Version: "20240101120002", @@ -110,7 +126,7 @@ func TestDependencyValidator_ValidateDependencies(t *testing.T) { } // Reset - tracker.appliedMigrations["core_20240101120000_base_migration"] = true + tracker.appliedMigrations["20240101120000_base_migration_postgresql_core"] = true }) t.Run("validate structured dependency - applied", func(t *testing.T) { diff --git a/api/internal/executor/executor.go b/api/internal/executor/executor.go index 5cf08dc..da234e2 100644 --- a/api/internal/executor/executor.go +++ b/api/internal/executor/executor.go @@ -349,6 +349,91 @@ func (e *Executor) resolveDependencies(migrations []*backends.MigrationScript) ( return e.topologicalSort(migrations) } +// expandWithPendingDependencies takes an initial set of migrations and expands it by +// including any pending dependency migrations referenced via structured dependencies. +// It uses the state tracker to ensure already-applied dependencies are not re-executed. +func (e *Executor) expandWithPendingDependencies(ctx context.Context, migrations []*backends.MigrationScript) ([]*backends.MigrationScript, error) { + if len(migrations) == 0 { + return migrations, nil + } + + // Build quick lookup of already selected migrations by ID so we don't duplicate. + selected := make(map[string]*backends.MigrationScript) + for _, m := range migrations { + selected[e.getMigrationID(m)] = m + } + + resolver := registry.NewDependencyResolver(e.registry, e.stateTracker) + + // Collect additional migrations to include. + var toInclude []*backends.MigrationScript + + for _, migration := range migrations { + if len(migration.StructuredDependencies) > 0 { + logger.Debug("Migration %s_%s has %d structured dependency(ies), expanding...", migration.Version, migration.Name, len(migration.StructuredDependencies)) + } + for _, dep := range migration.StructuredDependencies { + logger.Debug("Resolving dependency: connection=%s, schema=%s, target=%s, type=%s", dep.Connection, dep.Schema, dep.Target, dep.TargetType) + // Resolve targets for this dependency (may be cross-connection). + targetMigrations, err := resolver.ResolveDependencyTargets(dep) + if err != nil { + // Surface clear error so callers can see which dependency is invalid. + logger.Errorf("Failed to resolve dependency for %s_%s: connection=%s, schema=%s, target=%s, type=%s: %v", migration.Version, migration.Name, dep.Connection, dep.Schema, dep.Target, dep.TargetType, err) + return nil, fmt.Errorf("failed to resolve dependency for %s_%s: %w", migration.Version, migration.Name, err) + } + + logger.Debug("Found %d target migration(s) for dependency: connection=%s, schema=%s, target=%s", len(targetMigrations), dep.Connection, dep.Schema, dep.Target) + + if len(targetMigrations) == 0 { + logger.Warnf("No target migrations found for dependency: connection=%s, schema=%s, target=%s, type=%s", dep.Connection, dep.Schema, dep.Target, dep.TargetType) + continue + } + + for _, target := range targetMigrations { + targetID := e.getMigrationID(target) + logger.Debug("Checking dependency migration %s (connection=%s, schema=%s, version=%s, name=%s)", targetID, target.Connection, target.Schema, target.Version, target.Name) + + // If it's already in the initial set, nothing to do. + if _, exists := selected[targetID]; exists { + logger.Debug("Dependency migration %s already in execution set, skipping", targetID) + continue + } + + // Only include if the dependency migration is not yet applied. + applied, err := e.stateTracker.IsMigrationApplied(ctx, targetID) + if err != nil { + logger.Errorf("Error checking if migration %s is applied: %v", targetID, err) + return nil, fmt.Errorf("failed to check dependency migration status for %s: %w", targetID, err) + } + logger.Debug("Migration %s applied status: %v", targetID, applied) + if applied { + logger.Debug("Dependency migration %s already applied, skipping", targetID) + continue + } + + logger.Infof("Auto-including pending dependency migration: %s (connection=%s, schema=%s) for %s_%s", targetID, target.Connection, target.Schema, migration.Version, migration.Name) + selected[targetID] = target + toInclude = append(toInclude, target) + } + } + } + + // If nothing extra was found, return original slice. + if len(toInclude) == 0 { + logger.Debug("No pending dependency migrations to auto-include (all dependencies already applied or not found)") + return migrations, nil + } + + logger.Infof("Expanded migration set: %d initial + %d auto-included dependencies = %d total", len(migrations), len(toInclude), len(migrations)+len(toInclude)) + + // Merge initial migrations + newly included ones. + expanded := make([]*backends.MigrationScript, 0, len(migrations)+len(toInclude)) + expanded = append(expanded, migrations...) + expanded = append(expanded, toInclude...) + + return expanded, nil +} + // executeSync executes migrations synchronously func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTarget, connectionName string, schemaName string, dryRun bool) (*ExecuteResult, error) { // Find migrations matching the target @@ -366,52 +451,79 @@ func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTa }, nil } - // Get backend for the connection (needed for validation) + // Log initial migrations found + logger.Debug("Found %d migration(s) matching target (backend=%s, connection=%s, schema=%s)", len(migrations), target.Backend, target.Connection, target.Schema) + for _, m := range migrations { + logger.Debug(" - %s_%s (connection=%s, schema=%s)", m.Version, m.Name, m.Connection, m.Schema) + } + + // If any of the selected migrations declares structured dependencies, expand the set + // with any pending dependency migrations (including cross-connection) so that + // dependencies are executed automatically ahead of dependents. + migrations, err = e.expandWithPendingDependencies(ctx, migrations) + if err != nil { + return nil, fmt.Errorf("failed to expand migrations with dependencies: %w", err) + } + + // Get backend for the target connection (needed for validation) connectionConfig, err := e.getConnectionConfig(connectionName) if err != nil { return nil, fmt.Errorf("failed to get connection config: %w", err) } - backend, ok := e.backends[connectionConfig.Backend] + targetBackend, ok := e.backends[connectionConfig.Backend] if !ok { return nil, fmt.Errorf("backend %s not registered", connectionConfig.Backend) } - // Ensure backend is connected - if err := backend.Connect(connectionConfig); err != nil { + // Sort migrations topologically based on dependencies + // Use DependencyResolver for structured dependencies, fall back to simple topologicalSort for backward compatibility + sortedMigrations, err := e.resolveDependencies(migrations) + if err != nil { + // If dependency resolution fails, fall back to version-based sort and report error + logger.Warnf("Dependency resolution failed: %v, falling back to version-based sort", err) + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].Version < migrations[j].Version + }) + sortedMigrations = migrations + // Add error to result but continue execution + } + + // Ensure target backend is connected (for validation) + if err := targetBackend.Connect(connectionConfig); err != nil { return nil, fmt.Errorf("failed to connect to backend: %w", err) } - defer func() { _ = backend.Close() }() + defer func() { _ = targetBackend.Close() }() - // Validate dependencies before execution (for PostgreSQL backend) + // Validate dependencies after sorting (for PostgreSQL backend) + // Pass the sorted execution set so validator knows which migrations will be executed + // Only validate migrations that belong to the target connection if connectionConfig.Backend == "postgresql" { - pgBackend, ok := backend.(*postgresql.Backend) + pgBackend, ok := targetBackend.(*postgresql.Backend) if ok { validator := postgresql.NewDependencyValidator(pgBackend, e.stateTracker, e.registry) - for _, migration := range migrations { - validationErrors := validator.ValidateDependencies(ctx, migration, schemaName) - if len(validationErrors) > 0 { - var errorMsgs []string - for _, err := range validationErrors { - errorMsgs = append(errorMsgs, err.Error()) + for _, migration := range sortedMigrations { + // Only validate migrations for the target connection + if migration.Connection == connectionName { + validationErrors := validator.ValidateDependenciesWithExecutionSet(ctx, migration, schemaName, sortedMigrations) + if len(validationErrors) > 0 { + var errorMsgs []string + for _, err := range validationErrors { + errorMsgs = append(errorMsgs, err.Error()) + } + return nil, fmt.Errorf("dependency validation failed: %s", strings.Join(errorMsgs, "; ")) } - return nil, fmt.Errorf("dependency validation failed: %s", strings.Join(errorMsgs, "; ")) } } } } - // Sort migrations topologically based on dependencies - // Use DependencyResolver for structured dependencies, fall back to simple topologicalSort for backward compatibility - sortedMigrations, err := e.resolveDependencies(migrations) - if err != nil { - // If dependency resolution fails, fall back to version-based sort and report error - logger.Warnf("Dependency resolution failed: %v, falling back to version-based sort", err) - sort.Slice(migrations, func(i, j int) bool { - return migrations[i].Version < migrations[j].Version - }) - sortedMigrations = migrations - // Add error to result but continue execution + // Log final sorted migration execution order + if len(sortedMigrations) > 0 { + logger.Infof("Final migration execution set (%d total, sorted by dependencies):", len(sortedMigrations)) + for i, m := range sortedMigrations { + logger.Infof(" %d. %s_%s (connection=%s, schema=%s)", i+1, m.Version, m.Name, m.Connection, m.Schema) + } } result := &ExecuteResult{ @@ -427,15 +539,23 @@ func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTa // Process each migration for _, migration := range sortedMigrations { - migrationID := e.getMigrationID(migration) - // Resolve schema name (use provided or from migration) schema := schemaName if schema == "" { schema = migration.Schema } - // Check if already applied (use base migration ID for checking) + // For dynamic schemas (empty Schema in migration), use schema-specific ID + var migrationID string + if migration.Schema == "" && schema != "" { + // Dynamic schema mode: track per schema + migrationID = e.getMigrationIDWithSchema(migration, schema) + } else { + // Fixed schema mode: use base migration ID + migrationID = e.getMigrationID(migration) + } + + // Check if already applied applied, err := e.stateTracker.IsMigrationApplied(ctx, migrationID) if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("failed to check migration status for %s: %v", migrationID, err)) @@ -457,12 +577,13 @@ func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTa executedBy, executionMethod, executionContext := GetExecutionContext(ctx) // Record migration start + // Use migration.Connection (not connectionName) since this migration may be from a different connection record := &state.MigrationRecord{ MigrationID: migrationID, Schema: schema, Table: "", Version: migration.Version, - Connection: connectionName, + Connection: migration.Connection, Backend: migration.Backend, Status: "pending", AppliedAt: time.Now().Format(time.RFC3339), @@ -472,6 +593,40 @@ func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTa ExecutionContext: executionContext, } + // Get backend for this migration's connection (may differ from target connection for cross-connection dependencies) + migrationConnectionConfig, err := e.getConnectionConfig(migration.Connection) + if err != nil { + record.Status = "failed" + record.ErrorMessage = fmt.Sprintf("failed to get connection config for %s: %v", migration.Connection, err) + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", migrationID, err)) + if err := e.stateTracker.RecordMigration(ctx, record); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("failed to record migration %s: %v", migrationID, err)) + } + continue + } + + migrationBackend, ok := e.backends[migrationConnectionConfig.Backend] + if !ok { + record.Status = "failed" + record.ErrorMessage = fmt.Sprintf("backend %s not registered for connection %s", migrationConnectionConfig.Backend, migration.Connection) + result.Errors = append(result.Errors, fmt.Sprintf("%s: backend %s not registered", migrationID, migrationConnectionConfig.Backend)) + if err := e.stateTracker.RecordMigration(ctx, record); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("failed to record migration %s: %v", migrationID, err)) + } + continue + } + + // Connect to the migration's backend (may be different from target backend) + if err := migrationBackend.Connect(migrationConnectionConfig); err != nil { + record.Status = "failed" + record.ErrorMessage = fmt.Sprintf("failed to connect to backend for %s: %v", migration.Connection, err) + result.Errors = append(result.Errors, fmt.Sprintf("%s: failed to connect: %v", migrationID, err)) + if err := e.stateTracker.RecordMigration(ctx, record); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("failed to record migration %s: %v", migrationID, err)) + } + continue + } + // Convert executor.MigrationScript to backends.MigrationScript // Use provided schema instead of migration.Schema for dynamic schemas backendMigration := &backends.MigrationScript{ @@ -484,8 +639,9 @@ func (e *Executor) executeSync(ctx context.Context, target *registry.MigrationTa DownSQL: migration.DownSQL, } - // Execute the migration - err = backend.ExecuteMigration(ctx, backendMigration) + // Execute the migration using its own backend + err = migrationBackend.ExecuteMigration(ctx, backendMigration) + _ = migrationBackend.Close() // Close after execution if err != nil { record.Status = "failed" record.ErrorMessage = err.Error() @@ -512,9 +668,13 @@ func (e *Executor) GetAllMigrations() []*backends.MigrationScript { // GetMigrationByID finds a migration by its ID // Migration ID format: {version}_{name}_{backend}_{connection} +// Also supports schema-specific format: {schema}_{version}_{name}_{backend}_{connection} // Also supports legacy formats for backward compatibility func (e *Executor) GetMigrationByID(migrationID string) *backends.MigrationScript { allMigrations := e.registry.GetAll() + + // First, try to match against base IDs (exact match) + // This handles base IDs even if they have 5+ parts due to underscores in names for _, migration := range allMigrations { // Primary format: {version}_{name}_{backend}_{connection} id := e.getMigrationID(migration) @@ -531,6 +691,43 @@ func (e *Executor) GetMigrationByID(migrationID string) *backends.MigrationScrip if legacyIDWithConnection == migrationID { return migration } + } + + // If no exact match found, try schema-specific matching + // Check if migrationID could be schema-specific (format: {schema}_{version}_{name}_{backend}_{connection}) + parts := strings.Split(migrationID, "_") + if len(parts) >= 5 { + // Extract potential schema and base ID + potentialSchema := parts[0] + baseID := strings.Join(parts[1:], "_") + + for _, migration := range allMigrations { + // Only match schema-specific IDs if the migration has a schema + // Migrations without a schema should not match schema-specific IDs + if migration.Schema != "" && migration.Schema == potentialSchema { + // Check if the base ID matches this migration + id := e.getMigrationID(migration) + if id == baseID { + // Verify the schema-specific ID matches + schemaSpecificID := e.getMigrationIDWithSchema(migration, potentialSchema) + if schemaSpecificID == migrationID { + return migration + } + } + // Also check legacy formats with schema + legacyIDWithConnection := fmt.Sprintf("%s_%s_%s", migration.Connection, migration.Version, migration.Name) + if legacyIDWithConnection == baseID { + legacyIDWithSchema := fmt.Sprintf("%s_%s_%s_%s", migration.Schema, migration.Connection, migration.Version, migration.Name) + if legacyIDWithSchema == migrationID { + return migration + } + } + } + } + } + + // Try legacy format with schema matching (for migrations that have schema) + for _, migration := range allMigrations { // Legacy format with schema: {schema}_{connection}_{version}_{name} if migration.Schema != "" { legacyIDWithSchema := fmt.Sprintf("%s_%s_%s_%s", migration.Schema, migration.Connection, migration.Version, migration.Name) @@ -562,6 +759,21 @@ func (e *Executor) GetMigrationList(ctx context.Context, filters *state.Migratio return e.stateTracker.GetMigrationList(ctx, filters) } +// GetMigrationDetail retrieves detailed information about a single migration from migrations_list +func (e *Executor) GetMigrationDetail(ctx context.Context, migrationID string) (*state.MigrationDetail, error) { + return e.stateTracker.GetMigrationDetail(ctx, migrationID) +} + +// GetMigrationExecutions retrieves all execution records for a migration, ordered by created_at DESC +func (e *Executor) GetMigrationExecutions(ctx context.Context, migrationID string) ([]*state.MigrationExecution, error) { + return e.stateTracker.GetMigrationExecutions(ctx, migrationID) +} + +// GetRecentExecutions retrieves recent execution records across all migrations, ordered by created_at DESC +func (e *Executor) GetRecentExecutions(ctx context.Context, limit int) ([]*state.MigrationExecution, error) { + return e.stateTracker.GetRecentExecutions(ctx, limit) +} + // RegisterScannedMigration registers a scanned migration in migrations_list func (e *Executor) RegisterScannedMigration(ctx context.Context, migrationID, schema, table, version, name, connection, backend string) error { return e.stateTracker.RegisterScannedMigration(ctx, migrationID, schema, table, version, name, connection, backend) @@ -726,8 +938,43 @@ func (e *Executor) ReindexMigrations(ctx context.Context, sfmPath string) (*Rein // Find migrations to remove (in database but not in filesystem) for migrationID := range dbMigrationMap { - if _, exists := fileMigrations[migrationID]; !exists { - // Delete this migration from database + // First, check if the exact migration ID exists in filesystem (for base IDs) + if _, exists := fileMigrations[migrationID]; exists { + // Exact match found, keep this migration + continue + } + + // If not found, check if this is a schema-specific ID (format: {schema}_{version}_{name}_{backend}_{connection}) + // Extract base ID for comparison + parts := strings.Split(migrationID, "_") + var baseID string + var isSchemaSpecific bool + if len(parts) >= 5 { + // Schema-specific ID: extract base ID by removing schema prefix + baseID = strings.Join(parts[1:], "_") + isSchemaSpecific = true + } else { + // Base ID format - if not found in filesystem, it should be removed + baseID = migrationID + isSchemaSpecific = false + } + + // For schema-specific IDs, check if the base migration exists in filesystem + // For base IDs, we already checked above and it doesn't exist, so remove it + if isSchemaSpecific { + // Schema-specific ID: only keep if base migration exists in filesystem + if _, exists := fileMigrations[baseID]; !exists { + // Base migration doesn't exist in filesystem, remove this schema-specific instance + if err := e.stateTracker.DeleteMigration(ctx, migrationID); err != nil { + // Log error but continue + fmt.Printf("Warning: Failed to delete migration %s: %v\n", migrationID, err) + } else { + result.Removed = append(result.Removed, migrationID) + } + } + // If baseID exists in filesystem, keep the schema-specific migration + } else { + // Base ID not found in filesystem, remove it if err := e.stateTracker.DeleteMigration(ctx, migrationID); err != nil { // Log error but continue fmt.Printf("Warning: Failed to delete migration %s: %v\n", migrationID, err) @@ -928,25 +1175,22 @@ func (e *Executor) getMigrationIDWithSchema(migration *backends.MigrationScript, } // Rollback rolls back a migration -func (e *Executor) Rollback(ctx context.Context, migrationID string) (*RollbackResult, error) { +func (e *Executor) Rollback(ctx context.Context, migrationID string, schemas []string) (*RollbackResult, error) { // Get migration from registry migration := e.GetMigrationByID(migrationID) if migration == nil { return nil, fmt.Errorf("migration not found: %s", migrationID) } - // Check if migration is applied - applied, err := e.IsMigrationApplied(ctx, migrationID) - if err != nil { - return nil, fmt.Errorf("failed to check migration status: %w", err) - } - - if !applied { - return &RollbackResult{ - Success: false, - Message: "migration is not applied", - Errors: []string{"migration is not applied"}, - }, nil + // Use provided schemas, or fall back to migration.Schema if empty + // If both are empty, use empty string to process without schema + schemasToUse := schemas + if len(schemasToUse) == 0 { + if migration.Schema != "" { + schemasToUse = []string{migration.Schema} + } else { + schemasToUse = []string{""} + } } // Get connection config @@ -976,80 +1220,134 @@ func (e *Executor) Rollback(ctx context.Context, migrationID string) (*RollbackR }, nil } - // Create a rollback migration script - rollbackMigration := &backends.MigrationScript{ - Schema: migration.Schema, - Version: migration.Version, - Name: migration.Name + "_rollback", - Connection: migration.Connection, - Backend: migration.Backend, - UpSQL: migration.DownSQL, // Use DownSQL as UpSQL for rollback - DownSQL: migration.UpSQL, // Use UpSQL as DownSQL for rollback + result := &RollbackResult{ + Applied: []string{}, + Errors: []string{}, } - // Execute rollback - err = backend.ExecuteMigration(ctx, rollbackMigration) - if err != nil { + // Execute rollback for each schema + for _, schema := range schemasToUse { + // Check if migration is applied for this schema by checking executions table + // This is more accurate than checking migrations_list since executions table tracks per-schema + baseMigrationID := e.getMigrationID(migration) + executions, err := e.stateTracker.GetMigrationExecutions(ctx, baseMigrationID) + if err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("schema %s: failed to check migration status: %v", schema, err)) + continue + } + + // Find execution record matching this schema, version, connection, and backend + var foundExecution *state.MigrationExecution + for _, exec := range executions { + // Check if schema matches (handle comma-separated schemas) + schemaMatches := false + if exec.Schema == schema { + schemaMatches = true + } else if exec.Schema != "" { + // Handle comma-separated schemas + schemas := strings.Split(exec.Schema, ",") + for _, s := range schemas { + if strings.TrimSpace(s) == schema { + schemaMatches = true + break + } + } + } + if schemaMatches && exec.Version == migration.Version && + exec.Connection == migration.Connection && exec.Backend == migration.Backend { + foundExecution = exec + break + } + } + + schemaMigrationID := e.getMigrationIDWithSchema(migration, schema) + if foundExecution == nil || !foundExecution.Applied { + result.Skipped = append(result.Skipped, fmt.Sprintf("%s (not applied)", schemaMigrationID)) + continue + } + + // Create a rollback migration script with the specific schema + rollbackMigration := &backends.MigrationScript{ + Schema: schema, + Version: migration.Version, + Name: migration.Name + "_rollback", + Connection: migration.Connection, + Backend: migration.Backend, + UpSQL: migration.DownSQL, // Use DownSQL as UpSQL for rollback + DownSQL: migration.UpSQL, // Use UpSQL as DownSQL for rollback + } + + // Execute rollback + err = backend.ExecuteMigration(ctx, rollbackMigration) + if err != nil { + // Extract execution context + executedBy, executionMethod, executionContext := GetExecutionContext(ctx) + + // Record failed rollback + record := &state.MigrationRecord{ + MigrationID: schemaMigrationID + "_rollback", + Schema: schema, + Table: "", + Version: migration.Version, + Connection: migration.Connection, + Backend: migration.Backend, + Status: "failed", + AppliedAt: time.Now().Format(time.RFC3339), + ErrorMessage: err.Error(), + ExecutedBy: executedBy, + ExecutionMethod: executionMethod, + ExecutionContext: executionContext, + } + _ = e.stateTracker.RecordMigration(ctx, record) + + result.Errors = append(result.Errors, fmt.Sprintf("schema %s: %v", schema, err)) + continue + } + // Extract execution context executedBy, executionMethod, executionContext := GetExecutionContext(ctx) - // Record failed rollback + // Record successful rollback record := &state.MigrationRecord{ - MigrationID: migrationID + "_rollback", - Schema: migration.Schema, + MigrationID: schemaMigrationID + "_rollback", + Schema: schema, Table: "", Version: migration.Version, Connection: migration.Connection, Backend: migration.Backend, - Status: "failed", + Status: "rolled_back", AppliedAt: time.Now().Format(time.RFC3339), - ErrorMessage: err.Error(), + ErrorMessage: "", ExecutedBy: executedBy, ExecutionMethod: executionMethod, ExecutionContext: executionContext, } - _ = e.stateTracker.RecordMigration(ctx, record) + if err := e.stateTracker.RecordMigration(ctx, record); err != nil { + result.Errors = append(result.Errors, fmt.Sprintf("schema %s: failed to record migration: %v", schema, err)) + } else { + result.Applied = append(result.Applied, schemaMigrationID) + } + } - return &RollbackResult{ - Success: false, - Message: "rollback failed", - Errors: []string{err.Error()}, - }, nil + // Success is true only if there are no errors AND at least one migration was rolled back + result.Success = len(result.Errors) == 0 && len(result.Applied) > 0 + if len(result.Applied) > 0 { + result.Message = fmt.Sprintf("rollback completed successfully for %d schema(s)", len(result.Applied)) + } else if len(result.Errors) > 0 { + result.Message = "rollback failed" + } else { + result.Message = "no migrations to rollback" } - // Extract execution context - executedBy, executionMethod, executionContext := GetExecutionContext(ctx) - - // Remove migration from state (mark as not applied) - // We'll delete the record or mark it as rolled back - // For now, we'll create a rollback record - record := &state.MigrationRecord{ - MigrationID: migrationID + "_rollback", - Schema: migration.Schema, - Table: "", - Version: migration.Version, - Connection: migration.Connection, - Backend: migration.Backend, - Status: "rolled_back", - AppliedAt: time.Now().Format(time.RFC3339), - ErrorMessage: "", - ExecutedBy: executedBy, - ExecutionMethod: executionMethod, - ExecutionContext: executionContext, - } - _ = e.stateTracker.RecordMigration(ctx, record) - - return &RollbackResult{ - Success: true, - Message: "rollback completed successfully", - Errors: []string{}, - }, nil + return result, nil } // RollbackResult represents the result of a rollback operation type RollbackResult struct { Success bool Message string + Applied []string + Skipped []string Errors []string } diff --git a/api/internal/executor/executor_cross_connection_test.go b/api/internal/executor/executor_cross_connection_test.go new file mode 100644 index 0000000..1e0ec9f --- /dev/null +++ b/api/internal/executor/executor_cross_connection_test.go @@ -0,0 +1,158 @@ +package executor + +import ( + "context" + "testing" + + "github.com/toolsascode/bfm/api/internal/backends" + "github.com/toolsascode/bfm/api/internal/registry" + "github.com/toolsascode/bfm/api/internal/state" +) + +// fakeStateTracker implements the minimal methods we need from state.StateTracker +// for testing expandWithPendingDependencies without hitting a real database. +type fakeStateTracker struct { + applied map[string]bool +} + +func (f *fakeStateTracker) IsMigrationApplied(_ interface{}, migrationID string) (bool, error) { + return f.applied[migrationID], nil +} + +// The remaining methods are not used in these tests; provide empty implementations +// to satisfy the interface. + +func (f *fakeStateTracker) Initialize(_ interface{}) error { return nil } +func (f *fakeStateTracker) RecordMigration(_ interface{}, _ *state.MigrationRecord) error { return nil } +func (f *fakeStateTracker) GetMigrationHistory(_ interface{}, _ *state.MigrationFilters) ([]*state.MigrationRecord, error) { + return nil, nil +} +func (f *fakeStateTracker) GetMigrationList(_ interface{}, _ *state.MigrationFilters) ([]*state.MigrationListItem, error) { + return nil, nil +} +func (f *fakeStateTracker) RegisterScannedMigration(_ interface{}, _ string, _ string, _ string, _ string, _ string, _ string, _ string) error { + return nil +} +func (f *fakeStateTracker) UpdateMigrationInfo(_ interface{}, _ string, _ string, _ string, _ string, _ string, _ string, _ string) error { + return nil +} +func (f *fakeStateTracker) GetLastMigrationVersion(_ interface{}, _ string, _ string) (string, error) { + return "", nil +} +func (f *fakeStateTracker) DeleteMigration(_ interface{}, _ string) error { return nil } +func (f *fakeStateTracker) ReindexMigrations(_ interface{}, _ interface{}) error { return nil } +func (f *fakeStateTracker) GetMigrationDetail(_ interface{}, _ string) (*state.MigrationDetail, error) { + return nil, nil +} +func (f *fakeStateTracker) GetMigrationExecutions(_ interface{}, _ string) ([]*state.MigrationExecution, error) { + return nil, nil +} +func (f *fakeStateTracker) GetRecentExecutions(_ interface{}, _ int) ([]*state.MigrationExecution, error) { + return nil, nil +} +func (f *fakeStateTracker) Close() error { return nil } + +// fakeRegistry provides a minimal Registry for the dependency resolver. +type fakeRegistry struct { + migrations []*backends.MigrationScript +} + +func (r *fakeRegistry) GetAll() []*backends.MigrationScript { + return r.migrations +} + +func (r *fakeRegistry) GetMigrationByName(name string) []*backends.MigrationScript { + var out []*backends.MigrationScript + for _, m := range r.migrations { + if m.Name == name { + out = append(out, m) + } + } + return out +} + +// The remaining Registry methods are not needed for this test. +func (r *fakeRegistry) Register(_ *backends.MigrationScript) error { return nil } +func (r *fakeRegistry) FindByTarget(_ *registry.MigrationTarget) ([]*backends.MigrationScript, error) { + return nil, nil +} +func (r *fakeRegistry) GetByConnection(_ string) []*backends.MigrationScript { return nil } +func (r *fakeRegistry) GetByBackend(_ string) []*backends.MigrationScript { return nil } +func (r *fakeRegistry) GetMigrationByVersion(_ string) []*backends.MigrationScript { return nil } +func (r *fakeRegistry) GetMigrationByConnectionAndVersion(_, _ string) []*backends.MigrationScript { + return nil +} + +// TestExpandWithPendingDependenciesCrossConnection verifies that when we execute +// a migration on one connection that depends (via structured dependency) on a +// migration in another connection, the dependency migration is automatically +// included in the execution plan if it is still pending. +func TestExpandWithPendingDependenciesCrossConnection(t *testing.T) { + // Dependency migration in guard connection (e.g. guard_sso_create_sso_tables). + guardDep := &backends.MigrationScript{ + Schema: "guard", + Version: "20251222222820", + Name: "guard_sso_create_sso_tables", + Connection: "guard", + Backend: "postgresql", + } + + // Core migration that depends on the guard migration via structured dependency. + coreMigration := &backends.MigrationScript{ + Schema: "core", + Version: "20251222222821", + Name: "core_organizations_add_sso_fields", + Connection: "core", + Backend: "postgresql", + StructuredDependencies: []backends.Dependency{ + { + Connection: "guard", + Schema: "guard", + Target: "20251222222820", + TargetType: "version", + RequiresTable: "sso_providers", + }, + }, + } + + reg := &fakeRegistry{ + migrations: []*backends.MigrationScript{guardDep, coreMigration}, + } + + // State tracker reports that neither migration has been applied yet. + tracker := &fakeStateTracker{ + applied: map[string]bool{}, + } + + exec := &Executor{ + registry: reg, + stateTracker: tracker, + } + + ctx := context.Background() + + expanded, err := exec.expandWithPendingDependencies(ctx, []*backends.MigrationScript{coreMigration}) + if err != nil { + t.Fatalf("expandWithPendingDependencies returned error: %v", err) + } + + if len(expanded) != 2 { + t.Fatalf("expected 2 migrations after expansion, got %d", len(expanded)) + } + + // Ensure both core and guard migrations are present. + foundGuard := false + foundCore := false + for _, m := range expanded { + if m == guardDep { + foundGuard = true + } + if m == coreMigration { + foundCore = true + } + } + + if !foundGuard || !foundCore { + t.Fatalf("expanded set missing expected migrations: guard=%v core=%v", foundGuard, foundCore) + } +} diff --git a/api/internal/executor/executor_test.go b/api/internal/executor/executor_test.go index 5ca8658..48fd8b2 100644 --- a/api/internal/executor/executor_test.go +++ b/api/internal/executor/executor_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "time" @@ -131,6 +132,7 @@ type mockStateTracker struct { getMigrationHistoryError error registerScannedMigrationError error updateMigrationInfoError error + getMigrationExecutionsError error } func newMockStateTracker() *mockStateTracker { @@ -265,6 +267,75 @@ func (m *mockStateTracker) Initialize(ctx interface{}) error { return m.healthCheckError } +func (m *mockStateTracker) ReindexMigrations(ctx interface{}, registry interface{}) error { + return nil +} + +func (m *mockStateTracker) GetMigrationDetail(ctx interface{}, migrationID string) (*state.MigrationDetail, error) { + // Find migration in listItems + for _, item := range m.listItems { + if item.MigrationID == migrationID { + return &state.MigrationDetail{ + MigrationID: item.MigrationID, + Schema: item.Schema, + Version: item.Version, + Name: item.Name, + Connection: item.Connection, + Backend: item.Backend, + UpSQL: "", + DownSQL: "", + Dependencies: []string{}, + StructuredDependencies: []backends.Dependency{}, + Status: item.LastStatus, + }, nil + } + } + return nil, nil +} + +func (m *mockStateTracker) GetMigrationExecutions(ctx interface{}, migrationID string) ([]*state.MigrationExecution, error) { + if m.getMigrationExecutionsError != nil { + return nil, m.getMigrationExecutionsError + } + // Check if this migration is applied + applied := m.appliedMigrations[migrationID] + if !applied { + return []*state.MigrationExecution{}, nil + } + + // Parse migration ID to extract details: {version}_{name}_{backend}_{connection} + parts := strings.Split(migrationID, "_") + if len(parts) < 4 { + return []*state.MigrationExecution{}, nil + } + + // Extract version, backend, and connection + version := parts[0] + backend := parts[len(parts)-2] + connection := parts[len(parts)-1] + + // Return an execution record with applied=true + // Use empty schema since tests don't specify schemas + return []*state.MigrationExecution{ + { + ID: 1, + MigrationID: migrationID, + Schema: "", // Empty schema for tests + Version: version, + Connection: connection, + Backend: backend, + Status: "applied", + Applied: true, + AppliedAt: time.Now().Format(time.RFC3339), + CreatedAt: time.Now().Format(time.RFC3339), + UpdatedAt: time.Now().Format(time.RFC3339), + }, + }, nil +} +func (m *mockStateTracker) GetRecentExecutions(ctx interface{}, limit int) ([]*state.MigrationExecution, error) { + return []*state.MigrationExecution{}, nil +} + // mockBackend is a mock implementation of backends.Backend type mockBackend struct { name string @@ -1044,7 +1115,7 @@ func TestExecutor_Rollback_MigrationNotFound(t *testing.T) { tracker := newMockStateTracker() exec := NewExecutor(reg, tracker) - _, err := exec.Rollback(context.Background(), "nonexistent") + _, err := exec.Rollback(context.Background(), "nonexistent", []string{}) if err == nil { t.Error("Rollback() expected error for missing migration") } @@ -1068,10 +1139,21 @@ func TestExecutor_Rollback_NotApplied(t *testing.T) { } _ = reg.Register(migration) + connections := map[string]*backends.ConnectionConfig{ + "test": { + Backend: "postgresql", + Host: "localhost", + }, + } + _ = exec.SetConnections(connections) + + backend := newMockBackend("postgresql") + exec.RegisterBackend("postgresql", backend) + migrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) // Migration is not applied - result, err := exec.Rollback(context.Background(), migrationID) + result, err := exec.Rollback(context.Background(), migrationID, []string{}) if err != nil { t.Errorf("Rollback() error = %v", err) } @@ -1081,15 +1163,15 @@ func TestExecutor_Rollback_NotApplied(t *testing.T) { if result.Success { t.Error("Rollback() should not succeed for non-applied migration") } - if result.Message != "migration is not applied" { - t.Errorf("Expected message about migration not applied, got %v", result.Message) + if result.Message != "no migrations to rollback" { + t.Errorf("Expected message about no migrations to rollback, got %v", result.Message) } } func TestExecutor_Rollback_CheckStatusError(t *testing.T) { reg := newMockRegistry() tracker := newMockStateTracker() - tracker.isAppliedError = errors.New("check failed") + tracker.getMigrationExecutionsError = errors.New("check failed") exec := NewExecutor(reg, tracker) migration := &backends.MigrationScript{ @@ -1102,14 +1184,31 @@ func TestExecutor_Rollback_CheckStatusError(t *testing.T) { } _ = reg.Register(migration) + connections := map[string]*backends.ConnectionConfig{ + "test": { + Backend: "postgresql", + Host: "localhost", + }, + } + _ = exec.SetConnections(connections) + + backend := newMockBackend("postgresql") + exec.RegisterBackend("postgresql", backend) + migrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) - _, err := exec.Rollback(context.Background(), migrationID) - if err == nil { + result, err := exec.Rollback(context.Background(), migrationID, []string{}) + if err != nil { + t.Errorf("Rollback() error = %v", err) + } + if result == nil { + t.Fatal("Rollback() returned nil result") + } + if len(result.Errors) == 0 { t.Error("Rollback() expected error when status check fails") } - if err.Error() != "failed to check migration status: check failed" { - t.Errorf("Expected error about status check failure, got %v", err) + if !strings.Contains(result.Errors[0], "check failed") { + t.Errorf("Expected error about status check failure, got %v", result.Errors) } } @@ -1142,7 +1241,7 @@ func TestExecutor_Rollback_NoDownSQL(t *testing.T) { migrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) tracker.appliedMigrations[migrationID] = true - result, err := exec.Rollback(context.Background(), migrationID) + result, err := exec.Rollback(context.Background(), migrationID, []string{}) if err != nil { t.Errorf("Rollback() error = %v", err) } @@ -1184,9 +1283,10 @@ func TestExecutor_Rollback_Successful(t *testing.T) { exec.RegisterBackend("postgresql", backend) migrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) + // Mark migration as applied - this will make GetMigrationExecutions return an execution record tracker.appliedMigrations[migrationID] = true - result, err := exec.Rollback(context.Background(), migrationID) + result, err := exec.Rollback(context.Background(), migrationID, []string{}) if err != nil { t.Errorf("Rollback() error = %v", err) } @@ -1196,7 +1296,7 @@ func TestExecutor_Rollback_Successful(t *testing.T) { if !result.Success { t.Error("Rollback() should succeed for applied migration with down SQL") } - if result.Message != "rollback completed successfully" { + if !strings.Contains(result.Message, "rollback completed successfully") { t.Errorf("Expected success message, got %v", result.Message) } } @@ -1229,9 +1329,10 @@ func TestExecutor_Rollback_ExecutionError(t *testing.T) { exec.RegisterBackend("postgresql", backend) migrationID := fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) + // Mark migration as applied - this will make GetMigrationExecutions return an execution record tracker.appliedMigrations[migrationID] = true - result, err := exec.Rollback(context.Background(), migrationID) + result, err := exec.Rollback(context.Background(), migrationID, []string{}) if err != nil { t.Errorf("Rollback() error = %v", err) } diff --git a/api/internal/executor/loader.go b/api/internal/executor/loader.go index 2462a16..1c33617 100644 --- a/api/internal/executor/loader.go +++ b/api/internal/executor/loader.go @@ -418,61 +418,100 @@ func extractStructuredDependenciesFromGoFile(goFilePath string) []backends.Depen // Look for StructuredDependencies field // Pattern: StructuredDependencies: []migrations.Dependency{ ... } or []backends.Dependency{ ... } - // This is a simplified parser - for full parsing, we'd need a proper Go parser - // For now, we'll look for the pattern and extract basic information - depsRegex := regexp.MustCompile(`StructuredDependencies:\s*\[\](?:migrations|backends)\.Dependency\s*\{([^}]*)\}`) - matches := depsRegex.FindStringSubmatch(content) - if len(matches) < 2 { + // Find the opening brace after StructuredDependencies, then match balanced braces + depsRegex := regexp.MustCompile(`StructuredDependencies:\s*\[\](?:migrations|backends)\.Dependency\s*\{`) + idx := depsRegex.FindStringIndex(content) + if idx == nil { return nil } - depsStr := strings.TrimSpace(matches[1]) + // Find the matching closing brace by counting braces + start := idx[1] // Position after opening brace + braceCount := 1 + end := start + for end < len(content) && braceCount > 0 { + switch content[end] { + case '{': + braceCount++ + case '}': + braceCount-- + } + end++ + } + + if braceCount != 0 { + // Unbalanced braces, return empty + return []backends.Dependency{} + } + + // Extract the content between braces (excluding the closing brace) + depsStr := strings.TrimSpace(content[start : end-1]) if depsStr == "" { return []backends.Dependency{} } // Parse individual dependency structs - // Pattern: { Connection: "core", Schema: "core", Target: "name", TargetType: "name", RequiresTable: "table", RequiresSchema: "schema" } + // Match structs with balanced braces: { ... } var dependencies []backends.Dependency + pos := 0 + for pos < len(depsStr) { + // Find next opening brace + openIdx := strings.Index(depsStr[pos:], "{") + if openIdx == -1 { + break + } + openIdx += pos + + // Find matching closing brace + braceCount := 1 + closeIdx := openIdx + 1 + for closeIdx < len(depsStr) && braceCount > 0 { + switch depsStr[closeIdx] { + case '{': + braceCount++ + case '}': + braceCount-- + } + closeIdx++ + } - // Split by struct boundaries (simplified - assumes single-line structs or properly formatted) - // This is a basic parser - for production, consider using go/parser - depStructRegex := regexp.MustCompile(`\{[^}]*\}`) - structMatches := depStructRegex.FindAllString(depsStr, -1) + if braceCount == 0 { + structStr := depsStr[openIdx:closeIdx] + dep := backends.Dependency{} + // Extract Connection + if match := regexp.MustCompile(`Connection:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.Connection = strings.TrimSpace(match[1]) + } + // Extract Schema + if match := regexp.MustCompile(`Schema:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.Schema = strings.TrimSpace(match[1]) + } + // Extract Target + if match := regexp.MustCompile(`Target:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.Target = strings.TrimSpace(match[1]) + } + // Extract TargetType + if match := regexp.MustCompile(`TargetType:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.TargetType = strings.TrimSpace(match[1]) + } else { + dep.TargetType = "name" // Default + } + // Extract RequiresTable + if match := regexp.MustCompile(`RequiresTable:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.RequiresTable = strings.TrimSpace(match[1]) + } + // Extract RequiresSchema + if match := regexp.MustCompile(`RequiresSchema:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { + dep.RequiresSchema = strings.TrimSpace(match[1]) + } - for _, structStr := range structMatches { - dep := backends.Dependency{} - // Extract Connection - if match := regexp.MustCompile(`Connection:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.Connection = match[1] - } - // Extract Schema - if match := regexp.MustCompile(`Schema:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.Schema = match[1] - } - // Extract Target - if match := regexp.MustCompile(`Target:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.Target = match[1] - } - // Extract TargetType - if match := regexp.MustCompile(`TargetType:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.TargetType = match[1] - } else { - dep.TargetType = "name" // Default - } - // Extract RequiresTable - if match := regexp.MustCompile(`RequiresTable:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.RequiresTable = match[1] - } - // Extract RequiresSchema - if match := regexp.MustCompile(`RequiresSchema:\s*["` + "`" + `]([^"` + "`" + `]+)["` + "`" + `]`).FindStringSubmatch(structStr); len(match) >= 2 { - dep.RequiresSchema = match[1] + // Only add if Target is set (required field) + if dep.Target != "" { + dependencies = append(dependencies, dep) + } } - // Only add if Target is set (required field) - if dep.Target != "" { - dependencies = append(dependencies, dep) - } + pos = closeIdx } return dependencies diff --git a/api/internal/logger/logger.go b/api/internal/logger/logger.go index 859f466..c2ea33b 100644 --- a/api/internal/logger/logger.go +++ b/api/internal/logger/logger.go @@ -2,100 +2,149 @@ package logger import ( "fmt" - "log" "os" - "time" -) - -// LogLevel represents the logging level -type LogLevel int + "strings" -const ( - DEBUG LogLevel = iota - INFO - WARN - ERROR - FATAL + "github.com/sirupsen/logrus" ) var ( - currentLevel LogLevel = INFO - logger *log.Logger + log *logrus.Logger ) func init() { - logger = log.New(os.Stdout, "", 0) + log = logrus.New() + log.SetOutput(os.Stdout) // Set log level from environment levelStr := os.Getenv("BFM_LOG_LEVEL") - switch levelStr { - case "DEBUG", "debug": - currentLevel = DEBUG - case "INFO", "info": - currentLevel = INFO - case "WARN", "warn", "WARNING", "warning": - currentLevel = WARN - case "ERROR", "error": - currentLevel = ERROR - case "FATAL", "fatal": - currentLevel = FATAL + switch strings.ToUpper(levelStr) { + case "DEBUG": + log.SetLevel(logrus.DebugLevel) + case "INFO": + log.SetLevel(logrus.InfoLevel) + case "WARN", "WARNING": + log.SetLevel(logrus.WarnLevel) + case "ERROR": + log.SetLevel(logrus.ErrorLevel) + case "FATAL": + log.SetLevel(logrus.FatalLevel) default: - currentLevel = INFO + log.SetLevel(logrus.InfoLevel) + } + + // Set log format from environment (default to JSON) + formatStr := os.Getenv("BFM_LOG_FORMAT") + switch strings.ToLower(formatStr) { + case "plaintext", "plain", "text": + log.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + }) + case "json": + log.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02T15:04:05.000Z07:00", + }) + default: + // Default to JSON if not specified + log.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02T15:04:05.000Z07:00", + }) } } +// LogLevel represents the logging level (kept for backward compatibility) +type LogLevel int + +const ( + DEBUG LogLevel = iota + INFO + WARN + ERROR + FATAL +) + +// LogFormat represents the logging format (kept for backward compatibility) +type LogFormat int + +const ( + FormatJSON LogFormat = iota + FormatPlaintext +) + // SetLevel sets the logging level func SetLevel(level LogLevel) { - currentLevel = level -} - -// shouldLog checks if a message at the given level should be logged -func shouldLog(level LogLevel) bool { - return level >= currentLevel + switch level { + case DEBUG: + log.SetLevel(logrus.DebugLevel) + case INFO: + log.SetLevel(logrus.InfoLevel) + case WARN: + log.SetLevel(logrus.WarnLevel) + case ERROR: + log.SetLevel(logrus.ErrorLevel) + case FATAL: + log.SetLevel(logrus.FatalLevel) + } } -// formatMessage formats a log message with timestamp and level -func formatMessage(level string, format string, args ...interface{}) string { - timestamp := time.Now().Format("2006-01-02 15:04:05") - message := format - if len(args) > 0 { - message = fmt.Sprintf(format, args...) +// SetFormat sets the logging format +func SetFormat(format LogFormat) { + switch format { + case FormatJSON: + log.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02T15:04:05.000Z07:00", + }) + case FormatPlaintext: + log.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + }) } - return fmt.Sprintf("[%s] [%s] %s", timestamp, level, message) } // Debug logs a debug message func Debug(format string, args ...interface{}) { - if shouldLog(DEBUG) { - logger.Println(formatMessage("DEBUG", format, args...)) + if len(args) > 0 { + log.Debug(fmt.Sprintf(format, args...)) + } else { + log.Debug(format) } } // Info logs an info message func Info(format string, args ...interface{}) { - if shouldLog(INFO) { - logger.Println(formatMessage("INFO", format, args...)) + if len(args) > 0 { + log.Info(fmt.Sprintf(format, args...)) + } else { + log.Info(format) } } // Warn logs a warning message func Warn(format string, args ...interface{}) { - if shouldLog(WARN) { - logger.Println(formatMessage("WARN", format, args...)) + if len(args) > 0 { + log.Warn(fmt.Sprintf(format, args...)) + } else { + log.Warn(format) } } // Error logs an error message func Error(format string, args ...interface{}) { - if shouldLog(ERROR) { - logger.Println(formatMessage("ERROR", format, args...)) + if len(args) > 0 { + log.Error(fmt.Sprintf(format, args...)) + } else { + log.Error(format) } } // Fatal logs a fatal message and exits func Fatal(format string, args ...interface{}) { - if shouldLog(FATAL) { - logger.Fatalln(formatMessage("FATAL", format, args...)) + if len(args) > 0 { + log.Fatal(fmt.Sprintf(format, args...)) + } else { + log.Fatal(format) } } diff --git a/api/internal/queue/kafka/queue.go b/api/internal/queue/kafka/queue.go index b577dcf..57d7945 100644 --- a/api/internal/queue/kafka/queue.go +++ b/api/internal/queue/kafka/queue.go @@ -3,6 +3,7 @@ package kafka import ( "context" "fmt" + "github.com/toolsascode/bfm/api/internal/queue" ) diff --git a/api/internal/queue/pulsar/queue.go b/api/internal/queue/pulsar/queue.go index b16b782..cf8ea67 100644 --- a/api/internal/queue/pulsar/queue.go +++ b/api/internal/queue/pulsar/queue.go @@ -3,6 +3,7 @@ package pulsar import ( "context" "fmt" + "github.com/toolsascode/bfm/api/internal/queue" ) diff --git a/api/internal/queuefactory/factory.go b/api/internal/queuefactory/factory.go index 3f5bfba..39e9523 100644 --- a/api/internal/queuefactory/factory.go +++ b/api/internal/queuefactory/factory.go @@ -2,10 +2,11 @@ package queuefactory import ( "fmt" + "strings" + "github.com/toolsascode/bfm/api/internal/queue" "github.com/toolsascode/bfm/api/internal/queue/kafka" "github.com/toolsascode/bfm/api/internal/queue/pulsar" - "strings" ) // QueueConfig holds configuration for creating a queue diff --git a/api/internal/registry/dependency_resolver.go b/api/internal/registry/dependency_resolver.go index 49956f7..ba389bd 100644 --- a/api/internal/registry/dependency_resolver.go +++ b/api/internal/registry/dependency_resolver.go @@ -208,6 +208,12 @@ func NewDependencyResolver(reg Registry, tracker state.StateTracker) *Dependency } } +// ResolveDependencyTargets is a helper that exposes findDependencyTarget for callers +// that need to expand execution sets with dependency migrations. +func (r *DependencyResolver) ResolveDependencyTargets(dep backends.Dependency) ([]*backends.MigrationScript, error) { + return r.findDependencyTarget(dep) +} + // findDependencyTarget finds migration(s) matching a dependency specification func (r *DependencyResolver) findDependencyTarget(dep backends.Dependency) ([]*backends.MigrationScript, error) { var candidates []*backends.MigrationScript @@ -274,8 +280,8 @@ func (r *DependencyResolver) buildDependencyGraph(migrations []*backends.Migrati // Add edges for each target migration that's in our set for _, targetMigration := range targetMigrations { targetID := getMigrationID(targetMigration) - // Only add edge if target is in our current migration set - if _, exists := graph.nodes[targetID]; exists { + // Only add edge if target is in our current migration set and not a self-loop + if _, exists := graph.nodes[targetID]; exists && migrationID != targetID { graph.AddEdge(migrationID, targetID) } } @@ -293,7 +299,8 @@ func (r *DependencyResolver) buildDependencyGraph(migrations []*backends.Migrati // Add edges for each target migration that's in our set for _, targetMigration := range targetMigrations { targetID := getMigrationID(targetMigration) - if _, exists := graph.nodes[targetID]; exists { + // Only add edge if target is in our current migration set and not a self-loop + if _, exists := graph.nodes[targetID]; exists && migrationID != targetID { graph.AddEdge(migrationID, targetID) } } diff --git a/api/internal/registry/dependency_resolver_test.go b/api/internal/registry/dependency_resolver_test.go index e533167..35bc8ee 100644 --- a/api/internal/registry/dependency_resolver_test.go +++ b/api/internal/registry/dependency_resolver_test.go @@ -54,6 +54,21 @@ func (m *mockStateTracker) Initialize(ctx interface{}) error { return nil } +func (m *mockStateTracker) ReindexMigrations(ctx interface{}, registry interface{}) error { + return nil +} + +func (m *mockStateTracker) GetMigrationDetail(ctx interface{}, migrationID string) (*state.MigrationDetail, error) { + return nil, nil +} + +func (m *mockStateTracker) GetMigrationExecutions(ctx interface{}, migrationID string) ([]*state.MigrationExecution, error) { + return nil, nil +} +func (m *mockStateTracker) GetRecentExecutions(ctx interface{}, limit int) ([]*state.MigrationExecution, error) { + return nil, nil +} + func TestDependencyGraph_AddNode(t *testing.T) { graph := NewDependencyGraph() migration := &backends.MigrationScript{ diff --git a/api/internal/state/interface.go b/api/internal/state/interface.go index 808df7a..6de6059 100644 --- a/api/internal/state/interface.go +++ b/api/internal/state/interface.go @@ -1,5 +1,7 @@ package state +import "github.com/toolsascode/bfm/api/internal/backends" + // MigrationRecord represents a migration execution record in state tracking (moved here to avoid import cycle) type MigrationRecord struct { ID string @@ -60,6 +62,49 @@ type StateTracker interface { // Initialize sets up the state tracking tables Initialize(ctx interface{}) error + + // ReindexMigrations reloads the BfM migration list and updates the database state + // This should be called asynchronously in the background + ReindexMigrations(ctx interface{}, registry interface{}) error + + // GetMigrationDetail retrieves detailed information about a single migration from migrations_list + GetMigrationDetail(ctx interface{}, migrationID string) (*MigrationDetail, error) + + // GetMigrationExecutions retrieves all execution records for a migration, ordered by created_at DESC + GetMigrationExecutions(ctx interface{}, migrationID string) ([]*MigrationExecution, error) + + // GetRecentExecutions retrieves recent execution records across all migrations, ordered by created_at DESC + GetRecentExecutions(ctx interface{}, limit int) ([]*MigrationExecution, error) +} + +// MigrationDetail represents detailed information about a migration from migrations_list +type MigrationDetail struct { + MigrationID string + Schema string + Version string + Name string + Connection string + Backend string + UpSQL string + DownSQL string + Dependencies []string + StructuredDependencies []backends.Dependency + Status string +} + +// MigrationExecution represents an execution record in migrations_executions +type MigrationExecution struct { + ID int + MigrationID string + Schema string + Version string + Connection string + Backend string + Status string + Applied bool + AppliedAt string + CreatedAt string + UpdatedAt string } // MigrationFilters specifies filters for querying migrations diff --git a/api/internal/state/postgresql/tracker.go b/api/internal/state/postgresql/tracker.go index 012e0e1..c76af2a 100644 --- a/api/internal/state/postgresql/tracker.go +++ b/api/internal/state/postgresql/tracker.go @@ -3,13 +3,16 @@ package postgresql import ( "context" "database/sql" + "encoding/json" "fmt" + "os" + "strconv" "strings" "time" + "github.com/lib/pq" + "github.com/toolsascode/bfm/api/internal/backends" "github.com/toolsascode/bfm/api/internal/state" - - _ "github.com/lib/pq" ) // Tracker implements StateTracker for PostgreSQL @@ -25,6 +28,9 @@ func NewTracker(connStr string, schema string) (*Tracker, error) { return nil, fmt.Errorf("failed to open database: %w", err) } + // Configure connection pool settings + configureConnectionPool(db) + tracker := &Tracker{ db: db, schema: schema, @@ -58,20 +64,19 @@ func (t *Tracker) Initialize(ctx interface{}) error { createListTableSQL := fmt.Sprintf(` CREATE TABLE IF NOT EXISTS %s ( - id SERIAL PRIMARY KEY, - migration_id VARCHAR(255) NOT NULL UNIQUE, + migration_id VARCHAR(255) PRIMARY KEY, schema VARCHAR(255) NOT NULL, - table_name VARCHAR(255) NOT NULL, version VARCHAR(50) NOT NULL, name VARCHAR(255) NOT NULL, connection VARCHAR(255) NOT NULL, backend VARCHAR(50) NOT NULL, - last_status VARCHAR(20) NOT NULL DEFAULT 'pending', - last_applied_at TIMESTAMP, - last_error_message TEXT, - last_history_id INTEGER, - first_seen_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - last_updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + up_sql VARCHAR(255), + down_sql VARCHAR(255), + dependencies TEXT[], + structured_dependencies JSONB, + status VARCHAR(50) NOT NULL DEFAULT 'pending', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) `, listTableName) @@ -80,13 +85,15 @@ func (t *Tracker) Initialize(ctx interface{}) error { } // Create indexes for migrations_list + // Note: migration_id is PRIMARY KEY so already indexed, but explicit index is kept for consistency + // All tables with migration_id column must have an index on it for performance and foreign key constraints indexSQL1 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_list_migration_id ON %s (migration_id)", listTableName) _, _ = t.db.ExecContext(ctxVal, indexSQL1) indexSQL2 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_list_connection_backend ON %s (connection, backend)", listTableName) _, _ = t.db.ExecContext(ctxVal, indexSQL2) - indexSQL3 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_list_schema_table ON %s (schema, table_name)", listTableName) + indexSQL3 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_list_status ON %s (status)", listTableName) _, _ = t.db.ExecContext(ctxVal, indexSQL3) // Create migrations_history table @@ -100,7 +107,6 @@ func (t *Tracker) Initialize(ctx interface{}) error { id SERIAL PRIMARY KEY, migration_id VARCHAR(255) NOT NULL, schema VARCHAR(255) NOT NULL, - table_name VARCHAR(255) NOT NULL, version VARCHAR(50) NOT NULL, connection VARCHAR(255) NOT NULL, backend VARCHAR(50) NOT NULL, @@ -120,6 +126,7 @@ func (t *Tracker) Initialize(ctx interface{}) error { } // Create indexes for migrations_history + // Index on migration_id is required for foreign key performance and to avoid using migration names that don't exist in migrations_list indexSQL4 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_history_migration_id ON %s (migration_id)", historyTableName) _, _ = t.db.ExecContext(ctxVal, indexSQL4) @@ -129,8 +136,85 @@ func (t *Tracker) Initialize(ctx interface{}) error { indexSQL6 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_history_status ON %s (status)", historyTableName) _, _ = t.db.ExecContext(ctxVal, indexSQL6) - // Migrate existing data from bfm_migrations if it exists - if err := t.migrateExistingData(ctxVal, listTableName, historyTableName); err != nil { + // Create migrations_executions table + executionsTableName := "migrations_executions" + if t.schema != "" && t.schema != "public" { + executionsTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_executions")) + } + + createExecutionsTableSQL := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + migration_id VARCHAR(255) NOT NULL, + schema VARCHAR(255) NOT NULL, + version VARCHAR(50) NOT NULL, + connection VARCHAR(255) NOT NULL, + backend VARCHAR(50) NOT NULL, + status VARCHAR(20) NOT NULL, + applied BOOLEAN NOT NULL DEFAULT FALSE, + applied_at TIMESTAMP, + actions TEXT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (migration_id) REFERENCES %s(migration_id) ON DELETE CASCADE, + UNIQUE (migration_id, schema, version, connection, backend) + ) + `, executionsTableName, listTableName) + + if _, err := t.db.ExecContext(ctxVal, createExecutionsTableSQL); err != nil { + return fmt.Errorf("failed to create migrations_executions table: %w", err) + } + + // Create indexes for migrations_executions + // Index on migration_id is required for foreign key performance and to avoid using migration names that don't exist in migrations_list + indexSQL7 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_executions_migration_id ON %s (migration_id)", executionsTableName) + _, _ = t.db.ExecContext(ctxVal, indexSQL7) + + indexSQL8 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_executions_status ON %s (status)", executionsTableName) + _, _ = t.db.ExecContext(ctxVal, indexSQL8) + + indexSQL9 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_executions_created_at ON %s (created_at DESC)", executionsTableName) + _, _ = t.db.ExecContext(ctxVal, indexSQL9) + + // Create migrations_dependencies table + dependenciesTableName := "migrations_dependencies" + if t.schema != "" && t.schema != "public" { + dependenciesTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_dependencies")) + } + + createDependenciesTableSQL := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + id SERIAL PRIMARY KEY, + migration_id VARCHAR(255) NOT NULL, + dependency_id VARCHAR(255) NOT NULL, + connection VARCHAR(255) NOT NULL, + schema TEXT[] NOT NULL, + target VARCHAR(255) NOT NULL, + target_type VARCHAR(20) NOT NULL DEFAULT 'name', + requires_table VARCHAR(255), + requires_schema VARCHAR(255), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (migration_id) REFERENCES %s(migration_id) ON DELETE CASCADE, + FOREIGN KEY (dependency_id) REFERENCES %s(migration_id) ON DELETE CASCADE + ) + `, dependenciesTableName, listTableName, listTableName) + + if _, err := t.db.ExecContext(ctxVal, createDependenciesTableSQL); err != nil { + return fmt.Errorf("failed to create migrations_dependencies table: %w", err) + } + + // Create indexes for migrations_dependencies + // Index on migration_id is required for foreign key performance and to avoid using migration names that don't exist in migrations_list + indexSQL10 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_dependencies_migration_id ON %s (migration_id)", dependenciesTableName) + _, _ = t.db.ExecContext(ctxVal, indexSQL10) + + indexSQL11 := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_migrations_dependencies_dependency_id ON %s (dependency_id)", dependenciesTableName) + _, _ = t.db.ExecContext(ctxVal, indexSQL11) + + // Migrate existing data from old tables if they exist + executionsTableNameForMigration := executionsTableName + dependenciesTableNameForMigration := dependenciesTableName + if err := t.migrateExistingData(ctxVal, listTableName, historyTableName, executionsTableNameForMigration, dependenciesTableNameForMigration); err != nil { // Log warning but don't fail initialization fmt.Printf("Warning: Failed to migrate existing data: %v\n", err) } @@ -144,9 +228,11 @@ func (t *Tracker) RecordMigration(ctx interface{}, migration *state.MigrationRec listTableName := "migrations_list" historyTableName := "migrations_history" + executionsTableName := "migrations_executions" if t.schema != "" && t.schema != "public" { listTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_list")) historyTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_history")) + executionsTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_executions")) } appliedAt := time.Now() @@ -156,22 +242,45 @@ func (t *Tracker) RecordMigration(ctx interface{}, migration *state.MigrationRec } } - // Extract base migration_id (remove _rollback suffix if present) - baseMigrationID := migration.MigrationID - isRollback := strings.Contains(migration.MigrationID, "_rollback") + // Extract base migration_id + // Migration ID can be in formats: + // - Base: {version}_{name}_{backend}_{connection} + // - Schema-specific: {schema}_{version}_{name}_{backend}_{connection} + // - With rollback suffix: ..._rollback + // migrations_list should always use the base ID (without schema prefix) + migrationID := migration.MigrationID + isRollback := strings.Contains(migrationID, "_rollback") if isRollback { - baseMigrationID = strings.TrimSuffix(migration.MigrationID, "_rollback") + migrationID = strings.TrimSuffix(migrationID, "_rollback") } - // Insert into migrations_history - insertHistorySQL := fmt.Sprintf(` - INSERT INTO %s (migration_id, schema, table_name, version, connection, backend, - status, error_message, executed_by, execution_method, execution_context, applied_at, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) - RETURNING id - `, historyTableName) + // Remove schema prefix if present to get base migration_id + // Schema-specific format: {schema}_{version}_{name}_{backend}_{connection} + // Base format: {version}_{name}_{backend}_{connection} + // Version is typically 14 digits (YYYYMMDDHHMMSS), so we check if first part is a version + baseMigrationID := migrationID + parts := strings.Split(migrationID, "_") + if len(parts) >= 5 { + // Check if first part looks like a schema name (not a version number) + // Versions are 14 digits, so if first part is not all digits, it might be a schema prefix + firstPart := parts[0] + isVersion := len(firstPart) >= 10 && len(firstPart) <= 20 + if isVersion { + allDigits := true + for _, r := range firstPart { + if r < '0' || r > '9' { + allDigits = false + break + } + } + isVersion = allDigits + } + // If first part is not a version, it's likely a schema prefix - remove it + if !isVersion { + baseMigrationID = strings.Join(parts[1:], "_") + } + } - var historyID int executedBy := migration.ExecutedBy if executedBy == "" { executedBy = "system" @@ -181,63 +290,100 @@ func (t *Tracker) RecordMigration(ctx interface{}, migration *state.MigrationRec executionMethod = "api" } - err := t.db.QueryRowContext(ctxVal, insertHistorySQL, - baseMigrationID, migration.Schema, migration.Table, migration.Version, - migration.Connection, migration.Backend, migration.Status, migration.ErrorMessage, - executedBy, executionMethod, migration.ExecutionContext, appliedAt, appliedAt).Scan(&historyID) - if err != nil { - return fmt.Errorf("failed to insert into migrations_history: %w", err) + // Convert schema to array + schemas := []string{} + if migration.Schema != "" { + schemas = []string{migration.Schema} } - // Update or insert into migrations_list (only for base migration_id, not rollbacks) - if !isRollback { - // Extract name from migration_id - name := baseMigrationID - parts := strings.Split(baseMigrationID, "_") - if len(parts) >= 4 { - name = strings.Join(parts[3:], "_") - } + // Map status values + status := migration.Status + if status == "success" { + status = "applied" + } - upsertListSQL := fmt.Sprintf(` - INSERT INTO %s (migration_id, "schema", table_name, version, name, connection, backend, - last_status, last_applied_at, last_error_message, last_history_id, first_seen_at, last_updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) - ON CONFLICT (migration_id) DO UPDATE SET - "schema" = EXCLUDED."schema", - table_name = EXCLUDED.table_name, - last_status = EXCLUDED.last_status, - last_applied_at = EXCLUDED.last_applied_at, - last_error_message = EXCLUDED.last_error_message, - last_history_id = EXCLUDED.last_history_id, - last_updated_at = CURRENT_TIMESTAMP - `, listTableName) + // Only update status in migrations_list if migration exists (populated from sfm folder) + // migrations_list should only be populated via ReindexMigrations() or RegisterScannedMigration() + // This UPDATE will affect 0 rows if migration doesn't exist, which is acceptable + // The foreign key constraint will prevent history insert if migration doesn't exist in list + updateListSQL := fmt.Sprintf(` + UPDATE %s + SET status = $1, + updated_at = CURRENT_TIMESTAMP + WHERE migration_id = $2 + `, listTableName) + + listStatus := status + if isRollback { + listStatus = "rolled_back" + } + if status == "success" { + listStatus = "applied" + } + + _, err := t.db.ExecContext(ctxVal, updateListSQL, listStatus, baseMigrationID) + // Don't error if 0 rows affected - migration might not be in list yet (should be indexed from sfm first) + + // Skip insertion if no schemas specified + if len(schemas) == 0 { + return nil + } + + // Insert one record per schema into migrations_history + insertHistorySQL := fmt.Sprintf(` + INSERT INTO %s (migration_id, schema, version, connection, backend, + status, error_message, executed_by, execution_method, execution_context, applied_at, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + RETURNING id + `, historyTableName) + + // Insert one record per schema into migrations_executions + applied := status == "applied" + var appliedAtPtr *time.Time + if applied { + appliedAtPtr = &appliedAt + } + + execStatus := "pending" + if applied { + execStatus = "applied" + } else if status == "failed" { + execStatus = "failed" + } - _, err = t.db.ExecContext(ctxVal, upsertListSQL, - baseMigrationID, migration.Schema, migration.Table, migration.Version, name, - migration.Connection, migration.Backend, migration.Status, appliedAt, - migration.ErrorMessage, historyID, appliedAt, time.Now()) + insertExecutionSQL := fmt.Sprintf(` + INSERT INTO %s (migration_id, schema, version, connection, backend, status, applied, applied_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (migration_id, schema, version, connection, backend) DO UPDATE SET + status = EXCLUDED.status, + applied = EXCLUDED.applied, + applied_at = EXCLUDED.applied_at, + updated_at = CURRENT_TIMESTAMP + `, executionsTableName) + + // Create one record per schema + for _, schema := range schemas { + // Insert into migrations_history + var historyID int + err = t.db.QueryRowContext(ctxVal, insertHistorySQL, + baseMigrationID, schema, migration.Version, + migration.Connection, migration.Backend, status, migration.ErrorMessage, + executedBy, executionMethod, migration.ExecutionContext, appliedAt, appliedAt).Scan(&historyID) if err != nil { - return fmt.Errorf("failed to upsert into migrations_list: %w", err) + return fmt.Errorf("failed to insert into migrations_history: %w", err) } - } else { - // For rollbacks, update the status in migrations_list to "rolled_back" - // but keep the last_applied_at from the most recent successful execution - updateRollbackSQL := fmt.Sprintf(` - UPDATE %s - SET last_status = 'rolled_back', - last_history_id = $1, - last_updated_at = CURRENT_TIMESTAMP - WHERE migration_id = $2 - `, listTableName) - _, err = t.db.ExecContext(ctxVal, updateRollbackSQL, historyID, baseMigrationID) + // Insert into migrations_executions + _, err = t.db.ExecContext(ctxVal, insertExecutionSQL, + baseMigrationID, schema, migration.Version, + migration.Connection, migration.Backend, execStatus, applied, appliedAtPtr) if err != nil { - // If update fails, it might be because the migration doesn't exist in list yet - // This can happen if a rollback is executed before the migration is in the list - // Log but don't fail - fmt.Printf("Warning: Failed to update migrations_list for rollback: %v\n", err) + return fmt.Errorf("failed to insert into migrations_executions: %w", err) } } + if err != nil { + return fmt.Errorf("failed to insert into migrations_executions: %w", err) + } return nil } @@ -252,7 +398,7 @@ func (t *Tracker) GetMigrationHistory(ctx interface{}, filters *state.MigrationF } query := fmt.Sprintf(` - SELECT id, migration_id, schema, table_name, version, connection, backend, + SELECT id, migration_id, schema, version, connection, backend, applied_at, status, error_message, executed_by, execution_method, execution_context FROM %s WHERE 1=1 `, historyTableName) @@ -262,15 +408,12 @@ func (t *Tracker) GetMigrationHistory(ctx interface{}, filters *state.MigrationF if filters != nil { if filters.Schema != "" { - query += fmt.Sprintf(" AND schema = $%d", argIndex) + // For VARCHAR schema column, check if schema is in comma-separated string + // Match exact schema or schema in comma-separated list + query += fmt.Sprintf(" AND (schema = $%d OR schema LIKE $%d || ',%%' OR schema LIKE '%%,' || $%d || ',%%' OR schema LIKE '%%,' || $%d)", argIndex, argIndex, argIndex, argIndex) args = append(args, filters.Schema) argIndex++ } - if filters.Table != "" { - query += fmt.Sprintf(" AND table_name = $%d", argIndex) - args = append(args, filters.Table) - argIndex++ - } if filters.Connection != "" { query += fmt.Sprintf(" AND connection = $%d", argIndex) args = append(args, filters.Connection) @@ -310,7 +453,6 @@ func (t *Tracker) GetMigrationHistory(ctx interface{}, filters *state.MigrationF &id, &record.MigrationID, &record.Schema, - &record.Table, &record.Version, &record.Connection, &record.Backend, @@ -343,8 +485,8 @@ func (t *Tracker) GetMigrationList(ctx interface{}, filters *state.MigrationFilt } query := fmt.Sprintf(` - SELECT migration_id, schema, table_name, version, name, connection, backend, - last_status, last_applied_at, last_error_message + SELECT migration_id, schema, version, name, connection, backend, + status, created_at, updated_at FROM %s WHERE 1=1 `, listTableName) @@ -353,15 +495,12 @@ func (t *Tracker) GetMigrationList(ctx interface{}, filters *state.MigrationFilt if filters != nil { if filters.Schema != "" { - query += fmt.Sprintf(" AND schema = $%d", argIndex) + // For VARCHAR schema column, check if schema is in comma-separated string + // Match exact schema or schema in comma-separated list + query += fmt.Sprintf(" AND (schema = $%d OR schema LIKE $%d || ',%%' OR schema LIKE '%%,' || $%d || ',%%' OR schema LIKE '%%,' || $%d)", argIndex, argIndex, argIndex, argIndex) args = append(args, filters.Schema) argIndex++ } - if filters.Table != "" { - query += fmt.Sprintf(" AND table_name = $%d", argIndex) - args = append(args, filters.Table) - argIndex++ - } if filters.Connection != "" { query += fmt.Sprintf(" AND connection = $%d", argIndex) args = append(args, filters.Connection) @@ -373,7 +512,7 @@ func (t *Tracker) GetMigrationList(ctx interface{}, filters *state.MigrationFilt argIndex++ } if filters.Status != "" { - query += fmt.Sprintf(" AND last_status = $%d", argIndex) + query += fmt.Sprintf(" AND status = $%d", argIndex) args = append(args, filters.Status) argIndex++ } @@ -392,32 +531,35 @@ func (t *Tracker) GetMigrationList(ctx interface{}, filters *state.MigrationFilt var items []*state.MigrationListItem for rows.Next() { var item state.MigrationListItem - var lastAppliedAt sql.NullTime - var lastErrorMessage sql.NullString + var createdAt sql.NullTime + var updatedAt sql.NullTime err := rows.Scan( &item.MigrationID, &item.Schema, - &item.Table, &item.Version, &item.Name, &item.Connection, &item.Backend, &item.LastStatus, - &lastAppliedAt, - &lastErrorMessage, + &createdAt, + &updatedAt, ) if err != nil { return nil, fmt.Errorf("failed to scan migration list item: %w", err) } - if lastAppliedAt.Valid { - item.LastAppliedAt = lastAppliedAt.Time.Format(time.RFC3339) + // Map status values for compatibility + if item.LastStatus == "applied" { + item.Applied = true + } else { + item.Applied = false } - if lastErrorMessage.Valid { - item.LastErrorMessage = lastErrorMessage.String + + // Use updated_at as last_applied_at if status is applied + if item.Applied && updatedAt.Valid { + item.LastAppliedAt = updatedAt.Time.Format(time.RFC3339) } - item.Applied = item.LastStatus == "success" items = append(items, &item) } @@ -425,6 +567,247 @@ func (t *Tracker) GetMigrationList(ctx interface{}, filters *state.MigrationFilt return items, rows.Err() } +// GetMigrationDetail retrieves detailed information about a single migration from migrations_list +func (t *Tracker) GetMigrationDetail(ctx interface{}, migrationID string) (*state.MigrationDetail, error) { + ctxVal := ctx.(context.Context) + + listTableName := "migrations_list" + if t.schema != "" && t.schema != "public" { + listTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_list")) + } + + // Remove schema prefix if present to get base migration_id + baseMigrationID := migrationID + parts := strings.Split(migrationID, "_") + if len(parts) >= 5 { + // Check if first part is a version (all digits, 10-20 chars) + firstPart := parts[0] + isVersion := len(firstPart) >= 10 && len(firstPart) <= 20 + if isVersion { + allDigits := true + for _, r := range firstPart { + if r < '0' || r > '9' { + allDigits = false + break + } + } + isVersion = allDigits + } + // If first part is not a version, it's likely a schema prefix - remove it + if !isVersion { + baseMigrationID = strings.Join(parts[1:], "_") + } + } + + query := fmt.Sprintf(` + SELECT migration_id, schema, version, name, connection, backend, + up_sql, down_sql, dependencies, structured_dependencies, status, created_at, updated_at + FROM %s WHERE migration_id = $1 + `, listTableName) + + var detail state.MigrationDetail + var schemaStr sql.NullString + var upSQL, downSQL sql.NullString + var dependencies pq.StringArray + var structuredDepsJSON sql.NullString + var createdAt, updatedAt sql.NullTime + + err := t.db.QueryRowContext(ctxVal, query, baseMigrationID).Scan( + &detail.MigrationID, + &schemaStr, + &detail.Version, + &detail.Name, + &detail.Connection, + &detail.Backend, + &upSQL, + &downSQL, + &dependencies, + &structuredDepsJSON, + &detail.Status, + &createdAt, + &updatedAt, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to query migration detail: %w", err) + } + + if schemaStr.Valid { + detail.Schema = schemaStr.String + } + if upSQL.Valid { + detail.UpSQL = upSQL.String + } + if downSQL.Valid { + detail.DownSQL = downSQL.String + } + if dependencies != nil { + detail.Dependencies = []string(dependencies) + } + if structuredDepsJSON.Valid && structuredDepsJSON.String != "" { + var structuredDeps []backends.Dependency + if err := json.Unmarshal([]byte(structuredDepsJSON.String), &structuredDeps); err == nil { + detail.StructuredDependencies = structuredDeps + } + } + + return &detail, nil +} + +// GetMigrationExecutions retrieves all execution records for a migration, ordered by created_at DESC +func (t *Tracker) GetMigrationExecutions(ctx interface{}, migrationID string) ([]*state.MigrationExecution, error) { + ctxVal := ctx.(context.Context) + + executionsTableName := "migrations_executions" + if t.schema != "" && t.schema != "public" { + executionsTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_executions")) + } + + // Remove schema prefix if present to get base migration_id + baseMigrationID := migrationID + parts := strings.Split(migrationID, "_") + if len(parts) >= 5 { + // Check if first part is a version (all digits, 10-20 chars) + firstPart := parts[0] + isVersion := len(firstPart) >= 10 && len(firstPart) <= 20 + if isVersion { + allDigits := true + for _, r := range firstPart { + if r < '0' || r > '9' { + allDigits = false + break + } + } + isVersion = allDigits + } + // If first part is not a version, it's likely a schema prefix - remove it + if !isVersion { + baseMigrationID = strings.Join(parts[1:], "_") + } + } + + query := fmt.Sprintf(` + SELECT id, migration_id, schema, version, connection, backend, + status, applied, applied_at, created_at, updated_at + FROM %s WHERE migration_id = $1 + ORDER BY created_at DESC + `, executionsTableName) + + rows, err := t.db.QueryContext(ctxVal, query, baseMigrationID) + if err != nil { + return nil, fmt.Errorf("failed to query migration executions: %w", err) + } + defer func() { _ = rows.Close() }() + + var executions []*state.MigrationExecution + for rows.Next() { + var exec state.MigrationExecution + var schemaStr sql.NullString + var appliedAt, createdAt, updatedAt sql.NullTime + + err := rows.Scan( + &exec.ID, + &exec.MigrationID, + &schemaStr, + &exec.Version, + &exec.Connection, + &exec.Backend, + &exec.Status, + &exec.Applied, + &appliedAt, + &createdAt, + &updatedAt, + ) + if err != nil { + return nil, fmt.Errorf("failed to scan migration execution: %w", err) + } + + if schemaStr.Valid { + exec.Schema = schemaStr.String + } + if appliedAt.Valid { + exec.AppliedAt = appliedAt.Time.Format(time.RFC3339) + } + if createdAt.Valid { + exec.CreatedAt = createdAt.Time.Format(time.RFC3339) + } + if updatedAt.Valid { + exec.UpdatedAt = updatedAt.Time.Format(time.RFC3339) + } + + executions = append(executions, &exec) + } + + return executions, rows.Err() +} + +// GetRecentExecutions retrieves recent execution records across all migrations, ordered by created_at DESC +func (t *Tracker) GetRecentExecutions(ctx interface{}, limit int) ([]*state.MigrationExecution, error) { + ctxVal := ctx.(context.Context) + + executionsTableName := "migrations_executions" + if t.schema != "" && t.schema != "public" { + executionsTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_executions")) + } + + query := fmt.Sprintf(` + SELECT id, migration_id, schema, version, connection, backend, + status, applied, applied_at, created_at, updated_at + FROM %s + ORDER BY created_at DESC + LIMIT $1 + `, executionsTableName) + + rows, err := t.db.QueryContext(ctxVal, query, limit) + if err != nil { + return nil, fmt.Errorf("failed to query recent executions: %w", err) + } + defer func() { _ = rows.Close() }() + + var executions []*state.MigrationExecution + for rows.Next() { + var exec state.MigrationExecution + var schemaStr sql.NullString + var appliedAt, createdAt, updatedAt sql.NullTime + + err := rows.Scan( + &exec.ID, + &exec.MigrationID, + &schemaStr, + &exec.Version, + &exec.Connection, + &exec.Backend, + &exec.Status, + &exec.Applied, + &appliedAt, + &createdAt, + &updatedAt, + ) + if err != nil { + return nil, fmt.Errorf("failed to scan migration execution: %w", err) + } + + if schemaStr.Valid { + exec.Schema = schemaStr.String + } + if appliedAt.Valid { + exec.AppliedAt = appliedAt.Time.Format(time.RFC3339) + } + if createdAt.Valid { + exec.CreatedAt = createdAt.Time.Format(time.RFC3339) + } + if updatedAt.Valid { + exec.UpdatedAt = updatedAt.Time.Format(time.RFC3339) + } + + executions = append(executions, &exec) + } + + return executions, rows.Err() +} + // IsMigrationApplied checks if a migration has been applied func (t *Tracker) IsMigrationApplied(ctx interface{}, migrationID string) (bool, error) { ctxVal := ctx.(context.Context) @@ -434,7 +817,7 @@ func (t *Tracker) IsMigrationApplied(ctx interface{}, migrationID string) (bool, listTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_list")) } - query := fmt.Sprintf("SELECT EXISTS(SELECT 1 FROM %s WHERE migration_id = $1 AND last_status = 'success')", listTableName) + query := fmt.Sprintf("SELECT EXISTS(SELECT 1 FROM %s WHERE migration_id = $1 AND status = 'applied')", listTableName) var exists bool err := t.db.QueryRowContext(ctxVal, query, migrationID).Scan(&exists) if err != nil { @@ -456,13 +839,13 @@ func (t *Tracker) GetLastMigrationVersion(ctx interface{}, schema, table string) query := fmt.Sprintf(` SELECT version FROM %s - WHERE schema = $1 AND table_name = $2 AND last_status = 'success' + WHERE (schema = $1 OR schema LIKE $1 || ',%%' OR schema LIKE '%%,' || $1 || ',%%' OR schema LIKE '%%,' || $1) AND status = 'applied' ORDER BY version DESC LIMIT 1 `, listTableName) var version string - err := t.db.QueryRowContext(ctxVal, query, schema, table).Scan(&version) + err := t.db.QueryRowContext(ctxVal, query, schema).Scan(&version) if err == sql.ErrNoRows { return "", nil } @@ -482,14 +865,20 @@ func (t *Tracker) RegisterScannedMigration(ctx interface{}, migrationID, schema, listTableName = quoteIdentifier(t.schema) + "." + quoteIdentifier("migrations_list") } - insertListSQL := `INSERT INTO ` + listTableName + ` (migration_id, "schema", table_name, version, name, connection, backend, - last_status, first_seen_at, last_updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + // migrations_list should always be inserted (even with empty schema) for dependency resolution + // Use empty string if schema is not provided + schemaValue := schema + if schemaValue == "" { + schemaValue = "" // Empty string is allowed for migrations_list + } + + insertListSQL := `INSERT INTO ` + listTableName + ` (migration_id, schema, version, name, connection, backend, status, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (migration_id) DO NOTHING` now := time.Now() _, err := t.db.ExecContext(ctxVal, insertListSQL, - migrationID, schema, table, version, name, connection, backend, + migrationID, schemaValue, version, name, connection, backend, "pending", now, now) if err != nil { return fmt.Errorf("failed to register scanned migration: %w", err) @@ -507,20 +896,26 @@ func (t *Tracker) UpdateMigrationInfo(ctx interface{}, migrationID, schema, tabl listTableName = quoteIdentifier(t.schema) + "." + quoteIdentifier("migrations_list") } + // migrations_list should always be updated (even with empty schema) for dependency resolution + // Use empty string if schema is not provided + schemaValue := schema + if schemaValue == "" { + schemaValue = "" // Empty string is allowed for migrations_list + } + updateSQL := fmt.Sprintf(` UPDATE %s - SET "schema" = $1, - table_name = $2, - version = $3, - name = $4, - connection = $5, - backend = $6, - last_updated_at = CURRENT_TIMESTAMP - WHERE migration_id = $7 + SET schema = $1, + version = $2, + name = $3, + connection = $4, + backend = $5, + updated_at = CURRENT_TIMESTAMP + WHERE migration_id = $6 `, listTableName) result, err := t.db.ExecContext(ctxVal, updateSQL, - schema, table, version, name, connection, backend, migrationID) + schemaValue, version, name, connection, backend, migrationID) if err != nil { return fmt.Errorf("failed to update migration info: %w", err) } @@ -555,6 +950,368 @@ func (t *Tracker) DeleteMigration(ctx interface{}, migrationID string) error { return nil } +// getMigrationID generates a migration ID (same format as executor) +func (t *Tracker) getMigrationID(migration *backends.MigrationScript) string { + return fmt.Sprintf("%s_%s_%s_%s", migration.Version, migration.Name, migration.Backend, migration.Connection) +} + +// ReindexMigrations reloads the BfM migration list and updates the database state +// This should be called asynchronously in the background +func (t *Tracker) ReindexMigrations(ctx interface{}, registry interface{}) error { + ctxVal := ctx.(context.Context) + + // Type assert registry to get GetAll method + type Registry interface { + GetAll() []*backends.MigrationScript + } + reg, ok := registry.(Registry) + if !ok { + return fmt.Errorf("registry does not implement GetAll() method") + } + + listTableName := "migrations_list" + executionsTableName := "migrations_executions" + if t.schema != "" && t.schema != "public" { + listTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_list")) + executionsTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_executions")) + } + + // Step 1: Get all migrations from BfM registry + bfmMigrations := reg.GetAll() + bfmMigrationMap := make(map[string]*backends.MigrationScript) + + for _, migration := range bfmMigrations { + migrationID := t.getMigrationID(migration) + bfmMigrationMap[migrationID] = migration + } + + // Step 2: Get all migrations from database + dbMigrations, err := t.GetMigrationList(ctx, nil) + if err != nil { + return fmt.Errorf("failed to get database migrations: %w", err) + } + + dbMigrationMap := make(map[string]*state.MigrationListItem) + for _, migration := range dbMigrations { + dbMigrationMap[migration.MigrationID] = migration + } + + // Step 3: For each BfM migration, update or insert into migrations_list + for migrationID, migration := range bfmMigrationMap { + // Convert schema to array (handle single schema or multiple) + schemas := []string{} + if migration.Schema != "" { + schemas = []string{migration.Schema} + } + + // Convert dependencies to array + dependencies := migration.Dependencies + if dependencies == nil { + dependencies = []string{} + } + + // Convert structured dependencies to JSONB + structuredDepsJSON, err := json.Marshal(migration.StructuredDependencies) + if err != nil { + return fmt.Errorf("failed to marshal structured dependencies: %w", err) + } + + // Construct filenames for up_sql and down_sql + // Filename pattern: {version}_{name}.up.{sql|json} and {version}_{name}.down.{sql|json} + var upExt, downExt string + if migration.Backend == "etcd" || migration.Backend == "mongodb" { + upExt = ".up.json" + downExt = ".down.json" + } else { + upExt = ".up.sql" + downExt = ".down.sql" + } + upSQLFilename := fmt.Sprintf("%s_%s%s", migration.Version, migration.Name, upExt) + downSQLFilename := fmt.Sprintf("%s_%s%s", migration.Version, migration.Name, downExt) + + // Check if migration exists in database + dbMigration, exists := dbMigrationMap[migrationID] + + // Determine status based on execution state + status := "pending" + if exists { + // Check if migration has been executed + executed, err := t.IsMigrationApplied(ctx, migrationID) + if err == nil && executed { + status = "applied" + } else if exists && dbMigration.LastStatus == "failed" { + status = "failed" + } else if exists && dbMigration.LastStatus == "rolled_back" { + status = "rolled_back" + } else if exists { + // Map old status values + if dbMigration.LastStatus == "success" { + status = "applied" + } else { + status = dbMigration.LastStatus + } + } + } + + // Upsert into migrations_list + upsertSQL := fmt.Sprintf(` + INSERT INTO %s ( + migration_id, schema, version, name, connection, backend, + up_sql, down_sql, dependencies, structured_dependencies, status, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, CURRENT_TIMESTAMP) + ON CONFLICT (migration_id) DO UPDATE SET + schema = EXCLUDED.schema, + version = EXCLUDED.version, + name = EXCLUDED.name, + connection = EXCLUDED.connection, + backend = EXCLUDED.backend, + up_sql = EXCLUDED.up_sql, + down_sql = EXCLUDED.down_sql, + dependencies = EXCLUDED.dependencies, + structured_dependencies = EXCLUDED.structured_dependencies, + status = EXCLUDED.status, + updated_at = CURRENT_TIMESTAMP + `, listTableName) + + // migrations_list should always be inserted (even with empty schema) for dependency resolution + // Use empty string if no schema is specified + schemaValue := "" + if len(schemas) > 0 { + schemaValue = schemas[0] + } + + // Insert/update migrations_list (always, even with empty schema) + _, err = t.db.ExecContext(ctxVal, upsertSQL, + migrationID, + schemaValue, + migration.Version, + migration.Name, + migration.Connection, + migration.Backend, + upSQLFilename, + downSQLFilename, + pq.Array(dependencies), + string(structuredDepsJSON), + status, + ) + if err != nil { + return fmt.Errorf("failed to upsert migration %s: %w", migrationID, err) + } + + // Skip migrations_executions if no schemas specified + if len(schemas) == 0 { + // Still update dependencies even if no schema + if err := t.updateMigrationDependencies(ctxVal, migrationID, migration, listTableName); err != nil { + return fmt.Errorf("failed to update dependencies for %s: %w", migrationID, err) + } + continue + } + + // Insert into migrations_executions table - one record per schema + applied := status == "applied" + var appliedAt *time.Time + if applied && exists && dbMigration.LastAppliedAt != "" { + if parsed, err := time.Parse(time.RFC3339, dbMigration.LastAppliedAt); err == nil { + appliedAt = &parsed + } + } + + execStatus := "pending" + if applied { + execStatus = "applied" + } else if status == "failed" { + execStatus = "failed" + } + + insertExecutionSQL := fmt.Sprintf(` + INSERT INTO %s (migration_id, schema, version, connection, backend, status, applied, applied_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (migration_id, schema, version, connection, backend) DO UPDATE SET + status = EXCLUDED.status, + applied = EXCLUDED.applied, + applied_at = EXCLUDED.applied_at, + updated_at = CURRENT_TIMESTAMP + `, executionsTableName) + + // Create one record per schema + for _, schema := range schemas { + _, err = t.db.ExecContext(ctxVal, insertExecutionSQL, + migrationID, + schema, + migration.Version, + migration.Connection, + migration.Backend, + execStatus, + applied, + appliedAt, + ) + if err != nil { + return fmt.Errorf("failed to insert execution state for %s: %w", migrationID, err) + } + } + + // Update dependencies table + if err := t.updateMigrationDependencies(ctxVal, migrationID, migration, listTableName); err != nil { + return fmt.Errorf("failed to update dependencies for %s: %w", migrationID, err) + } + } + + // Step 4: Delete migrations that no longer exist in BfM + for migrationID := range dbMigrationMap { + if _, exists := bfmMigrationMap[migrationID]; !exists { + if err := t.DeleteMigration(ctx, migrationID); err != nil { + // Log but continue + fmt.Printf("Warning: Failed to delete migration %s: %v\n", migrationID, err) + } + } + } + + return nil +} + +// updateMigrationDependencies updates the migrations_dependencies table +func (t *Tracker) updateMigrationDependencies(ctx context.Context, migrationID string, migration *backends.MigrationScript, listTableName string) error { + dependenciesTableName := "migrations_dependencies" + if t.schema != "" && t.schema != "public" { + dependenciesTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("migrations_dependencies")) + } + + // Delete existing dependencies for this migration + deleteSQL := fmt.Sprintf("DELETE FROM %s WHERE migration_id = $1", dependenciesTableName) + _, err := t.db.ExecContext(ctx, deleteSQL, migrationID) + if err != nil { + return fmt.Errorf("failed to delete existing dependencies: %w", err) + } + + // Insert structured dependencies + for _, dep := range migration.StructuredDependencies { + // Find dependency_id by resolving the dependency target + dependencyID, err := t.resolveDependencyID(ctx, dep, listTableName) + if err != nil { + // Log but continue - dependency might not exist yet + fmt.Printf("Warning: Failed to resolve dependency for %s: %v\n", migrationID, err) + continue + } + + schemas := []string{} + if dep.Schema != "" { + schemas = []string{dep.Schema} + } + + insertSQL := fmt.Sprintf(` + INSERT INTO %s ( + migration_id, dependency_id, connection, schema, target, target_type, + requires_table, requires_schema + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + `, dependenciesTableName) + + targetType := dep.TargetType + if targetType == "" { + targetType = "name" + } + + _, err = t.db.ExecContext(ctx, insertSQL, + migrationID, + dependencyID, + dep.Connection, + pq.Array(schemas), + dep.Target, + targetType, + dep.RequiresTable, + dep.RequiresSchema, + ) + if err != nil { + return fmt.Errorf("failed to insert dependency: %w", err) + } + } + + // Insert simple dependencies (convert to structured format) + for _, depName := range migration.Dependencies { + // Find dependency_id by name + dependencyID, err := t.findMigrationIDByName(ctx, depName, listTableName) + if err != nil { + // Log but continue + fmt.Printf("Warning: Failed to find dependency %s for %s: %v\n", depName, migrationID, err) + continue + } + + schemas := []string{} + if migration.Schema != "" { + schemas = []string{migration.Schema} + } + + insertSQL := fmt.Sprintf(` + INSERT INTO %s ( + migration_id, dependency_id, connection, schema, target, target_type + ) + VALUES ($1, $2, $3, $4, $5, $6) + `, dependenciesTableName) + + _, err = t.db.ExecContext(ctx, insertSQL, + migrationID, + dependencyID, + migration.Connection, + pq.Array(schemas), + depName, + "name", + ) + if err != nil { + return fmt.Errorf("failed to insert simple dependency: %w", err) + } + } + + return nil +} + +// resolveDependencyID resolves a dependency to a migration_id +func (t *Tracker) resolveDependencyID(ctx context.Context, dep backends.Dependency, listTableName string) (string, error) { + var query string + var args []interface{} + + if dep.TargetType == "version" { + query = fmt.Sprintf(` + SELECT migration_id FROM %s + WHERE connection = $1 AND version = $2 + LIMIT 1 + `, listTableName) + args = []interface{}{dep.Connection, dep.Target} + } else { + query = fmt.Sprintf(` + SELECT migration_id FROM %s + WHERE connection = $1 AND name = $2 + LIMIT 1 + `, listTableName) + args = []interface{}{dep.Connection, dep.Target} + } + + var migrationID string + err := t.db.QueryRowContext(ctx, query, args...).Scan(&migrationID) + if err != nil { + return "", fmt.Errorf("dependency not found: %w", err) + } + + return migrationID, nil +} + +// findMigrationIDByName finds a migration_id by name +func (t *Tracker) findMigrationIDByName(ctx context.Context, name string, listTableName string) (string, error) { + query := fmt.Sprintf(` + SELECT migration_id FROM %s + WHERE name = $1 + LIMIT 1 + `, listTableName) + + var migrationID string + err := t.db.QueryRowContext(ctx, query, name).Scan(&migrationID) + if err != nil { + return "", fmt.Errorf("migration not found: %w", err) + } + + return migrationID, nil +} + // Close closes the database connection func (t *Tracker) Close() error { if t.db != nil { @@ -564,7 +1321,7 @@ func (t *Tracker) Close() error { } // migrateExistingData migrates data from old bfm_migrations table to new tables -func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, historyTableName string) error { +func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, historyTableName, executionsTableName, dependenciesTableName string) error { oldTableName := "bfm_migrations" if t.schema != "" && t.schema != "public" { oldTableName = fmt.Sprintf("%s.%s", quoteIdentifier(t.schema), quoteIdentifier("bfm_migrations")) @@ -713,48 +1470,103 @@ func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, histor // Determine last status lastStatus := latestRecord.status lastAppliedAt := latestRecord.appliedAt - lastErrorMessage := latestRecord.errorMsg + + // Map old status values to new ones + if lastStatus == "success" { + lastStatus = "applied" + } // If latest is a rollback, check if there's a more recent success if strings.Contains(latestRecord.migrationID, "_rollback") { lastStatus = "rolled_back" if latestSuccessRecord != nil { lastAppliedAt = latestSuccessRecord.appliedAt - lastErrorMessage = latestSuccessRecord.errorMsg } } - // Extract name from migration_id (format: {schema}_{connection}_{version}_{name}) + // Extract name from migration_id (format: {version}_{name}_{backend}_{connection}) name := baseMigrationID parts := strings.Split(baseMigrationID, "_") if len(parts) >= 4 { - name = strings.Join(parts[3:], "_") + // Format: {version}_{name}_{backend}_{connection} + name = parts[1] } // Use metadata from the first record (all records for same baseMigrationID should have same metadata) schema := latestRecord.schema - tableName := latestRecord.tableName version := latestRecord.version connection := latestRecord.connection backend := latestRecord.backend + // migrations_list should always be inserted (even with empty schema) for dependency resolution + // Use empty string if schema is not provided + schemaValue := schema + if schemaValue == "" { + schemaValue = "" // Empty string is allowed for migrations_list + } + + // Insert into migrations_list (one record per migration, even with empty schema) insertListSQL := fmt.Sprintf(` - INSERT INTO %s (migration_id, schema, table_name, version, name, connection, backend, - last_status, last_applied_at, last_error_message, first_seen_at, last_updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + INSERT INTO %s (migration_id, schema, version, name, connection, backend, + status, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (migration_id) DO UPDATE SET - last_status = EXCLUDED.last_status, - last_applied_at = EXCLUDED.last_applied_at, - last_error_message = EXCLUDED.last_error_message, - last_updated_at = CURRENT_TIMESTAMP + schema = EXCLUDED.schema, + version = EXCLUDED.version, + name = EXCLUDED.name, + connection = EXCLUDED.connection, + backend = EXCLUDED.backend, + status = EXCLUDED.status, + updated_at = CURRENT_TIMESTAMP `, listTableName) _, err := t.db.ExecContext(ctx, insertListSQL, - baseMigrationID, schema, tableName, version, name, connection, backend, - lastStatus, lastAppliedAt, lastErrorMessage, lastAppliedAt, time.Now()) + baseMigrationID, schemaValue, version, name, connection, backend, + lastStatus, lastAppliedAt, time.Now()) if err != nil { return fmt.Errorf("failed to insert into migrations_list: %w", err) } + + // Skip migrations_executions if schema is empty + if schema == "" { + continue + } + + // Populate migrations_executions table - one record per schema + applied := lastStatus == "applied" + var appliedAtPtr *time.Time + if applied { + appliedAtPtr = &lastAppliedAt + } + + execStatus := "pending" + if applied { + execStatus = "applied" + } else if lastStatus == "failed" { + execStatus = "failed" + } + + insertExecutionSQL := fmt.Sprintf(` + INSERT INTO %s (migration_id, schema, version, connection, backend, status, applied, applied_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, COALESCE($9, CURRENT_TIMESTAMP), CURRENT_TIMESTAMP) + ON CONFLICT (migration_id, schema, version, connection, backend) DO UPDATE SET + status = EXCLUDED.status, + applied = EXCLUDED.applied, + applied_at = EXCLUDED.applied_at, + updated_at = CURRENT_TIMESTAMP + `, executionsTableName) + + // Ensure lastAppliedAt is not zero value (use current time if zero) + createdAt := lastAppliedAt + if createdAt.IsZero() { + createdAt = time.Now() + } + + _, err = t.db.ExecContext(ctx, insertExecutionSQL, + baseMigrationID, schema, version, connection, backend, execStatus, applied, appliedAtPtr, createdAt) + if err != nil { + return fmt.Errorf("failed to insert into migrations_executions: %w", err) + } } // PHASE 2: Now insert all history records (foreign key constraint is satisfied) @@ -763,11 +1575,22 @@ func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, histor // Extract base migration_id (remove _rollback suffix if present) isRollback := strings.Contains(record.migrationID, "_rollback") - // Insert into migrations_history (all records, including rollbacks) + // Skip if schema is empty + if record.schema == "" { + continue + } + + // Map status values + status := record.status + if status == "success" { + status = "applied" + } + + // Insert into migrations_history (all records, including rollbacks) - one record per schema insertHistorySQL := fmt.Sprintf(` - INSERT INTO %s (migration_id, schema, table_name, version, connection, backend, + INSERT INTO %s (migration_id, schema, version, connection, backend, status, error_message, executed_by, execution_method, applied_at, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) `, historyTableName) executionMethod := "api" // Default for migrated data @@ -776,8 +1599,8 @@ func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, histor } _, err := t.db.ExecContext(ctx, insertHistorySQL, - baseMigrationID, record.schema, record.tableName, record.version, record.connection, record.backend, - record.status, record.errorMsg, "system", executionMethod, record.appliedAt, record.appliedAt) + baseMigrationID, record.schema, record.version, record.connection, record.backend, + status, record.errorMsg, "system", executionMethod, record.appliedAt, record.appliedAt) if err != nil { return fmt.Errorf("failed to insert into migrations_history: %w", err) } @@ -791,3 +1614,37 @@ func (t *Tracker) migrateExistingData(ctx context.Context, listTableName, histor func quoteIdentifier(name string) string { return `"` + strings.ReplaceAll(name, `"`, `""`) + `"` } + +// configureConnectionPool configures the database connection pool with reasonable defaults +// that can be overridden via environment variables +func configureConnectionPool(db *sql.DB) { + // Max open connections per pool (default: 5) + // This limits how many connections each sql.DB instance can open + maxOpenConns := getEnvInt("BFM_DB_MAX_OPEN_CONNS", 5) + db.SetMaxOpenConns(maxOpenConns) + + // Max idle connections per pool (default: 2) + // This keeps some connections ready for reuse + maxIdleConns := getEnvInt("BFM_DB_MAX_IDLE_CONNS", 2) + db.SetMaxIdleConns(maxIdleConns) + + // Connection max lifetime (default: 5 minutes) + // This prevents using stale connections + connMaxLifetime := time.Duration(getEnvInt("BFM_DB_CONN_MAX_LIFETIME_MINUTES", 5)) * time.Minute + db.SetConnMaxLifetime(connMaxLifetime) + + // Connection max idle time (default: 1 minute) + // This closes idle connections after this duration + connMaxIdleTime := time.Duration(getEnvInt("BFM_DB_CONN_MAX_IDLE_TIME_MINUTES", 1)) * time.Minute + db.SetConnMaxIdleTime(connMaxIdleTime) +} + +// getEnvInt gets an integer environment variable or returns the default value +func getEnvInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue +} diff --git a/api/internal/state/reindexer.go b/api/internal/state/reindexer.go new file mode 100644 index 0000000..2a8d7d9 --- /dev/null +++ b/api/internal/state/reindexer.go @@ -0,0 +1,71 @@ +package state + +import ( + "context" + "time" +) + +// Reindexer handles background reindexing of migrations +type Reindexer struct { + tracker StateTracker + registry interface{} // registry.Registry + interval time.Duration + ctx context.Context + cancel context.CancelFunc + running bool +} + +// NewReindexer creates a new reindexer +func NewReindexer(tracker StateTracker, registry interface{}, interval time.Duration) *Reindexer { + ctx, cancel := context.WithCancel(context.Background()) + return &Reindexer{ + tracker: tracker, + registry: registry, + interval: interval, + ctx: ctx, + cancel: cancel, + running: false, + } +} + +// Start starts the background reindexing process +func (r *Reindexer) Start() { + if r.running { + return + } + r.running = true + + go func() { + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + + // Run immediately on start + r.reindex() + + for { + select { + case <-r.ctx.Done(): + return + case <-ticker.C: + r.reindex() + } + } + }() +} + +// Stop stops the background reindexing process +func (r *Reindexer) Stop() { + if !r.running { + return + } + r.cancel() + r.running = false +} + +// reindex performs the reindexing operation +func (r *Reindexer) reindex() { + if err := r.tracker.ReindexMigrations(r.ctx, r.registry); err != nil { + // Log error (you may want to use a logger here) + _ = err + } +} diff --git a/deploy/docker-compose.dev.yml b/deploy/docker-compose.dev.yml index d5b56e2..4f8371c 100644 --- a/deploy/docker-compose.dev.yml +++ b/deploy/docker-compose.dev.yml @@ -28,6 +28,11 @@ services: - POSTGRES_PASSWORD=${BFM_STATE_DB_PASSWORD:-postgres} - POSTGRES_DB=migration_state - POSTGRES_MULTIPLE_DATABASES=dashcloud + command: > + postgres + -c max_connections=200 + -c shared_buffers=256MB + -c effective_cache_size=1GB volumes: - bfm-postgres-data-dev:/var/lib/postgresql/data networks: @@ -50,6 +55,9 @@ services: - "7070:7070" # HTTP API - "9090:9090" # gRPC API environment: + - BFM_LOG_FORMAT=json + - BFM_LOG_LEVEL=debug + - BFM_APP_MODE=debug # Server Configuration - BFM_HTTP_PORT=7070 - BFM_GRPC_PORT=9090 diff --git a/deploy/docker-compose.standalone.yml b/deploy/docker-compose.standalone.yml index 64ccadd..b3ba75c 100644 --- a/deploy/docker-compose.standalone.yml +++ b/deploy/docker-compose.standalone.yml @@ -33,6 +33,11 @@ services: - POSTGRES_PASSWORD=${BFM_STATE_DB_PASSWORD:-postgres} - POSTGRES_DB=migration_state - POSTGRES_MULTIPLE_DATABASES=dashcloud + command: > + postgres + -c max_connections=200 + -c shared_buffers=256MB + -c effective_cache_size=1GB volumes: - bfm-postgres-data:/var/lib/postgresql/data networks: diff --git a/docs/IMPLEMENTATION_SUMMARY.md b/docs/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index f109a4a..0000000 --- a/docs/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,161 +0,0 @@ -# Migration Dependency System - Implementation Summary - -## Overview -Successfully implemented a comprehensive migration dependency system for BFM with structured dependencies, validation, and full backward compatibility. - -## Implementation Status: ✅ COMPLETE - -### Phase 1: Core Infrastructure ✅ -- ✅ Created `Dependency` struct with all required fields -- ✅ Extended `MigrationScript` with `StructuredDependencies []Dependency` -- ✅ Maintained backward compatibility with `Dependencies []string` -- ✅ Updated migration template to support structured dependencies - -### Phase 2: Dependency Resolver ✅ -- ✅ Implemented `DependencyResolver` with full dependency resolution -- ✅ Created `DependencyGraph` with cycle detection (DFS-based) -- ✅ Implemented topological sort using Kahn's algorithm -- ✅ Supports both structured and simple dependencies - -### Phase 3: Dependency Validator ✅ -- ✅ Implemented `DependencyValidator` for PostgreSQL backend -- ✅ Added `TableExists` method to PostgreSQL backend -- ✅ Validates schema existence, table existence, and migration application status -- ✅ Provides clear error messages for validation failures - -### Phase 4: Executor Integration ✅ -- ✅ Updated `executeSync` to validate dependencies before execution -- ✅ Added `resolveDependencies` method using `DependencyResolver` -- ✅ Maintains backward compatibility with existing `topologicalSort` -- ✅ Graceful error handling with fallback to version-based sort - -### Phase 5: Loader Updates ✅ -- ✅ Enhanced dependency extraction from `.go` files -- ✅ Added `extractStructuredDependenciesFromGoFile` function -- ✅ Supports parsing both simple and structured dependencies - -### Phase 6: Registry Enhancements ✅ -- ✅ Added `GetMigrationByVersion` method -- ✅ Added `GetMigrationByConnectionAndVersion` method -- ✅ Updated all mock registries in tests - -### Phase 7: Example Migrations ✅ -- ✅ Updated migration template -- ✅ Added `Dependency` type alias to migrations package -- ✅ Updated example migrations to demonstrate structured dependencies - -### Phase 8: Frontend Integration ✅ -- ✅ Added `Dependency` interface to TypeScript types -- ✅ Updated `MigrationDetailResponse` DTO -- ✅ Enhanced UI to display structured dependencies with validation requirements -- ✅ Visual indicators for connection, schema, target type, and validation requirements - -### Phase 9: Testing ✅ -- ✅ Created comprehensive unit tests for `DependencyResolver` -- ✅ Created unit tests for `DependencyValidator` -- ✅ Added integration tests for dependency execution -- ✅ Added edge case tests (circular dependencies, missing dependencies, hybrid dependencies) -- ✅ All tests passing - -### Phase 10: Documentation ✅ -- ✅ Created `docs/MIGRATION_DEPENDENCIES.md` with complete usage guide -- ✅ Documented both simple and structured dependency formats -- ✅ Included examples and troubleshooting guide - -## Test Coverage - -### Unit Tests -- `api/internal/registry/dependency_resolver_test.go` - - ✅ DependencyGraph operations (AddNode, AddEdge) - - ✅ Cycle detection (simple and complex cycles) - - ✅ Topological sort (linear, no dependencies, circular) - - ✅ Dependency target finding (by name, version, connection) - - ✅ Dependency resolution (simple and structured) - -- `api/internal/backends/postgresql/validator_test.go` - - ✅ Dependency validation logic - - ✅ Migration target finding - -### Integration Tests -- `api/internal/executor/executor_test.go` - - ✅ `TestExecutor_ExecuteSync_WithStructuredDependencies` - - ✅ `TestExecutor_ExecuteSync_WithSimpleDependencies` - - ✅ `TestExecutor_ExecuteSync_CircularDependency` - - ✅ `TestExecutor_ExecuteSync_MissingDependency` - - ✅ `TestExecutor_ExecuteSync_BothDependencyTypes` - -## Key Features - -1. **Backward Compatibility**: Existing migrations with `Dependencies []string` continue to work -2. **Structured Dependencies**: Advanced dependency system with validation requirements -3. **Cross-Connection Support**: Dependencies can reference migrations across connections/backends -4. **Validation**: Schema and table existence validation before execution -5. **Cycle Detection**: Automatic detection and reporting of circular dependencies -6. **Clear Error Messages**: Detailed error messages for validation failures - -## Files Created/Modified - -### New Files -- `api/internal/registry/dependency_resolver.go` -- `api/internal/registry/dependency_resolver_test.go` -- `api/internal/backends/postgresql/validator.go` -- `api/internal/backends/postgresql/validator_test.go` -- `docs/MIGRATION_DEPENDENCIES.md` -- `docs/IMPLEMENTATION_SUMMARY.md` - -### Modified Files -- `api/internal/backends/interface.go` - Added Dependency struct -- `api/internal/executor/executor.go` - Integrated dependency resolver and validator -- `api/internal/executor/loader.go` - Enhanced dependency extraction -- `api/internal/registry/interface.go` - Added lookup methods -- `api/internal/backends/postgresql/backend.go` - Added TableExists method -- `api/migrations/types.go` - Added Dependency type alias -- `api/migrations/template.go` - Updated template -- `api/internal/api/http/dto/migrations.go` - Added DependencyResponse -- `api/internal/api/http/handler.go` - Populate structured dependencies -- `ffm/src/types/api.ts` - Added Dependency interface -- `ffm/src/components/MigrationDetail.tsx` - Enhanced UI for dependencies -- Example migration files updated - -## Usage Examples - -### Simple Dependency (Backward Compatible) -```go -Dependencies: []string{"bootstrap_solution"} -``` - -### Structured Dependency -```go -StructuredDependencies: []migrations.Dependency{ - { - Connection: "core", - Schema: "core", - Target: "bootstrap_solution", - TargetType: "name", - RequiresTable: "organizations", - RequiresSchema: "core", - }, -} -``` - -## Next Steps (Optional Enhancements) - -1. **Performance Testing**: Test with large numbers of migrations (100+) -2. **Additional Backend Support**: Extend validator to other backends (etcd, greptimedb) -3. **Dependency Visualization**: Add graph visualization in frontend -4. **Dependency Analysis Tools**: CLI tools for analyzing dependency graphs -5. **Migration Templates**: Enhanced templates with dependency helpers - -## Success Metrics - -- ✅ All unit tests passing -- ✅ All integration tests passing -- ✅ Code compiles without errors -- ✅ Backward compatibility maintained -- ✅ Documentation complete -- ✅ Frontend displays dependencies correctly -- ✅ Error handling robust - -## Conclusion - -The migration dependency system is fully implemented, tested, and documented. The system provides a robust foundation for managing complex migration dependencies while maintaining full backward compatibility with existing migrations. diff --git a/docs/MIGRATION_DEPENDENCIES.md b/docs/MIGRATION_DEPENDENCIES.md index ddfc859..8009331 100644 --- a/docs/MIGRATION_DEPENDENCIES.md +++ b/docs/MIGRATION_DEPENDENCIES.md @@ -81,6 +81,8 @@ The system automatically resolves dependencies using topological sorting (Kahn's - Required tables exist - Dependency migrations are applied +When migrations are executed via the API, BfM will also automatically **include pending dependency migrations** referenced by structured dependencies in the execution plan, even if they belong to different connections/schemas. Already-applied dependencies are never re-executed; only migrations that are still pending are added and ordered ahead of their dependents. + ## Validation Before executing a migration, the system validates: diff --git a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.down.sql b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.down.sql index 0d3872f..5ba7775 100644 --- a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.down.sql +++ b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.down.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS core.solution_runs; +DROP TABLE IF EXISTS solution_runs; diff --git a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.go b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.go index 3aa4adf..9aabc77 100644 --- a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.go +++ b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.go @@ -1,9 +1,11 @@ //go:build ignore + package core import ( - "github.com/toolsascode/bfm/api/migrations" _ "embed" + + "github.com/toolsascode/bfm/api/migrations" ) //go:embed 20250115000000_bootstrap_solution.up.sql @@ -14,14 +16,14 @@ var downSQL string func init() { migration := &migrations.MigrationScript{ - Schema: "", // Dynamic - provided in request - Version: "20250115000000", - Name: "bootstrap_solution", - Connection: "core", - Backend: "postgresql", - UpSQL: upSQL, - DownSQL: downSQL, - Dependencies: []string{ }, + Schema: "", // Dynamic - provided in request + Version: "20250115000000", + Name: "bootstrap_solution", + Connection: "core", + Backend: "postgresql", + UpSQL: upSQL, + DownSQL: downSQL, + Dependencies: []string{}, StructuredDependencies: []migrations.Dependency{}, } migrations.GlobalRegistry.Register(migration) diff --git a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.up.sql b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.up.sql index 6e24262..dfb5aa0 100644 --- a/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.up.sql +++ b/examples/sfm/postgresql/core/20250115000000_bootstrap_solution.up.sql @@ -1,6 +1,6 @@ -CREATE SCHEMA IF NOT EXISTS core; +-- CREATE SCHEMA IF NOT EXISTS core; -CREATE TABLE IF NOT EXISTS core.solution_runs ( +CREATE TABLE IF NOT EXISTS solution_runs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), environment_id UUID NOT NULL, feature_flag TEXT NOT NULL, @@ -10,4 +10,4 @@ CREATE TABLE IF NOT EXISTS core.solution_runs ( ); CREATE UNIQUE INDEX IF NOT EXISTS idx_solution_runs_environment_feature - ON core.solution_runs (environment_id, feature_flag); + ON solution_runs (environment_id, feature_flag); diff --git a/ffm/.eslintrc.cjs b/ffm/.eslintrc.cjs index 05a853e..b4c91a3 100644 --- a/ffm/.eslintrc.cjs +++ b/ffm/.eslintrc.cjs @@ -6,7 +6,7 @@ module.exports = { 'plugin:@typescript-eslint/recommended', 'plugin:react-hooks/recommended', ], - ignorePatterns: ['dist', '.eslintrc.cjs'], + ignorePatterns: ['dist', '.eslintrc.cjs', 'public/runtime-config.js'], parser: '@typescript-eslint/parser', plugins: ['react-refresh'], rules: { diff --git a/ffm/Dockerfile.dev b/ffm/Dockerfile.dev index c9a3080..5d78f6c 100644 --- a/ffm/Dockerfile.dev +++ b/ffm/Dockerfile.dev @@ -7,7 +7,7 @@ WORKDIR /app # Install dependencies (will be overridden by volume mount, but useful for initial setup) COPY package*.json .npmrc* ./ -RUN npm ci +RUN npm install # Copy source code (will be mounted as volume in docker-compose) # This copy is for initial setup, actual development uses volume mount diff --git a/ffm/nginx.conf b/ffm/nginx.conf deleted file mode 100644 index b312a61..0000000 --- a/ffm/nginx.conf +++ /dev/null @@ -1,57 +0,0 @@ -server { - listen 80; - server_name localhost; - root /usr/share/nginx/html; - index index.html; - - # Resolver for dynamic DNS resolution (prevents DNS caching issues) - # Uses Docker's embedded DNS server (127.0.0.11) with 30s cache validity - resolver 127.0.0.11 valid=30s; - - # Gzip compression - gzip on; - gzip_vary on; - gzip_min_length 1024; - gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml+rss application/json; - - # SPA routing - location / { - try_files $uri $uri/ /index.html; - } - - # API proxy to BFM server - location /api { - set $backend "http://bfm-server:7070"; - proxy_pass $backend; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache_bypass $http_upgrade; - - # CORS headers (if not handled by backend) - add_header Access-Control-Allow-Origin * always; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS, PATCH" always; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, X-Requested-With" always; - - # Handle preflight requests - if ($request_method = 'OPTIONS') { - add_header Access-Control-Allow-Origin * always; - add_header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS, PATCH" always; - add_header Access-Control-Allow-Headers "Content-Type, Authorization, X-Requested-With" always; - add_header Access-Control-Max-Age 1728000; - add_header Content-Type 'text/plain charset=UTF-8'; - add_header Content-Length 0; - return 204; - } - } - - # Cache static assets - location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg)$ { - expires 1y; - add_header Cache-Control "public, immutable"; - } -} diff --git a/ffm/package-lock.json b/ffm/package-lock.json index 22a400e..8fced82 100644 --- a/ffm/package-lock.json +++ b/ffm/package-lock.json @@ -10,12 +10,14 @@ "dependencies": { "axios": "^1.6.2", "date-fns": "^2.30.0", - "react": "^19.2.0", + "react": "^19.2.3", "react-dom": "^19.2.3", + "react-is": "^18.3.1", "react-router-dom": "^7.10.1", "recharts": "^3.5.1" }, "devDependencies": { + "@types/node": "^22.10.2", "@types/react": "^19.2.6", "@types/react-dom": "^19.2.3", "@typescript-eslint/eslint-plugin": "^8.48.1", @@ -1497,6 +1499,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "22.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.2.tgz", + "integrity": "sha512-LPM2G3Syo1GLzXLGJAKdqoU35XvrWzGJ21/7sgZTUpbkBaOasTj8tjwn6w+hCkqaa1TfJ/w67rJSwYItlJ2mYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/@types/react": { "version": "19.2.6", "dev": true, @@ -3968,7 +3980,9 @@ "license": "MIT" }, "node_modules/react": { - "version": "19.2.0", + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", + "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", "engines": { "node": ">=0.10.0" @@ -3986,6 +4000,12 @@ "react": "^19.2.3" } }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "license": "MIT" + }, "node_modules/react-redux": { "version": "9.2.0", "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", @@ -4462,6 +4482,13 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "node_modules/update-browserslist-db": { "version": "1.1.4", "dev": true, diff --git a/ffm/package.json b/ffm/package.json index 680b31d..7d4e7d6 100644 --- a/ffm/package.json +++ b/ffm/package.json @@ -13,12 +13,14 @@ "dependencies": { "axios": "^1.6.2", "date-fns": "^2.30.0", - "react": "^19.2.0", - "react-router-dom": "^7.10.1", + "react": "^19.2.3", "react-dom": "^19.2.3", + "react-is": "^18.3.1", + "react-router-dom": "^7.10.1", "recharts": "^3.5.1" }, "devDependencies": { + "@types/node": "^22.10.2", "@types/react": "^19.2.6", "@types/react-dom": "^19.2.3", "@typescript-eslint/eslint-plugin": "^8.48.1", diff --git a/ffm/src/components/Dashboard.tsx b/ffm/src/components/Dashboard.tsx index ce81c63..2eb5089 100644 --- a/ffm/src/components/Dashboard.tsx +++ b/ffm/src/components/Dashboard.tsx @@ -1,7 +1,8 @@ import { useState, useEffect } from "react"; import { Link } from "react-router-dom"; import { apiClient } from "../services/api"; -import type { MigrationListItem } from "../types/api"; +import { toastService } from "../services/toast"; +import type { MigrationListItem, MigrationExecution } from "../types/api"; import { format } from "date-fns"; import { BarChart, @@ -19,9 +20,13 @@ import { export default function Dashboard() { const [migrations, setMigrations] = useState([]); + const [recentExecutions, setRecentExecutions] = useState< + MigrationExecution[] + >([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [healthStatus, setHealthStatus] = useState("unknown"); + const [reindexing, setReindexing] = useState(false); useEffect(() => { loadData(); @@ -32,14 +37,16 @@ export default function Dashboard() { const loadData = async () => { try { setLoading(true); - const [migrationsData, health] = await Promise.all([ + const [migrationsData, health, executionsData] = await Promise.all([ apiClient.listMigrations(), apiClient .healthCheck() .catch(() => ({ status: "unknown", checks: {} })), + apiClient.getRecentExecutions(10).catch(() => ({ executions: [] })), ]); setMigrations(migrationsData.items); setHealthStatus(health.status); + setRecentExecutions(executionsData.executions); setError(null); } catch (err) { const errorMsg = @@ -51,6 +58,35 @@ export default function Dashboard() { } }; + const handleReindex = async () => { + if (reindexing) return; + + setReindexing(true); + try { + const result = await apiClient.reindexMigrations(); + const addedCount = result.added.length; + const removedCount = result.removed.length; + + let message = `Reindexing completed. Total migrations: ${result.total}`; + if (addedCount > 0 || removedCount > 0) { + message += ` (Added: ${addedCount}, Removed: ${removedCount})`; + } else { + message += " (No changes)"; + } + + toastService.success(message); + + // Reload migrations list to reflect changes + await loadData(); + } catch (err) { + const errorMsg = + err instanceof Error ? err.message : "Failed to reindex migrations"; + toastService.error(errorMsg); + } finally { + setReindexing(false); + } + }; + if (loading && migrations.length === 0) { return (
@@ -85,7 +121,9 @@ export default function Dashboard() { // Calculate statistics based on status field const total = migrations.length; - const applied = migrations.filter((m) => m.status === "success").length; + const applied = migrations.filter( + (m) => m.status === "success" || m.status === "applied", + ).length; const pending = migrations.filter((m) => m.status === "pending").length; const failed = migrations.filter((m) => m.status === "failed").length; const rolledBack = migrations.filter( @@ -134,8 +172,9 @@ export default function Dashboard() { { name: "Remaining", value: 100 - healthScore, fill: "#e5e7eb" }, ]; - // Calculate needle angle (0-180 degrees, where 180° is left, 0° is right) + // Calculate needle angle for gauge (0-180 degrees) // Health score 0% = 180° (left), 100% = 0° (right) + // For SVG: 0° points right, 90° points up, 180° points left const needleAngle = 180 - (healthScore / 100) * 180; // Group by backend @@ -176,17 +215,6 @@ export default function Dashboard() { value, })); - // Recent executions - show migrations that have been executed (not pending) - // Sort by applied_at if available, otherwise by migration_id - const recentMigrations = migrations - .filter((m) => m.status !== "pending" && m.applied_at) - .sort((a, b) => { - const dateA = new Date(a.applied_at || 0).getTime(); - const dateB = new Date(b.applied_at || 0).getTime(); - return dateB - dateA; // Most recent first - }) - .slice(0, 10); - // Recently added files - show migrations sorted by version (newest first) // The version field contains a timestamp (YYYYMMDDHHmmss), so higher version = more recent const recentlyAddedFiles = [...migrations] @@ -197,19 +225,28 @@ export default function Dashboard() { .slice(0, 5); return ( -
+

Migration Dashboard

-
- Status: {healthStatus === "healthy" ? "✓ Healthy" : "✗ Unhealthy"} +
+ +
+ Status: {healthStatus === "healthy" ? "✓ Healthy" : "✗ Unhealthy"} +
@@ -221,8 +258,21 @@ export default function Dashboard() {
Health Score
-
- +
+ {gaugeData.map((entry, index) => ( @@ -241,30 +292,33 @@ export default function Dashboard() { - {/* Needle overlay using absolute positioning */} -
- - {/* Needle */} - - - - - -
-
+ {/* Needle overlay - positioned to match Pie chart center */} + + {/* Needle pivot point at (100, 90) - center horizontally, 90% down vertically */} + + {/* Needle line */} + + {/* Needle center circle */} + + + + {/* Health score text overlay */} +
{healthScore}%
@@ -327,7 +381,12 @@ export default function Dashboard() {

Status Distribution

- + { // Only show label if slice is large enough (>= 5%) - if (percent >= 0.05) { + if (percent !== undefined && percent >= 0.05) { return `${name}\n${value} (${(percent * 100).toFixed(0)}%)`; } return ""; @@ -373,7 +432,12 @@ export default function Dashboard() {

Migrations by Backend

- + @@ -388,7 +452,12 @@ export default function Dashboard() {

Migrations by Connection

- + @@ -454,7 +523,8 @@ export default function Dashboard() { Migration ID - {/* + Schema - Table - */} + Connection + Backend @@ -501,54 +571,58 @@ export default function Dashboard() { Status - Applied At + Created At - {recentMigrations.length === 0 ? ( + {recentExecutions.length === 0 ? ( - No migrations applied yet + No executions found ) : ( - recentMigrations.map((migration) => ( - + recentExecutions.map((execution) => ( + - {migration.migration_id} + {execution.migration_id} - {/* {migration.schema} - {migration.table} */} - {migration.backend} + {execution.schema || "-"} + + + {execution.connection} + + + {execution.backend} - {migration.status === "rolled_back" + {execution.status === "rolled_back" ? "Rolled Back" - : migration.status} + : execution.status || "pending"} - {migration.applied_at + {execution.created_at ? format( - new Date(migration.applied_at), + new Date(execution.created_at), "yyyy-MM-dd HH:mm:ss", ) : "-"} diff --git a/ffm/src/components/Layout.tsx b/ffm/src/components/Layout.tsx index 1dc2f30..802fb08 100644 --- a/ffm/src/components/Layout.tsx +++ b/ffm/src/components/Layout.tsx @@ -193,6 +193,34 @@ export default function Layout({ onLogout }: LayoutProps) { + {authEnabled && (
({ up: false, down: false }); + const [schemaExecutions, setSchemaExecutions] = useState( + [], + ); + const [schemaExecutionsExpanded, setSchemaExecutionsExpanded] = + useState(true); + const [schemaExecutionsPage, setSchemaExecutionsPage] = useState(1); + const [schemaExecutionsPerPage, setSchemaExecutionsPerPage] = useState(10); + const [executions, setExecutions] = useState([]); + const [executionsLoading, setExecutionsLoading] = useState(false); + const [historyPage, setHistoryPage] = useState(1); + const [historyPerPage, setHistoryPerPage] = useState(10); useEffect(() => { if (id) { loadMigration(); loadStatus(); loadHistory(); + loadExecutions(); const interval = setInterval(() => { loadStatus(); loadHistory(); + loadExecutions(); }, 5000); // Refresh status every 5 seconds return () => clearInterval(interval); } @@ -211,6 +226,10 @@ export default function MigrationDetail() { const data = await apiClient.getMigration(id); setMigration(data); setError(null); + // Load schema executions after migration is loaded + if (data) { + loadSchemaExecutions(); + } } catch (err) { const errorMsg = err instanceof Error ? err.message : "Failed to load migration"; @@ -247,6 +266,74 @@ export default function MigrationDetail() { } }; + const loadExecutions = async () => { + if (!id) return; + try { + setExecutionsLoading(true); + const data = await apiClient.getMigrationExecutions(id); + setExecutions(data.executions); + } catch (err) { + // Silently fail executions updates + } finally { + setExecutionsLoading(false); + } + }; + + // Helper function to check if a string is a version number (timestamp format: YYYYMMDDHHMMSS) + const isVersionNumber = (str: string): boolean => { + // Check if it's a 14-digit number (timestamp format) + return /^\d{14}$/.test(str); + }; + + // Helper function to extract base ID from schema-specific ID + const getBaseMigrationID = (migrationID: string): string => { + const parts = migrationID.split("_"); + // Schema-specific format: {schema}_{version}_{name}_{backend}_{connection} + // Base format: {version}_{name}_{backend}_{connection} + // Base migrations start with a version number (timestamp), schema-specific start with schema name + if (parts.length > 0 && !isVersionNumber(parts[0])) { + // First part is not a version number, so it's a schema prefix - remove it + return parts.slice(1).join("_"); + } + return migrationID; + }; + + // Helper function to check if migration ID is schema-specific + const isSchemaSpecific = (migrationID: string): boolean => { + const parts = migrationID.split("_"); + // If first part is not a version number (timestamp), it's schema-specific + return parts.length > 0 && !isVersionNumber(parts[0]); + }; + + const loadSchemaExecutions = async () => { + if (!id || !migration) return; + + try { + // Fetch all migrations and filter for schema-specific executions + const allMigrations = await apiClient.listMigrations({}); + + // Determine the base migration ID + // If this is a schema-specific migration, extract the base ID + // If this is a base migration, use the ID directly + const baseID = getBaseMigrationID(id); + + // Find all schema-specific migrations that match this base + const executions = allMigrations.items.filter((item) => { + // Only include schema-specific migrations + if (!isSchemaSpecific(item.migration_id)) { + return false; + } + // Extract base ID by removing schema prefix + const itemBaseID = getBaseMigrationID(item.migration_id); + return itemBaseID === baseID; + }); + + setSchemaExecutions(executions); + } catch (err) { + // Silently fail + } + }; + // Compute actual applied status from history // A migration is applied if the latest record is not a rollback const isActuallyApplied = useMemo(() => { @@ -373,6 +460,8 @@ export default function MigrationDetail() { loadMigration(); loadStatus(); loadHistory(); // Reload history to get the latest status and applied_at + loadExecutions(); + loadSchemaExecutions(); } catch (err) { const errorMsg = err instanceof Error ? err.message : "Failed to execute migration"; @@ -449,13 +538,18 @@ export default function MigrationDetail() { onConfirm: async () => { setShowConfirmModal(false); try { - const result = await apiClient.rollbackMigration(id); + // Use migration schema if available + const schemas = migration?.schema ? [migration.schema] : []; + const result = await apiClient.rollbackMigration(id, schemas); if (result.success) { - toastService.success("Migration rolled back successfully"); + toastService.success( + result.message || "Migration rolled back successfully", + ); // Reload all data to reflect the rollback loadMigration(); loadStatus(); loadHistory(); + loadExecutions(); } else { toastService.error(`Rollback failed: ${result.message}`); } @@ -890,7 +984,7 @@ export default function MigrationDetail() {
- {history.length > 0 && ( -
-

- Execution History -

+ {/* Executions Table */} +
+

+ Executions ({executions.length}) +

+ {executionsLoading ? ( +
+ Loading executions... +
+ ) : executions.length === 0 ? ( +
+ No executions found for this migration. +
+ ) : (
+ + + - {history.map((record, index) => ( + {executions.map((execution) => ( - + + + + - - - + ))}
- Execution ID + ID - Status + Schema - Applied At + Version - Executed By + Connection - Method + Backend - Error Message + Status + + Applied + + Created At + + Actions
-
- - {record.migration_id} - - {record.migration_id.includes("_rollback") && ( - - Rollback - - )} -
+ 		+ {execution.id} + + {execution.schema || "-"} + + {execution.version} + + {execution.connection} + + {execution.backend} - {record.status === "rolled_back" - ? "Rolled Back" - : record.status} + {execution.status} - {format( - new Date(record.applied_at), - "yyyy-MM-dd HH:mm:ss", - )} - - {record.executed_by || ( - - - )} - - {record.execution_method ? ( - - {record.execution_method} + + {execution.applied ? ( + + Yes ) : ( - - + No )} + {execution.created_at + ? format( + new Date(execution.created_at), + "yyyy-MM-dd HH:mm:ss", + ) + : "-"} + - {record.error_message ? ( -
- {record.error_message} -
- ) : ( - - - )} +
+ {execution.applied && ( + + )} + {!execution.applied && ( + + )} +
+ )} +
+ + {schemaExecutions.length > 0 && ( +
+
+

+ Schema Executions ({schemaExecutions.length}) +

+ +
+ {schemaExecutionsExpanded && ( + <> +

+ This migration has been executed on the following schemas: +

+
+ + + + + + + + + + + + + {(() => { + const startIndex = + (schemaExecutionsPage - 1) * schemaExecutionsPerPage; + const endIndex = startIndex + schemaExecutionsPerPage; + const paginatedExecutions = schemaExecutions.slice( + startIndex, + endIndex, + ); + + return ( + <> + {paginatedExecutions.length === 0 ? ( + + + + ) : ( + paginatedExecutions.map((execution) => ( + + + + + + + + + )) + )} + + ); + })()} + +
+ Migration ID + + Schema + + Status + + Applied + + Applied At + + Actions +
+ No schema executions found +
+ + {execution.migration_id} + + + + {execution.schema || "-"} + + + + {execution.status === "rolled_back" + ? "Rolled Back" + : execution.status || "pending"} + + + + {execution.applied ? "Yes" : "No"} + + + {execution.applied_at + ? format( + new Date(execution.applied_at), + "yyyy-MM-dd HH:mm:ss", + ) + : "-"} + + + View + +
+
+ {schemaExecutions.length > schemaExecutionsPerPage && ( +
+
+ Showing{" "} + {Math.min( + (schemaExecutionsPage - 1) * schemaExecutionsPerPage + + 1, + schemaExecutions.length, + )} + - + {Math.min( + schemaExecutionsPage * schemaExecutionsPerPage, + schemaExecutions.length, + )}{" "} + of {schemaExecutions.length} +
+
+ + +
+
+ + +
+ Page {schemaExecutionsPage} of{" "} + {Math.ceil( + schemaExecutions.length / schemaExecutionsPerPage, + )} +
+ + +
+
+ )} + + )} +
+ )} + + {history.length > 0 && ( +
+

+ Execution History +

+
+ + + + + + + + + + + + + {(() => { + const startIndex = (historyPage - 1) * historyPerPage; + const endIndex = startIndex + historyPerPage; + const paginatedHistory = history.slice( + startIndex, + endIndex, + ); + + return ( + <> + {paginatedHistory.map((record, index) => ( + + + + + + + + + ))} + + ); + })()} + +
+ Execution ID + + Status + + Applied At + + Executed By + + Method + + Error Message +
+
+ + {record.migration_id} + + {record.migration_id.includes("_rollback") && ( + + Rollback + + )} +
+ 		+ + {record.status === "rolled_back" + ? "Rolled Back" + : record.status} + + + {format( + new Date(record.applied_at), + "yyyy-MM-dd HH:mm:ss", + )} + + {record.executed_by || ( + - + )} + + {record.execution_method ? ( + + {record.execution_method} + + ) : ( + - + )} + + {record.error_message ? ( +
+ {record.error_message} +
+ ) : ( + - + )} +
+
+ {history.length > historyPerPage && ( +
+
+ Showing{" "} + {Math.min( + (historyPage - 1) * historyPerPage + 1, + history.length, + )} + -{Math.min(historyPage * historyPerPage, history.length)} of{" "} + {history.length} +
+
+ + +
+
+ + +
+ Page {historyPage} of{" "} + {Math.ceil(history.length / historyPerPage)} +
+ + +
+
+ )}
)} diff --git a/ffm/src/components/MigrationList.tsx b/ffm/src/components/MigrationList.tsx index 7fc07f0..9458b98 100644 --- a/ffm/src/components/MigrationList.tsx +++ b/ffm/src/components/MigrationList.tsx @@ -33,6 +33,7 @@ export default function MigrationList() { const [rollbackModalOpen, setRollbackModalOpen] = useState(false); const [forceRollback, setForceRollback] = useState(false); const [rollingBack, setRollingBack] = useState(false); + const [searchQuery, setSearchQuery] = useState(""); useEffect(() => { loadMigrations(); @@ -201,16 +202,94 @@ export default function MigrationList() { setCurrentPage(1); }; + // Helper function to check if a string is a version number (timestamp format: YYYYMMDDHHMMSS) + const isVersionNumber = (str: string): boolean => { + // Check if it's a 14-digit number (timestamp format) + return /^\d{14}$/.test(str); + }; + + // Helper function to extract base ID from schema-specific ID + const getBaseMigrationID = (migrationID: string): string => { + const parts = migrationID.split("_"); + // Schema-specific format: {schema}_{version}_{name}_{backend}_{connection} + // Base format: {version}_{name}_{backend}_{connection} + // Base migrations start with a version number (timestamp), schema-specific start with schema name + if (parts.length > 0 && !isVersionNumber(parts[0])) { + // First part is not a version number, so it's a schema prefix - remove it + return parts.slice(1).join("_"); + } + return migrationID; + }; + + // Helper function to check if migration ID is schema-specific + const isSchemaSpecific = (migrationID: string): boolean => { + const parts = migrationID.split("_"); + // If first part is not a version number (timestamp), it's schema-specific + return parts.length > 0 && !isVersionNumber(parts[0]); + }; + + // Filter out schema-specific migrations - only show base migrations identified by BfM + const baseMigrations = useMemo(() => { + return migrations.filter((migration) => { + // Only include base migrations (not schema-specific) + return !isSchemaSpecific(migration.migration_id); + }); + }, [migrations]); + + // Calculate schema count for each base migration + const migrationsWithSchemaCount = useMemo(() => { + // Create a map to count schema-specific migrations for each base + const schemaCountMap = new Map(); + + migrations.forEach((migration) => { + if (isSchemaSpecific(migration.migration_id)) { + const baseID = getBaseMigrationID(migration.migration_id); + schemaCountMap.set(baseID, (schemaCountMap.get(baseID) || 0) + 1); + } + }); + + // Add schema count to base migrations + return baseMigrations.map((migration) => ({ + ...migration, + schemaCount: schemaCountMap.get(migration.migration_id) || 0, + })); + }, [baseMigrations, migrations]); + + // Flatten migrations for display (no grouping needed, just base migrations) + const flattenedMigrations = useMemo(() => { + return migrationsWithSchemaCount; + }, [migrationsWithSchemaCount]); + + // Filter migrations based on search query + const filteredMigrations = useMemo(() => { + if (!searchQuery.trim()) { + return flattenedMigrations; + } + const query = searchQuery.toLowerCase(); + return flattenedMigrations.filter((migration) => { + return ( + migration.migration_id.toLowerCase().includes(query) || + migration.version.toLowerCase().includes(query) || + migration.name.toLowerCase().includes(query) || + migration.table.toLowerCase().includes(query) || + migration.backend.toLowerCase().includes(query) || + (migration.connection && + migration.connection.toLowerCase().includes(query)) || + (migration.schema && migration.schema.toLowerCase().includes(query)) + ); + }); + }, [flattenedMigrations, searchQuery]); + // Calculate pagination - const totalPages = Math.ceil(migrations.length / itemsPerPage); + const totalPages = Math.ceil(filteredMigrations.length / itemsPerPage); const startIndex = (currentPage - 1) * itemsPerPage; const endIndex = startIndex + itemsPerPage; - const paginatedMigrations = migrations.slice(startIndex, endIndex); + const paginatedMigrations = filteredMigrations.slice(startIndex, endIndex); - // Reset to page 1 when filters change + // Reset to page 1 when filters or search query change useEffect(() => { setCurrentPage(1); - }, [filters]); + }, [filters, searchQuery]); // Clear selection when filters change useEffect(() => { @@ -233,7 +312,7 @@ export default function MigrationList() { // Handle select all on current page const handleSelectAll = () => { - if (selectedMigrations.size === paginatedMigrations.length) { + if (allPageSelected) { // Deselect all on current page setSelectedMigrations((prev) => { const newSet = new Set(prev); @@ -376,8 +455,11 @@ export default function MigrationList() { // Rollback each selected migration for (const migration of selectedMigrationObjects) { try { + // Use migration schema if available + const schemas = migration.schema ? [migration.schema] : []; const response = await apiClient.rollbackMigration( migration.migration_id, + schemas, ); if (response.success) { @@ -718,9 +800,43 @@ export default function MigrationList() {
)} -
+

Migrations

-
+
+
+ ) => + setSearchQuery(e.target.value) + } + onKeyDown={(e: React.KeyboardEvent) => { + if (e.key === "Enter") { + e.preventDefault(); + setCurrentPage(1); + } else if (e.key === "Escape") { + setSearchQuery(""); + } + }} + className="flex-1 sm:w-64 px-3 py-2 border border-gray-300 rounded text-sm bg-white text-gray-800 focus:outline-none focus:border-bfm-blue focus:ring-2 focus:ring-bfm-blue/20" + /> + + {searchQuery && ( + + )} +