From 6d63718afdadba989c3fb39ddcf9a024d605f3db Mon Sep 17 00:00:00 2001 From: hargitay Date: Sun, 27 Apr 2025 13:08:09 -0700 Subject: [PATCH 1/5] add parallel --- backend/internal/processes/bootstrap.go | 195 ++++++++++++++++++++++-- 1 file changed, 179 insertions(+), 16 deletions(-) diff --git a/backend/internal/processes/bootstrap.go b/backend/internal/processes/bootstrap.go index 9856e5b8..c6260539 100644 --- a/backend/internal/processes/bootstrap.go +++ b/backend/internal/processes/bootstrap.go @@ -2,18 +2,23 @@ package processes import ( "fmt" + "math/rand" + "os" + "path/filepath" + "strings" + "time" + "errors" log "github.com/sirupsen/logrus" "github.com/spf13/afero" // "github.com/unity-sds/unity-cs-manager/marketplace" - "path/filepath" - "strings" "github.com/unity-sds/unity-management-console/backend/internal/application" "github.com/unity-sds/unity-management-console/backend/internal/application/config" "github.com/unity-sds/unity-management-console/backend/internal/aws" "github.com/unity-sds/unity-management-console/backend/internal/database" + "github.com/unity-sds/unity-management-console/backend/internal/terraform" "github.com/unity-sds/unity-management-console/backend/types" ) @@ -97,17 +102,6 @@ func BootstrapEnv(appconf *config.AppConfig) error { return err } - log.Infof("Setting Up Health Status Lambda") - err = installHealthStatusLambda(store, appconf) - if err != nil { - log.WithError(err).Error("Error installing Health Status") - err = store.AddToAudit(application.Bootstrap_Unsuccessful, "test") - if err != nil { - log.WithError(err).Error("Problem writing to auditlog") - } - return err - } - log.Infof("Setting Up Basic API Gateway from Marketplace") err = installBasicAPIGateway(store, appconf) if err != nil { @@ -119,10 +113,10 @@ func BootstrapEnv(appconf *config.AppConfig) error { return err } - log.Infof("Setting Up Unity UI from Marketplace") - err = installUnityUi(store, appconf) + log.Infof("Setting Up Health Status Lambda and Unity UI from Marketplace in parallel") + err = installHealthStatusLambdaAndUnityUi(store, appconf) if err != nil { - log.WithError(err).Error("Error installing unity-portal") + log.WithError(err).Error("Error installing Health Status Lambda and Unity UI") err = store.AddToAudit(application.Bootstrap_Unsuccessful, "test") if err != nil { log.WithError(err).Error("Problem writing to auditlog") @@ -481,3 +475,172 @@ func installUnityUi(store database.Datastore, appConfig *config.AppConfig) error } return nil } + +func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *config.AppConfig) error { + // Get Health Status Lambda metadata and package + var lambdaName, lambdaVersion string + for _, item := range appConfig.MarketplaceItems { + if item.Name == "unity-cs-monitoring-lambda" { + lambdaName = item.Name + lambdaVersion = item.Version + break + } + } + if lambdaName == "" || lambdaVersion == "" { + log.Error("unity-cs-monitoring-lambda not found in MarketplaceItems") + return fmt.Errorf("unity-cs-monitoring-lambda not found in MarketplaceItems") + } + log.Infof("Found marketplace item - Name: %s, Version: %s", lambdaName, lambdaVersion) + + lambdaMetadata, err := FetchMarketplaceMetadata(lambdaName, lambdaVersion, appConfig) + if err != nil { + log.Errorf("Unable to fetch metadata for application: %s, %v", lambdaName, err) + return errors.New("Unable to fetch package") + } + + lambdaLocation, err := FetchPackage(&lambdaMetadata, appConfig) + if err != nil { + log.Errorf("Unable to fetch package for application: %s, %v", lambdaName, err) + return errors.New("Unable to fetch package") + } + + // Get Unity UI metadata and package + var uiName, uiVersion string + for _, item := range appConfig.MarketplaceItems { + if item.Name == "unity-portal" { + uiName = item.Name + uiVersion = item.Version + break + } + } + if uiName == "" || uiVersion == "" { + log.Error("unity-portal not found in MarketplaceItems") + return fmt.Errorf("unity-portal not found in MarketplaceItems") + } + log.Infof("Found marketplace item - Name: %s, Version: %s", uiName, uiVersion) + + uiMetadata, err := FetchMarketplaceMetadata(uiName, uiVersion, appConfig) + if err != nil { + log.Errorf("Unable to fetch metadata for application: %s, %v", uiName, err) + return errors.New("Unable to fetch package") + } + + uiLocation, err := FetchPackage(&uiMetadata, appConfig) + if err != nil { + log.Errorf("Unable to fetch package for application: %s, %v", uiName, err) + return errors.New("Unable to fetch package") + } + + // Generate module names and create application records + rand.Seed(time.Now().UnixNano()) + chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + lambdaRandomChars := make([]byte, 5) + for i, v := range rand.Perm(52)[:5] { + lambdaRandomChars[i] = chars[v] + } + lambdaDeploymentName := fmt.Sprintf("default-%s", lambdaName) + lambdaModuleName := fmt.Sprintf("%s-%s", lambdaDeploymentName, string(lambdaRandomChars)) + + uiRandomChars := make([]byte, 5) + for i, v := range rand.Perm(52)[:5] { + uiRandomChars[i] = chars[v] + } + uiDeploymentName := fmt.Sprintf("default-%s", uiName) + uiModuleName := fmt.Sprintf("%s-%s", uiDeploymentName, string(uiRandomChars)) + + // Create application records in database + lambdaApp := &types.InstalledMarketplaceApplication{ + Name: lambdaName, + Version: lambdaVersion, + DeploymentName: lambdaDeploymentName, + PackageName: lambdaMetadata.Name, + Source: lambdaMetadata.Package, + Status: "STAGED", + TerraformModuleName: lambdaModuleName, + Variables: nil, + } + + uiApp := &types.InstalledMarketplaceApplication{ + Name: uiName, + Version: uiVersion, + DeploymentName: uiDeploymentName, + PackageName: uiMetadata.Name, + Source: uiMetadata.Package, + Status: "STAGED", + TerraformModuleName: uiModuleName, + Variables: nil, + } + + // Store applications in the database + store.StoreInstalledMarketplaceApplication(lambdaApp) + store.StoreInstalledMarketplaceApplication(uiApp) + + // Update status to INSTALLING + lambdaApp.Status = "INSTALLING" + uiApp.Status = "INSTALLING" + store.UpdateInstalledMarketplaceApplication(lambdaApp) + store.UpdateInstalledMarketplaceApplication(uiApp) + + // Run pre-install scripts if needed + err = runPreInstall(appConfig, &lambdaMetadata, lambdaApp) + if err != nil { + return err + } + + err = runPreInstall(appConfig, &uiMetadata, uiApp) + if err != nil { + return err + } + + // Add both applications to the Terraform stack + terraform.AddApplicationToStack(appConfig, lambdaLocation, &lambdaMetadata, lambdaApp, store) + terraform.AddApplicationToStack(appConfig, uiLocation, &uiMetadata, uiApp, store) + + // Run a single Terraform apply for both modules + executor := &terraform.RealTerraformExecutor{} + logDir := filepath.Join(appConfig.Workdir, "install_logs") + if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create install_logs directory: %w", err) + } + + logfile := filepath.Join(logDir, "batch_install_log") + err = terraform.RunTerraformLogOutToFile(appConfig, logfile, executor, "") + if err != nil { + lambdaApp.Status = "FAILED" + uiApp.Status = "FAILED" + store.UpdateInstalledMarketplaceApplication(lambdaApp) + store.UpdateInstalledMarketplaceApplication(uiApp) + return err + } + + // Update status and run post-install + lambdaApp.Status = "INSTALLED" + uiApp.Status = "INSTALLED" + store.UpdateInstalledMarketplaceApplication(lambdaApp) + store.UpdateInstalledMarketplaceApplication(uiApp) + + // Run post-install scripts + err = runPostInstallNew(appConfig, &lambdaMetadata, lambdaApp) + if err != nil { + lambdaApp.Status = "POSTINSTALL FAILED" + store.UpdateInstalledMarketplaceApplication(lambdaApp) + return err + } + + err = runPostInstallNew(appConfig, &uiMetadata, uiApp) + if err != nil { + uiApp.Status = "POSTINSTALL FAILED" + store.UpdateInstalledMarketplaceApplication(uiApp) + return err + } + + // Update final status + lambdaApp.Status = "COMPLETE" + uiApp.Status = "COMPLETE" + store.UpdateInstalledMarketplaceApplication(lambdaApp) + store.UpdateInstalledMarketplaceApplication(uiApp) + + fetchAllApplications(store) + return nil +} From 85abd8552b032383a93ef0183fb11e96be25bc30 Mon Sep 17 00:00:00 2001 From: hargitay Date: Sun, 27 Apr 2025 15:01:16 -0700 Subject: [PATCH 2/5] add parallel --- backend/internal/processes/bootstrap.go | 247 +++--------------------- backend/internal/processes/installer.go | 131 +++++++++++++ 2 files changed, 153 insertions(+), 225 deletions(-) diff --git a/backend/internal/processes/bootstrap.go b/backend/internal/processes/bootstrap.go index c6260539..1811e600 100644 --- a/backend/internal/processes/bootstrap.go +++ b/backend/internal/processes/bootstrap.go @@ -178,7 +178,7 @@ required_providers { } } backend "s3" { - dynamodb_table = "%s-%s-terraform-state" + use_lockfile = true } } @@ -391,93 +391,8 @@ func installUnityCloudEnv(store database.Datastore, appConfig *config.AppConfig) return nil } -func installHealthStatusLambda(store database.Datastore, appConfig *config.AppConfig) error { - - // Find the marketplace item for the health status lambda - var name, version string - for _, item := range appConfig.MarketplaceItems { - if item.Name == "unity-cs-monitoring-lambda" { - name = item.Name - version = item.Version - break - } - } - - // Print the name and version - log.Infof("Found marketplace item - Name: %s, Version: %s", name, version) - - // If the item wasn't found, log an error and return - if name == "" || version == "" { - log.Error("unity-cs-monitoring-lambda not found in MarketplaceItems") - return fmt.Errorf("unity-cs-monitoring-lambda not found in MarketplaceItems") - } - - // applications := marketplace.Install_Applications{ - // Name: name, - // Version: version, - // Variables: nil, - // Displayname: fmt.Sprintf("%s-%s", appConfig.InstallPrefix, name), - // } - // install := marketplace.Install{ - // Applications: &applications, - // DeploymentName: "Unity Health Status Lambda", - // } - - installParams := types.ApplicationInstallParams{ - Name: name, - Version: version, - Variables: nil, - DisplayName: "Unity Health Status Lambda", - DeploymentName: fmt.Sprintf("default-%s", name), - } - - err := TriggerInstall(store, &installParams, appConfig, true) - if err != nil { - log.WithError(err).Error("Issue installing Unity Health Status Lambda") - return err - } - return nil -} - -func installUnityUi(store database.Datastore, appConfig *config.AppConfig) error { - - // Find the marketplace item for the unity-portal - var name, version string - for _, item := range appConfig.MarketplaceItems { - if item.Name == "unity-portal" { - name = item.Name - version = item.Version - break - } - } - - // Print the name and version - log.Infof("Found marketplace item - Name: %s, Version: %s", name, version) - - // If the item wasn't found, log an error and return - if name == "" || version == "" { - log.Error("unity-portal not found in MarketplaceItems") - return fmt.Errorf("unity-portal not found in MarketplaceItems") - } - - installParams := types.ApplicationInstallParams{ - Name: name, - Version: version, - Variables: nil, - DisplayName: "Unity Navbar UI", - DeploymentName: fmt.Sprintf("default-%s", name), - } - - err := TriggerInstall(store, &installParams, appConfig, true) - if err != nil { - log.WithError(err).Error("Issue installing Unity Navbar UI") - return err - } - return nil -} - func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *config.AppConfig) error { - // Get Health Status Lambda metadata and package + // Find the marketplace item for health status lambda var lambdaName, lambdaVersion string for _, item := range appConfig.MarketplaceItems { if item.Name == "unity-cs-monitoring-lambda" { @@ -490,21 +405,8 @@ func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *co log.Error("unity-cs-monitoring-lambda not found in MarketplaceItems") return fmt.Errorf("unity-cs-monitoring-lambda not found in MarketplaceItems") } - log.Infof("Found marketplace item - Name: %s, Version: %s", lambdaName, lambdaVersion) - - lambdaMetadata, err := FetchMarketplaceMetadata(lambdaName, lambdaVersion, appConfig) - if err != nil { - log.Errorf("Unable to fetch metadata for application: %s, %v", lambdaName, err) - return errors.New("Unable to fetch package") - } - - lambdaLocation, err := FetchPackage(&lambdaMetadata, appConfig) - if err != nil { - log.Errorf("Unable to fetch package for application: %s, %v", lambdaName, err) - return errors.New("Unable to fetch package") - } - - // Get Unity UI metadata and package + + // Find the marketplace item for unity-portal var uiName, uiVersion string for _, item := range appConfig.MarketplaceItems { if item.Name == "unity-portal" { @@ -517,130 +419,25 @@ func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *co log.Error("unity-portal not found in MarketplaceItems") return fmt.Errorf("unity-portal not found in MarketplaceItems") } - log.Infof("Found marketplace item - Name: %s, Version: %s", uiName, uiVersion) - - uiMetadata, err := FetchMarketplaceMetadata(uiName, uiVersion, appConfig) - if err != nil { - log.Errorf("Unable to fetch metadata for application: %s, %v", uiName, err) - return errors.New("Unable to fetch package") - } - - uiLocation, err := FetchPackage(&uiMetadata, appConfig) - if err != nil { - log.Errorf("Unable to fetch package for application: %s, %v", uiName, err) - return errors.New("Unable to fetch package") - } - - // Generate module names and create application records - rand.Seed(time.Now().UnixNano()) - chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - - lambdaRandomChars := make([]byte, 5) - for i, v := range rand.Perm(52)[:5] { - lambdaRandomChars[i] = chars[v] - } - lambdaDeploymentName := fmt.Sprintf("default-%s", lambdaName) - lambdaModuleName := fmt.Sprintf("%s-%s", lambdaDeploymentName, string(lambdaRandomChars)) - uiRandomChars := make([]byte, 5) - for i, v := range rand.Perm(52)[:5] { - uiRandomChars[i] = chars[v] - } - uiDeploymentName := fmt.Sprintf("default-%s", uiName) - uiModuleName := fmt.Sprintf("%s-%s", uiDeploymentName, string(uiRandomChars)) - - // Create application records in database - lambdaApp := &types.InstalledMarketplaceApplication{ - Name: lambdaName, - Version: lambdaVersion, - DeploymentName: lambdaDeploymentName, - PackageName: lambdaMetadata.Name, - Source: lambdaMetadata.Package, - Status: "STAGED", - TerraformModuleName: lambdaModuleName, - Variables: nil, - } - - uiApp := &types.InstalledMarketplaceApplication{ - Name: uiName, - Version: uiVersion, - DeploymentName: uiDeploymentName, - PackageName: uiMetadata.Name, - Source: uiMetadata.Package, - Status: "STAGED", - TerraformModuleName: uiModuleName, - Variables: nil, - } - - // Store applications in the database - store.StoreInstalledMarketplaceApplication(lambdaApp) - store.StoreInstalledMarketplaceApplication(uiApp) - - // Update status to INSTALLING - lambdaApp.Status = "INSTALLING" - uiApp.Status = "INSTALLING" - store.UpdateInstalledMarketplaceApplication(lambdaApp) - store.UpdateInstalledMarketplaceApplication(uiApp) - - // Run pre-install scripts if needed - err = runPreInstall(appConfig, &lambdaMetadata, lambdaApp) - if err != nil { - return err + // Create installation parameters for both applications + params := []*types.ApplicationInstallParams{ + { + Name: lambdaName, + Version: lambdaVersion, + Variables: nil, + DisplayName: "Unity Health Status Lambda", + DeploymentName: fmt.Sprintf("default-%s", lambdaName), + }, + { + Name: uiName, + Version: uiVersion, + Variables: nil, + DisplayName: "Unity Navbar UI", + DeploymentName: fmt.Sprintf("default-%s", uiName), + }, } - err = runPreInstall(appConfig, &uiMetadata, uiApp) - if err != nil { - return err - } - - // Add both applications to the Terraform stack - terraform.AddApplicationToStack(appConfig, lambdaLocation, &lambdaMetadata, lambdaApp, store) - terraform.AddApplicationToStack(appConfig, uiLocation, &uiMetadata, uiApp, store) - - // Run a single Terraform apply for both modules - executor := &terraform.RealTerraformExecutor{} - logDir := filepath.Join(appConfig.Workdir, "install_logs") - if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create install_logs directory: %w", err) - } - - logfile := filepath.Join(logDir, "batch_install_log") - err = terraform.RunTerraformLogOutToFile(appConfig, logfile, executor, "") - if err != nil { - lambdaApp.Status = "FAILED" - uiApp.Status = "FAILED" - store.UpdateInstalledMarketplaceApplication(lambdaApp) - store.UpdateInstalledMarketplaceApplication(uiApp) - return err - } - - // Update status and run post-install - lambdaApp.Status = "INSTALLED" - uiApp.Status = "INSTALLED" - store.UpdateInstalledMarketplaceApplication(lambdaApp) - store.UpdateInstalledMarketplaceApplication(uiApp) - - // Run post-install scripts - err = runPostInstallNew(appConfig, &lambdaMetadata, lambdaApp) - if err != nil { - lambdaApp.Status = "POSTINSTALL FAILED" - store.UpdateInstalledMarketplaceApplication(lambdaApp) - return err - } - - err = runPostInstallNew(appConfig, &uiMetadata, uiApp) - if err != nil { - uiApp.Status = "POSTINSTALL FAILED" - store.UpdateInstalledMarketplaceApplication(uiApp) - return err - } - - // Update final status - lambdaApp.Status = "COMPLETE" - uiApp.Status = "COMPLETE" - store.UpdateInstalledMarketplaceApplication(lambdaApp) - store.UpdateInstalledMarketplaceApplication(uiApp) - - fetchAllApplications(store) - return nil + // Install both applications in a single batch operation + return BatchTriggerInstall(store, params, appConfig) } diff --git a/backend/internal/processes/installer.go b/backend/internal/processes/installer.go index b7d17c0a..1e3427aa 100644 --- a/backend/internal/processes/installer.go +++ b/backend/internal/processes/installer.go @@ -267,6 +267,137 @@ func TriggerInstall(store database.Datastore, applicationInstallParams *types.Ap return InstallMarketplaceApplication(conf, location, applicationInstallParams, &metadata, store, sync) } +// BatchTriggerInstall handles the installation of multiple applications in a single Terraform operation +// to avoid state locking issues when installing multiple modules in parallel +func BatchTriggerInstall(store database.Datastore, params []*types.ApplicationInstallParams, conf *config.AppConfig) error { + if len(params) == 0 { + return nil + } + + applications := make([]*types.InstalledMarketplaceApplication, 0, len(params)) + metadatas := make([]*marketplace.MarketplaceMetadata, 0, len(params)) + locations := make([]string, 0, len(params)) + + // First pass: validate and prepare all applications + for _, param := range params { + // Check if application is already installed + existingApplication, err := store.GetInstalledMarketplaceApplication(param.Name, param.DeploymentName) + if err != nil { + log.WithError(err).Error("Error finding applications") + return errors.New("Unable to search application list") + } + + if existingApplication != nil && existingApplication.Status != "UNINSTALLED" { + errMsg := fmt.Sprintf("Application with name %s already exists. Status: %s", param.Name, existingApplication.Status) + return errors.New(errMsg) + } + + // Fetch metadata and package + metadata, err := FetchMarketplaceMetadata(param.Name, param.Version, conf) + if err != nil { + log.Errorf("Unable to fetch metadata for application: %s, %v", param.Name, err) + return errors.New("Unable to fetch package") + } + + location, err := FetchPackage(&metadata, conf) + if err != nil { + log.Errorf("Unable to fetch package for application: %s, %v", param.Name, err) + return errors.New("Unable to fetch package") + } + + // Generate unique module name + rand.Seed(time.Now().UnixNano()) + chars := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + randomChars := make([]byte, 5) + for i, v := range rand.Perm(52)[:5] { + randomChars[i] = chars[v] + } + terraformModuleName := fmt.Sprintf("%s-%s", param.DeploymentName, string(randomChars)) + + // Create application record + application := &types.InstalledMarketplaceApplication{ + Name: param.Name, + Version: param.Version, + DeploymentName: param.DeploymentName, + PackageName: metadata.Name, + Source: metadata.Package, + Status: "STAGED", + TerraformModuleName: terraformModuleName, + Variables: param.Variables, + } + + // Store application record + store.StoreInstalledMarketplaceApplication(application) + + // Update status to INSTALLING + application.Status = "INSTALLING" + store.UpdateInstalledMarketplaceApplication(application) + + // Add to our tracking lists + applications = append(applications, application) + metadatas = append(metadatas, &metadata) + locations = append(locations, location) + } + + // Second pass: prepare all Terraform configurations + for i, application := range applications { + // Run pre-install script if needed + err := runPreInstall(conf, metadatas[i], application) + if err != nil { + log.WithError(err).Errorf("Error running pre-install for %s", application.Name) + application.Status = "FAILED" + store.UpdateInstalledMarketplaceApplication(application) + return err + } + + // Add application to Terraform stack + err = terraform.AddApplicationToStack(conf, locations[i], metadatas[i], application, store) + if err != nil { + log.WithError(err).Errorf("Error adding application to stack: %s", application.Name) + application.Status = "FAILED" + store.UpdateInstalledMarketplaceApplication(application) + return err + } + } + + // Third pass: run a single Terraform apply for all modules + executor := &terraform.RealTerraformExecutor{} + logDir := filepath.Join(conf.Workdir, "install_logs") + if err := os.MkdirAll(logDir, 0755); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create install_logs directory: %w", err) + } + + logfile := filepath.Join(logDir, "batch_install_log") + err := terraform.RunTerraformLogOutToFile(conf, logfile, executor, "") + if err != nil { + // Mark all applications as failed + for _, application := range applications { + application.Status = "FAILED" + store.UpdateInstalledMarketplaceApplication(application) + } + return err + } + + // Fourth pass: run post-install scripts and update status + for i, application := range applications { + application.Status = "INSTALLED" + store.UpdateInstalledMarketplaceApplication(application) + + err := runPostInstallNew(conf, metadatas[i], application) + if err != nil { + log.WithError(err).Errorf("Error running post-install for %s", application.Name) + application.Status = "POSTINSTALL FAILED" + store.UpdateInstalledMarketplaceApplication(application) + return err + } + + application.Status = "COMPLETE" + store.UpdateInstalledMarketplaceApplication(application) + } + + return nil +} + func TriggerUninstall(wsManager *websocket.WebSocketManager, userid string, store database.Datastore, received *marketplace.Uninstall, conf *config.AppConfig) error { if received.All == true { return UninstallAll(conf, wsManager, userid, received) From 86df6b13ba44521ea26008d5c4983f0616316872 Mon Sep 17 00:00:00 2001 From: hargitay Date: Sun, 27 Apr 2025 15:02:49 -0700 Subject: [PATCH 3/5] add parallel --- backend/internal/processes/bootstrap.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/backend/internal/processes/bootstrap.go b/backend/internal/processes/bootstrap.go index 1811e600..528522c9 100644 --- a/backend/internal/processes/bootstrap.go +++ b/backend/internal/processes/bootstrap.go @@ -2,12 +2,8 @@ package processes import ( "fmt" - "math/rand" - "os" "path/filepath" "strings" - "time" - "errors" log "github.com/sirupsen/logrus" "github.com/spf13/afero" @@ -18,7 +14,6 @@ import ( "github.com/unity-sds/unity-management-console/backend/internal/application/config" "github.com/unity-sds/unity-management-console/backend/internal/aws" "github.com/unity-sds/unity-management-console/backend/internal/database" - "github.com/unity-sds/unity-management-console/backend/internal/terraform" "github.com/unity-sds/unity-management-console/backend/types" ) From d8c226f66805995dc4da09aaebe9f975bb39b308 Mon Sep 17 00:00:00 2001 From: hargitay Date: Sun, 27 Apr 2025 15:04:33 -0700 Subject: [PATCH 4/5] add parallel --- backend/internal/processes/bootstrap.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/internal/processes/bootstrap.go b/backend/internal/processes/bootstrap.go index 528522c9..37dc5169 100644 --- a/backend/internal/processes/bootstrap.go +++ b/backend/internal/processes/bootstrap.go @@ -173,13 +173,14 @@ required_providers { } } backend "s3" { - use_lockfile = true + use_lockfile = true } } provider "aws" { region = "us-west-2" -}`, appConfig.Project, appConfig.Venue) +} +`) err := fs.MkdirAll(filepath.Join(appConfig.Workdir, "workspace"), 0755) if err != nil { From 875048a79ff479f84b2a9617e65d34e792633bb5 Mon Sep 17 00:00:00 2001 From: hargitay Date: Thu, 1 May 2025 14:16:58 -0700 Subject: [PATCH 5/5] add parallel for proxy and apigateway --- backend/internal/processes/bootstrap.go | 213 ++++++++++-------------- 1 file changed, 86 insertions(+), 127 deletions(-) diff --git a/backend/internal/processes/bootstrap.go b/backend/internal/processes/bootstrap.go index 37dc5169..def82821 100644 --- a/backend/internal/processes/bootstrap.go +++ b/backend/internal/processes/bootstrap.go @@ -86,21 +86,10 @@ func BootstrapEnv(appconf *config.AppConfig) error { // log.WithError(err).Error("Problem updating ssm config") //} - log.Infof("Setting Up HTTPD Gateway from Marketplace") - err = installGateway(store, appconf) + log.Infof("Setting Up HTTPD Gateway and API Gateway from Marketplace in parallel") + err = installGatewayAndApiGateway(store, appconf) if err != nil { - log.WithError(err).Error("Error installing HTTPD Gateway") - err = store.AddToAudit(application.Bootstrap_Unsuccessful, "test") - if err != nil { - log.WithError(err).Error("Problem writing to auditlog") - } - return err - } - - log.Infof("Setting Up Basic API Gateway from Marketplace") - err = installBasicAPIGateway(store, appconf) - if err != nil { - log.WithError(err).Error("Error installing API Gateway") + log.WithError(err).Error("Error installing Gateways") err = store.AddToAudit(application.Bootstrap_Unsuccessful, "test") if err != nil { log.WithError(err).Error("Problem writing to auditlog") @@ -223,89 +212,110 @@ func storeDefaultSSMParameters(appConfig *config.AppConfig, store database.Datas return nil } -func installGateway(store database.Datastore, appConfig *config.AppConfig) error { +func installGatewayAndApiGateway(store database.Datastore, appConfig *config.AppConfig) error { // Find the marketplace item for unity-proxy - var name, version string + var proxyName, proxyVersion string for _, item := range appConfig.MarketplaceItems { if item.Name == "unity-proxy" { - name = item.Name - version = item.Version + proxyName = item.Name + proxyVersion = item.Version break } } - - // Print the name and version - log.Infof("Found marketplace item - Name: %s, Version: %s", name, version) - - // If the item wasn't found, log an error and return - if name == "" || version == "" { + if proxyName == "" || proxyVersion == "" { log.Error("unity-proxy not found in MarketplaceItems") return fmt.Errorf("unity-proxy not found in MarketplaceItems") } - - simplevars := make(map[string]string) - simplevars["mgmt_dns"] = appConfig.ConsoleHost - // variables := marketplace.Install_Variables{Values: simplevars} - // applications := marketplace.Install_Applications{ - // Name: name, - // Version: version, - // Variables: &variables, - // Displayname: fmt.Sprintf("%s-%s", appConfig.InstallPrefix, name), - // } - // install := marketplace.Install{ - // Applications: &applications, - // DeploymentName: "Core Mgmt Gateway", - // } - - installParams := types.ApplicationInstallParams{ - Name: name, - Version: version, - Variables: simplevars, - DisplayName: "Unity Health Status Lambda", - DeploymentName: fmt.Sprintf("default-%s", name), - } - err := TriggerInstall(store, &installParams, appConfig, true) - if err != nil { - log.WithError(err).Error("Issue installing Mgmt Gateway") - return err - } - return nil -} - -func installBasicAPIGateway(store database.Datastore, appConfig *config.AppConfig) error { + // Find the marketplace item for unity-apigateway - var name, version string + var apiName, apiVersion string for _, item := range appConfig.MarketplaceItems { if item.Name == "unity-apigateway" { - name = item.Name - version = item.Version + apiName = item.Name + apiVersion = item.Version break } } - - // Print the name and version - log.Infof("Found marketplace item - Name: %s, Version: %s", name, version) - - // If the item wasn't found, log an error and return - if name == "" || version == "" { + if apiName == "" || apiVersion == "" { log.Error("unity-apigateway not found in MarketplaceItems") return fmt.Errorf("unity-apigateway not found in MarketplaceItems") } - - installParams := types.ApplicationInstallParams{ - Name: name, - Version: version, - Variables: nil, - DisplayName: "Core API Gateway", - DeploymentName: fmt.Sprintf("default-%s", name), + + // Set up variables for unity-proxy + proxyVars := make(map[string]string) + proxyVars["mgmt_dns"] = appConfig.ConsoleHost + + // Create installation parameters for both applications + params := []*types.ApplicationInstallParams{ + { + Name: proxyName, + Version: proxyVersion, + Variables: proxyVars, + DisplayName: "Core Mgmt Gateway", + DeploymentName: fmt.Sprintf("default-%s", proxyName), + }, + { + Name: apiName, + Version: apiVersion, + Variables: nil, + DisplayName: "Core API Gateway", + DeploymentName: fmt.Sprintf("default-%s", apiName), + }, } + + // Install both applications in a single batch operation + return BatchTriggerInstall(store, params, appConfig) +} - err := TriggerInstall(store, &installParams, appConfig, true) - if err != nil { - log.WithError(err).Error("Issue installing API Gateway") - return err +func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *config.AppConfig) error { + // Find the marketplace item for health status lambda + var lambdaName, lambdaVersion string + for _, item := range appConfig.MarketplaceItems { + if item.Name == "unity-cs-monitoring-lambda" { + lambdaName = item.Name + lambdaVersion = item.Version + break + } } - return nil + if lambdaName == "" || lambdaVersion == "" { + log.Error("unity-cs-monitoring-lambda not found in MarketplaceItems") + return fmt.Errorf("unity-cs-monitoring-lambda not found in MarketplaceItems") + } + + // Find the marketplace item for unity-portal + var uiName, uiVersion string + for _, item := range appConfig.MarketplaceItems { + if item.Name == "unity-portal" { + uiName = item.Name + uiVersion = item.Version + break + } + } + if uiName == "" || uiVersion == "" { + log.Error("unity-portal not found in MarketplaceItems") + return fmt.Errorf("unity-portal not found in MarketplaceItems") + } + + // Create installation parameters for both applications + params := []*types.ApplicationInstallParams{ + { + Name: lambdaName, + Version: lambdaVersion, + Variables: nil, + DisplayName: "Unity Health Status Lambda", + DeploymentName: fmt.Sprintf("default-%s", lambdaName), + }, + { + Name: uiName, + Version: uiVersion, + Variables: nil, + DisplayName: "Unity Navbar UI", + DeploymentName: fmt.Sprintf("default-%s", uiName), + }, + } + + // Install both applications in a single batch operation + return BatchTriggerInstall(store, params, appConfig) } func installUnityCloudEnv(store database.Datastore, appConfig *config.AppConfig) error { @@ -386,54 +396,3 @@ func installUnityCloudEnv(store database.Datastore, appConfig *config.AppConfig) } return nil } - -func installHealthStatusLambdaAndUnityUi(store database.Datastore, appConfig *config.AppConfig) error { - // Find the marketplace item for health status lambda - var lambdaName, lambdaVersion string - for _, item := range appConfig.MarketplaceItems { - if item.Name == "unity-cs-monitoring-lambda" { - lambdaName = item.Name - lambdaVersion = item.Version - break - } - } - if lambdaName == "" || lambdaVersion == "" { - log.Error("unity-cs-monitoring-lambda not found in MarketplaceItems") - return fmt.Errorf("unity-cs-monitoring-lambda not found in MarketplaceItems") - } - - // Find the marketplace item for unity-portal - var uiName, uiVersion string - for _, item := range appConfig.MarketplaceItems { - if item.Name == "unity-portal" { - uiName = item.Name - uiVersion = item.Version - break - } - } - if uiName == "" || uiVersion == "" { - log.Error("unity-portal not found in MarketplaceItems") - return fmt.Errorf("unity-portal not found in MarketplaceItems") - } - - // Create installation parameters for both applications - params := []*types.ApplicationInstallParams{ - { - Name: lambdaName, - Version: lambdaVersion, - Variables: nil, - DisplayName: "Unity Health Status Lambda", - DeploymentName: fmt.Sprintf("default-%s", lambdaName), - }, - { - Name: uiName, - Version: uiVersion, - Variables: nil, - DisplayName: "Unity Navbar UI", - DeploymentName: fmt.Sprintf("default-%s", uiName), - }, - } - - // Install both applications in a single batch operation - return BatchTriggerInstall(store, params, appConfig) -}