From 6a6f36bbeb6ce4abbfbf38102c0259175d3d8b5c Mon Sep 17 00:00:00 2001 From: Piotr Kontek Date: Thu, 11 Dec 2025 20:47:00 +0100 Subject: [PATCH] Enhance fabric setup and connection string generation - Enable folders in fabric setup. - Support Warehouse item type in connection string generation and item handling. - Update subprocess calls to use sys.executable for better compatibility. --- automation/scripts/fabric_setup.py | 28 +++++++++-------- .../scripts/generate_connection_string.py | 31 ++++++++++--------- .../locale/locale_build_parameter_file.py | 2 +- .../scripts/locale/locale_maintain_feature.py | 2 +- automation/scripts/locale/locale_release.py | 2 +- automation/scripts/locale/locale_setup.py | 2 +- .../scripts/modules/fabric_cli_functions.py | 27 +++++++++++++++- .../scripts/utils_build_parameter_file.py | 8 ++--- 8 files changed, 65 insertions(+), 37 deletions(-) diff --git a/automation/scripts/fabric_setup.py b/automation/scripts/fabric_setup.py index b91d4bd..8d9e7b0 100644 --- a/automation/scripts/fabric_setup.py +++ b/automation/scripts/fabric_setup.py @@ -33,6 +33,7 @@ # Authenticate fabcli.run_command("config set encryption_fallback_enabled true") +fabcli.run_command("config set folder_listing_enabled true") fabcli.run_command(f"auth login -u {client_id} -p {client_secret} --tenant {tenant_id}") # Load JSON environment files (main and environment specific) and merge @@ -171,7 +172,7 @@ print_item_header = True for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: has_item_connections = True if not item.get("skip_item_creation", False): @@ -179,11 +180,11 @@ print(f" • Creating workspace items:") print_item_header = False - misc.print_info(f" ◦ {item_type}: {item.get("item_name")}...", end="") - - if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'): - fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'") - item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}", retry_count=2) + item_folder = f'{item.get("item_folder")}/' if item.get("item_folder") else "" + misc.print_info(f" ◦ {item_type}: {item_folder}{item.get("item_name")}...", end="") + if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'): + fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'") + item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}", retry_count=2) if item_type in {"Lakehouse"}: # Wait until SQL endpoint provisioning completes; treat missing metadata as still provisioning @@ -200,14 +201,14 @@ break print(".", end="") time.sleep(2) - item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") + item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}") if item["item_metadata"]: misc.print_success(" ✔") else: misc.print_error(" ✖ Failed!") else: - item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") + item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}") misc.print_warning(f" ⚠ Already exists") if layer_definition.get("private_endpoints"): @@ -243,7 +244,7 @@ git_settings = env_definition.get("generic").get("git_settings") git_settings["gitProviderDetails"]["directoryName"] = layer_definition.get("git_directoryName") - if git_connection.get("id"): + if git_connection.get("id"): connect_response = fabcli.connect_workspace_to_git(workspace_id, git_settings) if connect_response: init_response = fabcli.initialize_git_connection(workspace_id) @@ -269,7 +270,7 @@ for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer, environment=environment) item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}") @@ -278,11 +279,12 @@ if item["item_metadata"]: server = ( item.get("item_metadata").get("properties").get("sqlEndpointProperties").get("connectionString") if item_type == "Lakehouse" else - item.get("item_metadata").get("properties").get("serverFqdn") + item.get("item_metadata").get("properties").get("serverFqdn") if item_type == "SQLDatabase" else + item.get("item_metadata").get("properties").get("connectionString") ) database = ( - item.get("item_name") if item_type == "Lakehouse" else + item.get("item_name") if item_type in ("Lakehouse","Warehouse") else item.get("item_metadata").get("properties").get("databaseName") ) @@ -362,7 +364,7 @@ if layer_definition.get("items"): for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer, environment=environment) misc.print_info(f" • Deleting connection '{connection_name}'... ", bold=False, end="") if fabcli.run_command(f"exists .connections/{connection_name}.Connection").replace("*", "").strip().lower() == "true": diff --git a/automation/scripts/generate_connection_string.py b/automation/scripts/generate_connection_string.py index 32503e0..2313f06 100644 --- a/automation/scripts/generate_connection_string.py +++ b/automation/scripts/generate_connection_string.py @@ -35,21 +35,22 @@ solution_name = env_definition.get("name") workspace_name = solution_name.format(layer=layer, environment=environment) -workspace_name_escaped = workspace_name.replace("/", "\\/") -sqldb_item = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{database}.SQLDatabase") - -# Example: You may want to adjust the server/database names per environment -server = sqldb_item.get('properties').get('serverFqdn') -database = sqldb_item.get("properties").get("databaseName") - -connection_string = ( - f"Server={server};" - f"Database={database};" - f"Authentication=Active Directory Service Principal;" - f"User Id={client_id};" - f"Password={client_secret};" - f"Encrypt=True;" - f"Connection Timeout=60;" + +item_type = None +for env_item_type in env_definition.get("layers").get(layer).get("items"): + if env_item_type in ["Warehouse","SQLDatabase"]: + for item in env_definition.get("layers").get(layer).get("items").get(env_item_type): + item_name = item.get("item_name") + if item_name == database: + item_type = env_item_type + break + +connection_string = fabcli.generate_connection_string( + workspace_name=workspace_name, + item_type=item_type, + database=database, + client_id=client_id, + client_secret=client_secret ) with open(args.output_file, "w") as f: diff --git a/automation/scripts/locale/locale_build_parameter_file.py b/automation/scripts/locale/locale_build_parameter_file.py index 4df07f2..82a1db0 100644 --- a/automation/scripts/locale/locale_build_parameter_file.py +++ b/automation/scripts/locale/locale_build_parameter_file.py @@ -22,7 +22,7 @@ "--client_secret", env_credentials.get("client_secret") ] -process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') +process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') # Print the output line by line as it is generated for line in process.stdout: diff --git a/automation/scripts/locale/locale_maintain_feature.py b/automation/scripts/locale/locale_maintain_feature.py index 9527708..eb3b1ad 100644 --- a/automation/scripts/locale/locale_maintain_feature.py +++ b/automation/scripts/locale/locale_maintain_feature.py @@ -23,7 +23,7 @@ "--action", action, "--branch_name", branch_name] -process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') +process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') # Print the output line by line as it is generated for line in process.stdout: diff --git a/automation/scripts/locale/locale_release.py b/automation/scripts/locale/locale_release.py index 997242f..bc5afd0 100644 --- a/automation/scripts/locale/locale_release.py +++ b/automation/scripts/locale/locale_release.py @@ -28,7 +28,7 @@ "--client_secret", env_credentials.get("client_secret") ] - process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8') + process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8') # Print output live for line in process.stdout: diff --git a/automation/scripts/locale/locale_setup.py b/automation/scripts/locale/locale_setup.py index e74f115..f2034fa 100644 --- a/automation/scripts/locale/locale_setup.py +++ b/automation/scripts/locale/locale_setup.py @@ -26,7 +26,7 @@ "--github_pat", env_credentials.get("github_pat") ] - process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') + process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8') # Print the output line by line as it is generated for line in process.stdout: diff --git a/automation/scripts/modules/fabric_cli_functions.py b/automation/scripts/modules/fabric_cli_functions.py index 83381e5..192c0f2 100644 --- a/automation/scripts/modules/fabric_cli_functions.py +++ b/automation/scripts/modules/fabric_cli_functions.py @@ -251,4 +251,29 @@ def poll_operation_status(operation_id): else: return None - return None # Operation timed out or failed \ No newline at end of file + return None # Operation timed out or failed + +def generate_connection_string(workspace_name, item_type, database, client_id, client_secret): + print(f"Generating connection string for {item_type} '{database}' in workspace '{workspace_name}'...") + workspace_name_escaped = workspace_name.replace("/", "\\/") + sqldb_item = get_item(f"/{workspace_name_escaped}.Workspace/{database}.{item_type}") + print(sqldb_item) + if item_type == "SQLDatabase": + server = sqldb_item.get('properties').get('serverFqdn') + database = sqldb_item.get("properties").get("databaseName") + elif item_type == "Lakehouse": + server = sqldb_item.get("properties").get('sqlEndpointProperties').get('connectionString') + else: + server = sqldb_item.get('properties').get('connectionString') + + connection_string = ( + f"Server={server};" + f"Database={database};" + f"Authentication=Active Directory Service Principal;" + f"User Id={client_id};" + f"Password={client_secret};" + f"Encrypt=True;" + f"Connection Timeout=60;" + ) + + return connection_string \ No newline at end of file diff --git a/automation/scripts/utils_build_parameter_file.py b/automation/scripts/utils_build_parameter_file.py index 7b1f7a7..31b5771 100644 --- a/automation/scripts/utils_build_parameter_file.py +++ b/automation/scripts/utils_build_parameter_file.py @@ -74,11 +74,11 @@ "type": item.get("type") } - if item.get("type") in {"Lakehouse", "SQLDatabase"}: + if item.get("type") in {"Lakehouse", "SQLDatabase", "Warehouse"}: item_details = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('displayName')}.{item.get('type')}", retry_count=2) fabric_item.update({ - "connectionString": item_details.get("properties").get("connectionString") if item.get("type") == "SQLDatabase" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") , - "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else None, + "connectionString": item_details.get("properties").get("connectionString") if item.get("type") != "Lakehouse" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") , + "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else item_details.get("displayName") if item.get("type") == "Warehouse" else None, "serverFqdn": item_details.get("properties").get("serverFqdn") if item.get("type") == "SQLDatabase" else None, "sqlEndpointId": item_details.get("properties").get("sqlEndpointProperties").get("id") if item.get("type") == "Lakehouse" else None, }) @@ -89,7 +89,7 @@ if layer_definition.get("items"): for item_type, items in layer_definition.get("items").items(): for item in items: - if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}: + if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}: connection_name = item.get("connection_name").format(layer=layer_name, environment=environment) if fabcli.connection_exists(connection_name): connection = fabcli.get_item(f".connections/{connection_name}.Connection")