Enhance fabric setup and connection string generation

automation/scripts/fabric_setup.py

@@ -33,6 +33,7 @@
 
 # Authenticate
 fabcli.run_command("config set encryption_fallback_enabled true")
+fabcli.run_command("config set folder_listing_enabled true")
 fabcli.run_command(f"auth login -u {client_id} -p {client_secret} --tenant {tenant_id}")
 
 # Load JSON environment files (main and environment specific) and merge
@@ -171,19 +172,19 @@
                 print_item_header = True
                 for item_type, items in layer_definition.get("items").items():
                     for item in items:
-                        if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}:
+                        if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}:
                             has_item_connections = True
 
                         if not item.get("skip_item_creation", False):
                             if print_item_header:
                                 print(f"  • Creating workspace items:") 
                                 print_item_header = False
 
-                            misc.print_info(f"    ◦ {item_type}: {item.get("item_name")}...", end="")
-
-                            if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'):    
-                                fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item.get("item_name")}.{item_type}'")
-                                item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}", retry_count=2)
+                            item_folder = f'{item.get("item_folder")}/' if item.get("item_folder") else ""
+                            misc.print_info(f"    ◦ {item_type}: {item_folder}{item.get("item_name")}...", end="")
+                            if not fabcli.item_exists(f'{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'):    
+                                fabcli.run_command(f"create '{workspace_name_escaped}.Workspace/{item_folder}{item.get("item_name")}.{item_type}'")
+                                item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}", retry_count=2)
 
                                 if item_type in {"Lakehouse"}: 
                                     # Wait until SQL endpoint provisioning completes; treat missing metadata as still provisioning
@@ -200,14 +201,14 @@
                                             break
                                         print(".", end="")
                                         time.sleep(2)
-                                        item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}")
+                                        item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}")
 
                                 if item["item_metadata"]:                           
                                     misc.print_success(" ✔")
                                 else:
                                     misc.print_error(" ✖ Failed!")
                             else:
-                                item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}")
+                                item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item_folder}{item.get('item_name')}.{item_type}")
                                 misc.print_warning(f" ⚠ Already exists")
 
             if layer_definition.get("private_endpoints"):
@@ -243,7 +244,7 @@
                     git_settings = env_definition.get("generic").get("git_settings")
                     git_settings["gitProviderDetails"]["directoryName"] = layer_definition.get("git_directoryName")                    
 
-                    if git_connection.get("id"):                   
+                    if git_connection.get("id"):
                         connect_response = fabcli.connect_workspace_to_git(workspace_id, git_settings)
                         if connect_response:                            
                             init_response = fabcli.initialize_git_connection(workspace_id)
@@ -269,7 +270,7 @@
                     for item_type, items in layer_definition.get("items").items():
                         for item in items:
 
-                            if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}:
+                            if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}:
                                 connection_name = item.get("connection_name").format(layer=layer, environment=environment)
                                 item["item_metadata"] = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('item_name')}.{item_type}")
 
@@ -278,11 +279,12 @@
                                 if item["item_metadata"]:
                                     server = (
                                         item.get("item_metadata").get("properties").get("sqlEndpointProperties").get("connectionString") if item_type == "Lakehouse" else
-                                        item.get("item_metadata").get("properties").get("serverFqdn") 
+                                        item.get("item_metadata").get("properties").get("serverFqdn") if item_type == "SQLDatabase" else
+                                        item.get("item_metadata").get("properties").get("connectionString")  
                                     )
 
                                     database = (
-                                        item.get("item_name") if item_type == "Lakehouse" else
+                                        item.get("item_name") if item_type in ("Lakehouse","Warehouse") else
                                         item.get("item_metadata").get("properties").get("databaseName") 
                                     )
 
@@ -362,7 +364,7 @@
         if layer_definition.get("items"):
             for item_type, items in layer_definition.get("items").items():
                 for item in items:    
-                    if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}:
+                    if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}:
                         connection_name = item.get("connection_name").format(layer=layer, environment=environment)
                         misc.print_info(f"  • Deleting connection '{connection_name}'... ", bold=False, end="")
                         if fabcli.run_command(f"exists .connections/{connection_name}.Connection").replace("*", "").strip().lower() == "true":

automation/scripts/generate_connection_string.py

@@ -35,21 +35,22 @@
 
 solution_name = env_definition.get("name")
 workspace_name = solution_name.format(layer=layer, environment=environment)
-workspace_name_escaped = workspace_name.replace("/", "\\/")
-sqldb_item = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{database}.SQLDatabase")
-
-# Example: You may want to adjust the server/database names per environment
-server = sqldb_item.get('properties').get('serverFqdn')
-database = sqldb_item.get("properties").get("databaseName")
-
-connection_string = (
-    f"Server={server};"
-    f"Database={database};"
-    f"Authentication=Active Directory Service Principal;"
-    f"User Id={client_id};"
-    f"Password={client_secret};"
-    f"Encrypt=True;"
-    f"Connection Timeout=60;"
+
+item_type = None
+for env_item_type in env_definition.get("layers").get(layer).get("items"):
+    if env_item_type in ["Warehouse","SQLDatabase"]:
+        for item in env_definition.get("layers").get(layer).get("items").get(env_item_type):
+            item_name = item.get("item_name")
+            if item_name == database:
+                item_type = env_item_type
+                break 
+
+connection_string = fabcli.generate_connection_string(
+    workspace_name=workspace_name,
+    item_type=item_type,
+    database=database,
+    client_id=client_id,
+    client_secret=client_secret
 )
 
 with open(args.output_file, "w") as f:

automation/scripts/locale/locale_build_parameter_file.py

@@ -22,7 +22,7 @@
         "--client_secret", env_credentials.get("client_secret")
         ]
 
-process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
+process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
 
 # Print the output line by line as it is generated
 for line in process.stdout:

automation/scripts/locale/locale_maintain_feature.py

@@ -23,7 +23,7 @@
         "--action", action,
         "--branch_name", branch_name]
 
-process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
+process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
 
 # Print the output line by line as it is generated
 for line in process.stdout:

automation/scripts/locale/locale_release.py

@@ -28,7 +28,7 @@
             "--client_secret", env_credentials.get("client_secret")
             ]
 
-    process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8')
+    process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, encoding='utf-8')
 
     # Print output live
     for line in process.stdout:

automation/scripts/locale/locale_setup.py

@@ -26,7 +26,7 @@
             "--github_pat", env_credentials.get("github_pat")
             ]
 
-    process = subprocess.Popen(['python', '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
+    process = subprocess.Popen([sys.executable, '-u', script_path] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding='utf-8')
 
     # Print the output line by line as it is generated
     for line in process.stdout:

automation/scripts/modules/fabric_cli_functions.py

@@ -251,4 +251,29 @@ def poll_operation_status(operation_id):
         else:
             return None
 
-    return None  # Operation timed out or failed
+    return None  # Operation timed out or failed
+
+def generate_connection_string(workspace_name, item_type, database, client_id, client_secret):
+    print(f"Generating connection string for {item_type} '{database}' in workspace '{workspace_name}'...")
+    workspace_name_escaped = workspace_name.replace("/", "\\/")
+    sqldb_item = get_item(f"/{workspace_name_escaped}.Workspace/{database}.{item_type}")
+    print(sqldb_item)
+    if item_type == "SQLDatabase":
+        server = sqldb_item.get('properties').get('serverFqdn')
+        database = sqldb_item.get("properties").get("databaseName")
+    elif item_type == "Lakehouse":
+        server = sqldb_item.get("properties").get('sqlEndpointProperties').get('connectionString')  
+    else:
+        server = sqldb_item.get('properties').get('connectionString')
+
+    connection_string = (
+        f"Server={server};"
+        f"Database={database};"
+        f"Authentication=Active Directory Service Principal;"
+        f"User Id={client_id};"
+        f"Password={client_secret};"
+        f"Encrypt=True;"
+        f"Connection Timeout=60;"
+    )
+
+    return connection_string

automation/scripts/utils_build_parameter_file.py

@@ -74,11 +74,11 @@
                         "type": item.get("type")
                     }
 
-                    if item.get("type") in {"Lakehouse", "SQLDatabase"}:
+                    if item.get("type") in {"Lakehouse", "SQLDatabase", "Warehouse"}:
                         item_details = fabcli.get_item(f"/{workspace_name_escaped}.Workspace/{item.get('displayName')}.{item.get('type')}", retry_count=2)
                         fabric_item.update({
-                            "connectionString": item_details.get("properties").get("connectionString") if item.get("type") == "SQLDatabase" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") ,
-                            "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else None,
+                            "connectionString": item_details.get("properties").get("connectionString") if item.get("type") != "Lakehouse" else item_details.get("properties").get("sqlEndpointProperties").get("connectionString") ,
+                            "databaseName": item_details.get("properties").get("databaseName") if item.get("type") == "SQLDatabase" else item_details.get("displayName") if item.get("type") == "Warehouse" else None,
                             "serverFqdn": item_details.get("properties").get("serverFqdn") if item.get("type") == "SQLDatabase" else None,
                             "sqlEndpointId": item_details.get("properties").get("sqlEndpointProperties").get("id") if item.get("type") == "Lakehouse" else None,
                         })
@@ -89,7 +89,7 @@
                 if layer_definition.get("items"):
                     for item_type, items in layer_definition.get("items").items():
                         for item in items: 
-                            if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase"}:
+                            if item.get("connection_name") and item_type in {"Lakehouse", "SQLDatabase", "Warehouse"}:
                                 connection_name = item.get("connection_name").format(layer=layer_name, environment=environment)
                                 if fabcli.connection_exists(connection_name):
                                     connection = fabcli.get_item(f".connections/{connection_name}.Connection")