diff --git a/README.md b/README.md index 949b9f9..fc29031 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Direct TDS via ODBC is not used; SQL reads are executed via the Custom API over - For Web API (OData), tokens target your Dataverse org URL scope: https://yourorg.crm.dynamics.com/.default. The SDK requests this scope from the provided TokenCredential. - For complete functionalities, please use one of the PREPROD BAP environments, otherwise McpExecuteSqlQuery might not work. +- For CreateInstantEntities call, it's a prerequisite to import solution https://microsoft-my.sharepoint.com/:u:/p/cdietric/EXuJB0ZywshPuVAc54b2HxUBpnlv9jjQl47QCrx-VhSErA?e=4cdYm0 ### Configuration (DataverseConfig) @@ -69,6 +70,7 @@ The quickstart demonstrates: - Bulk create (CreateMultiple) to insert many records in one call - Retrieve multiple with paging (contrasting `$top` vs `page_size`) - Executing a read-only SQL query +- Use CreateInstantEntities API to quickly create entities ## Examples @@ -218,6 +220,26 @@ info = client.create_table( }, ) +# Alternatively create a custom table with use_instant option +# Only text type column is supported and lookups and display_name are required inputs +# info = client.create_table( +# "new_SampleItemInstant", +# { +# "code": "text", +# "count": "text", +# }, +# use_instant=True, +# display_name="Sample Item", +# lookups=[ +# { +# "AttributeName": "new_Account", +# "AttributeDisplayName": "Account (Demo Lookup)", +# "ReferencedEntityName": "account", +# "RelationshipName": "new_newSampleItem_account", +# } +# ], +# ) + entity_set = info["entity_set_name"] # e.g., "new_sampleitems" logical = info["entity_logical_name"] # e.g., "new_sampleitem" diff --git a/examples/quickstart.py b/examples/quickstart.py index 0db2f49..13ea88f 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -26,6 +26,8 @@ # Ask once whether to pause between steps during this run pause_choice = input("Pause between test steps? (y/N): ").strip() or "n" pause_between_steps = (str(pause_choice).lower() in ("y", "yes", "true", "1")) +instant_create_choice = input("Run instant create demo? (y/N): ").strip() or "n" +run_instant_create = (str(instant_create_choice).lower() in ("y", "yes", "true", "1")) # Create a credential we can reuse (for DataverseClient) credential = InteractiveBrowserCredential() client = DataverseClient(base_url=base_url, credential=credential) @@ -42,6 +44,20 @@ def pause(next_step: str) -> None: # If stdin is not available, just proceed pass +# Helper: delete a table if it exists +def delete_table_if_exists(table_name: str) -> None: + try: + log_call(f"client.get_table_info('{table_name}')") + info = client.get_table_info(table_name) + if info: + log_call(f"client.delete_table('{table_name}')") + client.delete_table(table_name) + print({"table_deleted": True}) + else: + print({"table_deleted": False, "reason": "not found"}) + except Exception as e: + print({f"Delete table failed": str(e)}) + # Small generic backoff helper used only in this quickstart # Include common transient statuses like 429/5xx to improve resilience. def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403, 404, 409, 412, 429, 500, 502, 503, 504), retry_if=None): @@ -68,6 +84,11 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 table_info = None created_this_run = False +# Timing metrics for comparison (seconds) +instant_create_seconds: float | None = None +standard_create_seconds: float | None = None +warm_up_seconds: float | None = None + # Check for existing table using list_tables log_call("client.list_tables()") tables = client.list_tables() @@ -87,6 +108,7 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 # Create it since it doesn't exist try: log_call("client.create_table('new_SampleItem', schema={code,count,amount,when,active})") + _t0_standard = time.perf_counter() table_info = client.create_table( "new_SampleItem", { @@ -97,6 +119,7 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 "active": "bool", }, ) + standard_create_seconds = time.perf_counter() - _t0_standard created_this_run = True if table_info and table_info.get("columns_created") else False print({ "table": table_info.get("entity_schema") if table_info else None, @@ -124,6 +147,19 @@ def backoff_retry(op, *, delays=(0, 2, 5, 10, 20), retry_http_statuses=(400, 403 entity_set = table_info.get("entity_set_name") logical = table_info.get("entity_logical_name") or entity_set.rstrip("s") +if run_instant_create: + # call early to warm up for instant create + log_call("client.warm_up_instant_create()") + try: + _t0_warm = time.perf_counter() + client.warm_up_instant_create() + warm_up_seconds = time.perf_counter() - _t0_warm + print({"warm_up_for_instant_create": True, "warm_up_seconds": warm_up_seconds}) + except Exception as warm_ex: + print({"warm_up_for_instant_create_error": str(warm_ex)}) + # Abort instant demo if warm-up fails + sys.exit(1) + # Derive attribute logical name prefix from the entity logical name (segment before first underscore) attr_prefix = logical.split("_", 1)[0] if "_" in logical else logical code_key = f"{attr_prefix}_code" @@ -410,21 +446,77 @@ def _del_one(rid: str) -> tuple[str, bool, str | None]: except Exception as e: print(f"Delete failed: {e}") +# 6) (Optional) Instant create path demo +if not run_instant_create: + print("Skipping instant create demo as per user choice.") +else: + pause("Next: instant create demo") + + print("Instant create demo") + print("Delete Instant table first if exists") + # Delete instant table first + delete_table_if_exists("new_SampleItemInstant") + + # Create Instant + log_call("client.create_table('new_SampleItemInstant', instant_create)") + instant_schema = { + "code": "text", + "count": "text", + } + # Demo dummy lookup definition (must supply at least one for instant path) + instant_lookups = [ + { + "AttributeName": "new_Account", + "AttributeDisplayName": "Account (Demo Lookup)", + "ReferencedEntityName": "account", + "RelationshipName": "new_newSampleItem_account", + } + ] + try: + _t0_instant = time.perf_counter() + _table_instant = client.create_table( + "new_SampleItemInstant", + instant_schema, + use_instant=True, + display_name="Sample Item", + lookups=instant_lookups, + ) + instant_create_seconds = time.perf_counter() - _t0_instant + table_info = _table_instant + logical = table_info.get("entity_logical_name") if isinstance(table_info, dict) else None + print(table_info) + except Exception as instant_ex: + print({"instant_create_error": str(instant_ex)}) + sys.exit(1) + + # Timing comparison summary for table creation + _standard_create_ran = standard_create_seconds is not None + _instant_create_ran = instant_create_seconds is not None + print({ + "table_creation_timing_compare": { + "warm_up_seconds": warm_up_seconds, + "instant_seconds": instant_create_seconds, + "warm_up+instant_seconds": warm_up_seconds + instant_create_seconds, + "standard_seconds": standard_create_seconds if _standard_create_ran else "standard table pre-existed; omitted", + "delta_standard_minus_instant": ( + (standard_create_seconds - instant_create_seconds) + if (_standard_create_ran and _instant_create_ran) + else None + ), + } + }) + + pause("Next: Cleanup table") -# 6) Cleanup: delete the custom table if it exists +# 7) Cleanup: delete the custom table if it exists print("Cleanup (Metadata):") if delete_table_at_end: - try: - log_call("client.get_table_info('new_SampleItem')") - info = client.get_table_info("new_SampleItem") - if info: - log_call("client.delete_table('new_SampleItem')") - client.delete_table("new_SampleItem") - print({"table_deleted": True}) - else: - print({"table_deleted": False, "reason": "not found"}) - except Exception as e: - print(f"Delete table failed: {e}") + delete_table_if_exists("new_SampleItem") else: print({"table_deleted": False, "reason": "user opted to keep table"}) + +# Put instant table delete at the end to avoid metadata cache issues when deletion immediately follows creation +if run_instant_create: + print("Cleanup instant (Metadata):") + delete_table_if_exists("new_SampleItemInstant") diff --git a/src/dataverse_sdk/client.py b/src/dataverse_sdk/client.py index 5adf09b..1413a07 100644 --- a/src/dataverse_sdk/client.py +++ b/src/dataverse_sdk/client.py @@ -198,8 +198,22 @@ def get_table_info(self, tablename: str) -> Optional[Dict[str, Any]]: """ return self._get_odata().get_table_info(tablename) - def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any]: - """Create a simple custom table. + def create_table( + self, + tablename: str, + schema: Dict[str, str], + use_instant: bool = False, + display_name: Optional[str] = None, + lookups: Optional[List[Dict[str, str]]] = None, + ) -> Dict[str, Any]: + """Create a custom table (standard or instant path). + + Standard path (default): uses supported metadata APIs and allows a variety of column types + (``string``, ``int``, ``decimal``, ``float``, ``datetime``, ``bool``). + + Instant path (``use_instant=True``): attempts creation via the CreateInstantEntities API. + This path currently only supports ``text`` columns and requires at least one lookup + relationship Parameters ---------- @@ -207,7 +221,15 @@ def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any] Friendly name (``"SampleItem"``) or a full schema name (``"new_SampleItem"``). schema : dict[str, str] Column definitions mapping logical names (without prefix) to types. - Supported: ``string``, ``int``, ``decimal``, ``float``, ``datetime``, ``bool``. + Supported for standard path: ``string``, ``int``, ``decimal``, ``float``, ``datetime``, ``bool``. + For instant path you must supply only text columns (``text``). + use_instant : bool, default False + If True, use the instant entity creation path. Must call warm_up_instant_create before using it + display_name : str | None + Required when ``use_instant`` is True. Singular display label (collection name pluralized with an ``s``). + lookups : list[dict] | None + Required when ``use_instant`` is True. Each item must include: + ``AttributeName``, ``AttributeDisplayName``, ``ReferencedEntityName``, ``RelationshipName``. Returns ------- @@ -215,7 +237,13 @@ def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any] Metadata summary including ``entity_schema``, ``entity_set_name``, ``entity_logical_name``, ``metadata_id``, and ``columns_created``. """ - return self._get_odata().create_table(tablename, schema) + return self._get_odata().create_table( + tablename, + schema, + use_instant=use_instant, + display_name=display_name, + lookups=lookups, + ) def delete_table(self, tablename: str) -> None: """Delete a custom table by name. @@ -237,6 +265,15 @@ def list_tables(self) -> list[str]: """ return self._get_odata().list_tables() + # Instant create warm-up + def warm_up_instant_create(self) -> None: + """Perform required warm-up for instant table creation. + + Must be called (and succeed) before invoking ``create_table`` + with ``use_instant=True``. + """ + self._get_odata().warm_up_instant_create() + __all__ = ["DataverseClient"] diff --git a/src/dataverse_sdk/odata.py b/src/dataverse_sdk/odata.py index 8ae7c2c..41aeaa5 100644 --- a/src/dataverse_sdk/odata.py +++ b/src/dataverse_sdk/odata.py @@ -477,6 +477,130 @@ def _attribute_payload(self, schema_name: str, dtype: str, *, is_primary_name: b }, } return None + + def _build_instant_entity_payload( + self, + schema_name: str, + display_name: str, + primary_attribute: str, + columns: Dict[str, str], + lookups: List[Dict[str, str]], + ) -> Dict[str, Any]: + """Build an expando entity definition for CreateInstantEntities. + + Parameters + ---------- + schema_name : str + Entity SchemaName (e.g. new_SampleItem). + display_name : str + Singular display label (Collection name will append 's'). + primary_attribute : str + Logical name for the primary name attribute (e.g. new_name). + columns : dict[str, str] + Mapping of column logical (or friendly) name -> datatype. Only "text" is currently supported. + lookups : list[dict] + Lookup definitions required by CreateInstantEntities internal API. + Required keys per item: AttributeName, AttributeDisplayName, ReferencedEntityName, RelationshipName. + + Returns + ------- + dict + A payload body usable as the JSON for POST /CreateInstantEntities (already wrapped with ExecuteInParallel/Entities structure). + + Notes + ----- + All columns must be text; any other types are not supported. + """ + if not schema_name or "_" not in schema_name: + raise ValueError("schema_name must be a full schema (with publisher prefix).") + if not display_name: + raise ValueError("display_name is required.") + if not primary_attribute: + raise ValueError("primary_attribute is required.") + if not columns: + raise ValueError("At least one column required for instant creation.") + if not lookups: + raise ValueError("At least one lookup is required for instant creation.") + + # Validate columns are all text types + expando_attrs: List[Dict[str, Any]] = [] + publisher_prefix = schema_name.split('_', 1)[0] if "_" in schema_name else "new" + for original_name, dtype in columns.items(): + dt = (dtype or "").strip().lower() + if dt not in ("text"): + raise ValueError(f"Unsupported column type '{dtype}' for '{original_name}'. Only text columns are allowed.") + # Ensure logical name has publisher prefix; if caller omitted it, prepend automatically. + if original_name.lower().startswith(f"{publisher_prefix.lower()}_"): + logical_name = original_name + else: + logical_name = f"{publisher_prefix}_{original_name}" if original_name else original_name + display_label = original_name or logical_name + expando_attrs.append({ + "@odata.type": "Microsoft.Dynamics.CRM.expando", + "Name": logical_name, + "DisplayName": display_label, + "Type": "text", + "TypeMetadata": { + "@odata.type": "Microsoft.Dynamics.CRM.expando", + "Length": 200, + }, + }) + + expando_lookups: List[Dict[str, Any]] = [] + required = {"AttributeName", "AttributeDisplayName", "ReferencedEntityName", "RelationshipName"} + for lk in lookups: + if not required.issubset(lk.keys()): + raise ValueError(f"Lookup missing required keys: {lk}") + expando_lookups.append( + { + "@odata.type": "Microsoft.Dynamics.CRM.expando", + "AttributeName": lk["AttributeName"], + "AttributeDisplayName": lk["AttributeDisplayName"], + "ReferencedEntityName": lk["ReferencedEntityName"], + "RelationshipName": lk["RelationshipName"], + } + ) + + entity_expando = { + "@odata.type": "Microsoft.Dynamics.CRM.expando", + "Name": schema_name, + "DisplayName": display_name, + "CollectionDisplayName": display_name + "s", + "NameAttribute": primary_attribute.lower(), # API only works with lowercase here + "Attributes@odata.type": "#Collection(Microsoft.Dynamics.CRM.expando)", + "Attributes": expando_attrs, + "Lookups@odata.type": "#Collection(Microsoft.Dynamics.CRM.expando)", + "Lookups": expando_lookups, + } + return {"ExecuteInParallel": "true", "Entities": [entity_expando]} + + def _create_entity_instant( + self, + schema_name: str, + display_name: str, + primary_attribute: str, + columns: Dict[str, str], + lookups: List[Dict[str, str]], + ) -> str: + payload = self._build_instant_entity_payload( + schema_name=schema_name, + display_name=display_name, + primary_attribute=primary_attribute, + columns=columns, + lookups=lookups, + ) + headers = self._headers() + + # CreateInstantEntities must always be called after CreateInstantEntityWarmUp + create_url = f"{self.base_url}/api/data/v9.0/CreateInstantEntities" + r = self._request("post", create_url, headers=headers, json=payload) + r.raise_for_status() + ent = self._wait_for_entity_ready(schema_name) + if not ent or not ent.get("EntitySetName"): + raise RuntimeError( + f"Failed to create or retrieve entity '{schema_name}' (EntitySetName not available)." + ) + return ent["MetadataId"] def get_table_info(self, tablename: str) -> Optional[Dict[str, Any]]: """Return basic metadata for a custom table if it exists. @@ -524,7 +648,34 @@ def delete_table(self, tablename: str) -> None: r = self._request("delete", url, headers=headers) r.raise_for_status() - def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any]: + def create_table( + self, + tablename: str, + schema: Dict[str, str], + use_instant: bool = False, + display_name: Optional[str] = None, + lookups: Optional[List[Dict[str, str]]] = None, + ) -> Dict[str, Any]: + """Create a custom table (entity). + + Parameters + ---------- + tablename : str + Friendly name (without publisher prefix) or full SchemaName (with prefix underscore form). + schema : dict[str,str] + Mapping of column friendly/logical name -> datatype (string/int/decimal/datetime/bool, etc.). + use_instant : bool, default False + If True, attempt creation via CreateInstantEntities API (fast path). Must call warm_up_instant_create before using it. + display_name : str | None + Required when use_instant=True; used for DisplayName / CollectionDisplayName. + lookups : list[dict] | None + Required when use_instant=True. Items must include AttributeName, AttributeDisplayName, ReferencedEntityName, RelationshipName. + + Returns + ------- + dict + { entity_schema, entity_logical_name, entity_set_name, metadata_id, columns_created } + """ # Accept a friendly name and construct a default schema under 'new_'. # If a full SchemaName is passed (contains '_'), use as-is. entity_schema = tablename if "_" in tablename else f"new_{self._to_pascal(tablename)}" @@ -533,13 +684,12 @@ def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any] if ent: raise RuntimeError(f"Table '{entity_schema}' already exists. No update performed.") - created_cols: List[str] = [] primary_attr_schema = "new_Name" if "_" not in entity_schema else f"{entity_schema.split('_',1)[0]}_Name" + created_cols: List[str] = [] attributes: List[Dict[str, Any]] = [] - attributes.append(self._attribute_payload(primary_attr_schema, "string", is_primary_name=True)) + publisher = entity_schema.split("_", 1)[0] if "_" in entity_schema else "new" for col_name, dtype in schema.items(): # Use same publisher prefix segment as entity_schema if present; else default to 'new_'. - publisher = entity_schema.split("_", 1)[0] if "_" in entity_schema else "new" if col_name.lower().startswith(f"{publisher}_"): attr_schema = col_name else: @@ -550,7 +700,18 @@ def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any] attributes.append(payload) created_cols.append(attr_schema) - metadata_id = self._create_entity(entity_schema, tablename, attributes) + if use_instant: + metadata_id = self._create_entity_instant( + schema_name=entity_schema, + display_name=display_name, + primary_attribute=primary_attr_schema, + columns=schema, + lookups=lookups, + ) + else: + attributes.append(self._attribute_payload(primary_attr_schema, "string", is_primary_name=True)) + metadata_id = self._create_entity(entity_schema, tablename, attributes) + ent2: Dict[str, Any] = self._wait_for_entity_ready(entity_schema) or {} logical_name = ent2.get("LogicalName") @@ -561,3 +722,24 @@ def create_table(self, tablename: str, schema: Dict[str, str]) -> Dict[str, Any] "metadata_id": metadata_id, "columns_created": created_cols, } + + def warm_up_instant_create(self) -> None: + """Explicit warm-up required before using the instant table creation. + + Raises + ------ + RuntimeError + If the warm-up request fails or returns a non-success status code. + """ + headers = self._headers().copy() + warm_url = f"{self.base_url}/api/data/v9.0/CreateInstantEntityWarmUp" + resp = self._request("post", warm_url, headers=headers, json={}) + if resp.status_code not in (200, 202, 204): + body_preview = None + try: + body_preview = resp.text[:400] if resp.text else None + except Exception: + body_preview = None + raise RuntimeError( + f"CreateInstantEntityWarmUp failed: {resp.status_code} {body_preview or ''}".strip() + )