From 2385b230cac92f5f0e18835eeb120f76118bb713 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Wed, 19 Nov 2025 16:28:50 -0800 Subject: [PATCH 01/13] add client side correlation id, client request id. expose these and also service request id correctly --- README.md | 19 ++ src/PowerPlatform/Dataverse/client.py | 219 +++++++++++++-------- src/PowerPlatform/Dataverse/core/errors.py | 19 +- src/PowerPlatform/Dataverse/data/_odata.py | 45 ++++- tests/unit/core/test_http_errors.py | 81 ++++++++ 5 files changed, 288 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index b07b232..869a779 100644 --- a/README.md +++ b/README.md @@ -300,6 +300,25 @@ except ValidationError as e: print(f"Validation error: {e.message}") ``` +### Correlate multiple requests to debug + +Give your own identifier to every HTTP call by wrapping operations in +`DataverseClient.correlation_scope()`: + +```python +from uuid import uuid4 + +with client.correlation_scope(str(uuid4())): + client.create("account", {"name": "Scoped Request"}) + pages = client.get("account", filter="statecode eq 0") + for batch in pages: + ... +``` + +All nested SDK calls inside the block (including pagination and retries) reuse +the provided value for the `x-ms-correlation-request-id` header, which makes it +easy to align Dataverse traces. If you omit the context manager, the SDK automatically generates unique correlation IDs. + ### Authentication issues **Common fixes:** diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 2fb11e8..5c63340 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -3,7 +3,8 @@ from __future__ import annotations -from typing import Any, Dict, Optional, Union, List, Iterable +from typing import Any, Dict, Optional, Union, List, Iterable, Iterator +from contextlib import contextmanager from azure.core.credentials import TokenCredential @@ -99,6 +100,13 @@ def _get_odata(self) -> _ODataClient: ) return self._odata + @contextmanager + def _scoped_odata(self) -> Iterator[_ODataClient]: + """Yield the low-level client while ensuring a correlation scope is active.""" + od = self._get_odata() + with od._call_scope(): + yield od + # ---------------- Unified CRUD: create/update/delete ---------------- def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dict[str, Any]]]) -> List[str]: """ @@ -132,19 +140,19 @@ def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dic ids = client.create("account", records) print(f"Created {len(ids)} accounts") """ - od = self._get_odata() - entity_set = od._entity_set_from_schema_name(table_schema_name) - if isinstance(records, dict): - rid = od._create(entity_set, table_schema_name, records) - # _create returns str on single input - if not isinstance(rid, str): - raise TypeError("_create (single) did not return GUID string") - return [rid] - if isinstance(records, list): - ids = od._create_multiple(entity_set, table_schema_name, records) - if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): - raise TypeError("_create (multi) did not return list[str]") - return ids + with self._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(table_schema_name) + if isinstance(records, dict): + rid = od._create(entity_set, table_schema_name, records) + # _create returns str on single input + if not isinstance(rid, str): + raise TypeError("_create (single) did not return GUID string") + return [rid] + if isinstance(records, list): + ids = od._create_multiple(entity_set, table_schema_name, records) + if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): + raise TypeError("_create (multi) did not return list[str]") + return ids raise TypeError("records must be dict or list[dict]") def update( @@ -192,16 +200,16 @@ def update( ] client.update("account", ids, changes) """ - od = self._get_odata() - if isinstance(ids, str): - if not isinstance(changes, dict): - raise TypeError("For single id, changes must be a dict") - od._update(table_schema_name, ids, changes) # discard representation + with self._scoped_odata() as od: + if isinstance(ids, str): + if not isinstance(changes, dict): + raise TypeError("For single id, changes must be a dict") + od._update(table_schema_name, ids, changes) # discard representation + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + od._update_by_ids(table_schema_name, ids, changes) return None - if not isinstance(ids, list): - raise TypeError("ids must be str or list[str]") - od._update_by_ids(table_schema_name, ids, changes) - return None def delete( self, @@ -235,21 +243,21 @@ def delete( job_id = client.delete("account", [id1, id2, id3]) """ - od = self._get_odata() - if isinstance(ids, str): - od._delete(table_schema_name, ids) - return None - if not isinstance(ids, list): - raise TypeError("ids must be str or list[str]") - if not ids: + with self._scoped_odata() as od: + if isinstance(ids, str): + od._delete(table_schema_name, ids) + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + if not ids: + return None + if not all(isinstance(rid, str) for rid in ids): + raise TypeError("ids must contain string GUIDs") + if use_bulk_delete: + return od._delete_multiple(table_schema_name, ids) + for rid in ids: + od._delete(table_schema_name, rid) return None - if not all(isinstance(rid, str) for rid in ids): - raise TypeError("ids must contain string GUIDs") - if use_bulk_delete: - return od._delete_multiple(table_schema_name, ids) - for rid in ids: - od._delete(table_schema_name, rid) - return None def get( self, @@ -328,24 +336,29 @@ def get( ): print(f"Batch size: {len(batch)}") """ - od = self._get_odata() if record_id is not None: if not isinstance(record_id, str): raise TypeError("record_id must be str") - return od._get( - table_schema_name, - record_id, - select=select, - ) - return od._get_multiple( - table_schema_name, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - ) + with self._scoped_odata() as od: + return od._get( + table_schema_name, + record_id, + select=select, + ) + + def _paged() -> Iterable[List[Dict[str, Any]]]: + with self._scoped_odata() as od: + yield from od._get_multiple( + table_schema_name, + select=select, + filter=filter, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + ) + + return _paged() # SQL via Web API sql parameter def query_sql(self, sql: str): @@ -381,7 +394,8 @@ def query_sql(self, sql: str): sql = "SELECT a.name, a.telephone1 FROM account AS a WHERE a.statecode = 0" results = client.query_sql(sql) """ - return self._get_odata()._query_sql(sql) + with self._scoped_odata() as od: + return od._query_sql(sql) # Table metadata helpers def get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: @@ -404,7 +418,8 @@ def get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: print(f"Logical name: {info['table_logical_name']}") print(f"Entity set: {info['entity_set_name']}") """ - return self._get_odata()._get_table_info(table_schema_name) + with self._scoped_odata() as od: + return od._get_table_info(table_schema_name) def create_table( self, @@ -474,12 +489,13 @@ class ItemStatus(IntEnum): primary_column_schema_name="new_ProductName" ) """ - return self._get_odata()._create_table( - table_schema_name, - columns, - solution_unique_name, - primary_column_schema_name, - ) + with self._scoped_odata() as od: + return od._create_table( + table_schema_name, + columns, + solution_unique_name, + primary_column_schema_name, + ) def delete_table(self, table_schema_name: str) -> None: """ @@ -499,7 +515,8 @@ def delete_table(self, table_schema_name: str) -> None: client.delete_table("new_MyTestTable") """ - self._get_odata()._delete_table(table_schema_name) + with self._scoped_odata() as od: + od._delete_table(table_schema_name) def list_tables(self) -> list[str]: """ @@ -515,7 +532,8 @@ def list_tables(self) -> list[str]: for table in tables: print(table) """ - return self._get_odata()._list_tables() + with self._scoped_odata() as od: + return od._list_tables() def create_columns( self, @@ -545,10 +563,11 @@ def create_columns( ) print(created) # ['new_Scratch', 'new_Flags'] """ - return self._get_odata()._create_columns( - table_schema_name, - columns, - ) + with self._scoped_odata() as od: + return od._create_columns( + table_schema_name, + columns, + ) def delete_columns( self, @@ -573,10 +592,11 @@ def delete_columns( ) print(removed) # ['new_Scratch', 'new_Flags'] """ - return self._get_odata()._delete_columns( - table_schema_name, - columns, - ) + with self._scoped_odata() as od: + return od._delete_columns( + table_schema_name, + columns, + ) # File upload def upload_file( @@ -640,18 +660,18 @@ def upload_file( mode="auto" ) """ - od = self._get_odata() - entity_set = od._entity_set_from_schema_name(table_schema_name) - od._upload_file( - entity_set, - record_id, - file_name_attribute, - path, - mode=mode, - mime_type=mime_type, - if_none_match=if_none_match, - ) - return None + with self._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(table_schema_name) + od._upload_file( + entity_set, + record_id, + file_name_attribute, + path, + mode=mode, + mime_type=mime_type, + if_none_match=if_none_match, + ) + return None # Cache utilities def flush_cache(self, kind) -> int: @@ -675,7 +695,40 @@ def flush_cache(self, kind) -> int: removed = client.flush_cache("picklist") print(f"Cleared {removed} cached picklist entries") """ - return self._get_odata()._flush_cache(kind) + with self._scoped_odata() as od: + return od._flush_cache(kind) + + # Other utilities + @contextmanager + def correlation_scope(self, correlation_id: str) -> Iterator["DataverseClient"]: + """Share a caller-specified correlation id across nested SDK calls. + + Use this context manager to stamp your own identifier on every Dataverse + request made within the ``with`` block. Nested SDK calls reuse the + existing correlation id, and concurrent scopes remain isolated. + + :param correlation_id: Non-empty identifier to propagate to + ``x-ms-correlation-request-id``. + :type correlation_id: :class:`str` + :raises TypeError: If ``correlation_id`` is not a string. + :raises ValueError: If ``correlation_id`` is empty after trimming. + + Example:: + + with client.correlation_scope("6f187988-5fb4-4bd2-9f25-4d7a1c9e24ce"): + client.create("account", {"name": "Scoped Run"}) + for batch in client.get("account", filter="statecode eq 0"): + ... + """ + + if not isinstance(correlation_id, str): + raise TypeError("correlation_id must be str") + trimmed = correlation_id.strip() + if not trimmed: + raise ValueError("correlation_id cannot be empty") + od = self._get_odata() + with od._call_scope(trimmed): + yield self __all__ = ["DataverseClient"] diff --git a/src/PowerPlatform/Dataverse/core/errors.py b/src/PowerPlatform/Dataverse/core/errors.py index 4fd4adb..5b6027e 100644 --- a/src/PowerPlatform/Dataverse/core/errors.py +++ b/src/PowerPlatform/Dataverse/core/errors.py @@ -141,10 +141,12 @@ class HttpError(DataverseError): :type subcode: :class:`str` | None :param service_error_code: Optional Dataverse-specific error code from the API response. :type service_error_code: :class:`str` | None - :param correlation_id: Optional correlation ID for tracking requests across services. + :param correlation_id: Client-generated Correlation ID for tracking requests within a SDK call. :type correlation_id: :class:`str` | None - :param request_id: Optional request ID from the API response headers. - :type request_id: :class:`str` | None + :param client_request_id: Client-generated request ID injected into outbound headers. + :type client_request_id: :class:`str` | None + :param service_request_id: ``x-ms-service-request-id`` returned by Dataverse servers. + :type service_request_id: :class:`str` | None :param traceparent: Optional W3C trace context for distributed tracing. :type traceparent: :class:`str` | None :param body_excerpt: Optional excerpt of the response body for diagnostics. @@ -159,11 +161,12 @@ def __init__( self, message: str, status_code: int, + correlation_id: Optional[str], + client_request_id: Optional[str], + service_request_id: Optional[str], is_transient: bool = False, subcode: Optional[str] = None, service_error_code: Optional[str] = None, - correlation_id: Optional[str] = None, - request_id: Optional[str] = None, traceparent: Optional[str] = None, body_excerpt: Optional[str] = None, retry_after: Optional[int] = None, @@ -174,8 +177,10 @@ def __init__( d["service_error_code"] = service_error_code if correlation_id is not None: d["correlation_id"] = correlation_id - if request_id is not None: - d["request_id"] = request_id + if client_request_id is not None: + d["client_request_id"] = client_request_id + if service_request_id is not None: + d["service_request_id"] = service_request_id if traceparent is not None: d["traceparent"] = traceparent if body_excerpt is not None: diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 8eda7ad..78319f5 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -11,8 +11,11 @@ import time import re import json +import uuid from datetime import datetime, timezone import importlib.resources as ir +from contextlib import contextmanager +from contextvars import ContextVar from ..core._http import _HttpClient from ._upload import _ODataFileUpload @@ -35,6 +38,9 @@ _USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" _GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar( + "_CALL_SCOPE_CORRELATION_ID", default=None +) class _ODataClient(_ODataFileUpload): @@ -113,6 +119,17 @@ def __init__( self._picklist_label_cache = {} self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL + @contextmanager + def _call_scope(self, correlation_id: Optional[str] = None): + """Context manager to share a correlation id across nested SDK calls.""" + existing = _CALL_SCOPE_CORRELATION_ID.get() + shared_id = correlation_id or existing or str(uuid.uuid4()) + token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) + try: + yield shared_id + finally: + _CALL_SCOPE_CORRELATION_ID.reset(token) + def _headers(self) -> Dict[str, str]: """Build standard OData headers with bearer auth.""" scope = f"{self.base_url}/.default" @@ -139,7 +156,23 @@ def _raw_request(self, method: str, url: str, **kwargs): def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 201, 202, 204), **kwargs): headers_in = kwargs.pop("headers", None) - kwargs["headers"] = self._merge_headers(headers_in) + id_headers = { + "x-ms-client-request-id": str(uuid.uuid4()), + "x-ms-correlation-request-id": _CALL_SCOPE_CORRELATION_ID.get(), + } + if headers_in: + id_headers.update(headers_in) + merged_headers = self._merge_headers(id_headers) + kwargs["headers"] = merged_headers + print( + "[DataverseSDK] request", + method.upper(), + url, + "corr:", + merged_headers.get("x-ms-correlation-request-id"), + "client:", + merged_headers.get("x-ms-client-request-id"), + ) r = self._raw_request(method, url, **kwargs) if r.status_code in expected: return r @@ -164,9 +197,10 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 pass sc = r.status_code subcode = _http_subcode(sc) - correlation_id = headers.get("x-ms-correlation-request-id") or headers.get("x-ms-correlation-id") request_id = ( - headers.get("x-ms-client-request-id") or headers.get("request-id") or headers.get("x-ms-request-id") + headers.get("x-ms-service-request-id") + or headers.get("req_id") + or headers.get("x-ms-request-id") ) traceparent = headers.get("traceparent") ra = headers.get("Retry-After") @@ -180,10 +214,11 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 raise HttpError( msg, status_code=sc, + correlation_id=merged_headers.get("x-ms-correlation-request-id"), + client_request_id=merged_headers.get("x-ms-client-request-id"), + service_request_id=request_id, subcode=subcode, service_error_code=svc_code, - correlation_id=correlation_id, - request_id=request_id, traceparent=traceparent, body_excerpt=body_excerpt, retry_after=retry_after, diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 137aea5..145df1b 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -2,6 +2,9 @@ # Licensed under the MIT license. import pytest +from azure.core.credentials import TokenCredential +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.core.config import DataverseConfig from PowerPlatform.Dataverse.core.errors import HttpError from PowerPlatform.Dataverse.core._error_codes import HTTP_404, HTTP_429, HTTP_500 from PowerPlatform.Dataverse.data._odata import _ODataClient @@ -55,6 +58,25 @@ def __init__(self, responses): self._http = DummyHTTP(responses) +class RecordingHTTP(DummyHTTP): + def __init__(self, responses): + super().__init__(responses) + self.recorded_headers = [] + + def _request(self, method, url, **kwargs): + headers = (kwargs.get("headers") or {}).copy() + self.recorded_headers.append(headers) + return super()._request(method, url, **kwargs) + + +class DummyCredential(TokenCredential): + def get_token(self, *scopes, **kwargs): + class Tok: + token = "dummy-token" + + return Tok() + + # --- Tests --- @@ -120,3 +142,62 @@ def test_http_non_mapped_status_code_subcode_fallback(): c._request("get", c.api + "/accounts") err = ei.value.to_dict() assert err["subcode"] == "http_418" + + +def test_correlation_id_diff_without_scope(): + responses = [ + (200, {}, {"value": []}), + (200, {}, {"value": []}), + ] + c = MockClient([]) + recorder = RecordingHTTP(responses) + c._http = recorder + c._request("get", c.api + "/accounts") + c._request("get", c.api + "/accounts") + assert len(recorder.recorded_headers) == 2 + h1, h2 = recorder.recorded_headers + assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] + assert h1["x-ms-correlation-request-id"] != h2["x-ms-correlation-request-id"] + + +def test_correlation_id_shared_inside_call_scope(): + responses = [ + (200, {}, {"value": []}), + (200, {}, {"value": []}), + ] + c = MockClient([]) + recorder = RecordingHTTP(responses) + c._http = recorder + with c._call_scope(): + c._request("get", c.api + "/accounts") + c._request("get", c.api + "/accounts") + assert len(recorder.recorded_headers) == 2 + h1, h2 = recorder.recorded_headers + assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] + assert h1["x-ms-correlation-request-id"] == h2["x-ms-correlation-request-id"] + + +def test_dataverse_client_correlation_scope_uses_user_id(): + responses = [ + (200, {}, {"value": []}), + (200, {}, {"value": []}), + ] + client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) + mock = MockClient([]) + recorder = RecordingHTTP(responses) + mock._http = recorder + client._odata = mock + with client.correlation_scope("custom-correlation-id"): + mock._request("get", mock.api + "/accounts") + mock._request("get", mock.api + "/accounts") + assert len(recorder.recorded_headers) == 2 + h1, h2 = recorder.recorded_headers + assert h1["x-ms-correlation-request-id"] == "custom-correlation-id" + assert h2["x-ms-correlation-request-id"] == "custom-correlation-id" + + +def test_correlation_scope_rejects_blank_identifier(): + client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) + with pytest.raises(ValueError): + with client.correlation_scope(" "): + pass From 3f9473ec8d4809a02810ba86c1cff9ff949027d0 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Wed, 19 Nov 2025 16:32:33 -0800 Subject: [PATCH 02/13] cleanup --- src/PowerPlatform/Dataverse/data/_odata.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 78319f5..4e52961 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -164,15 +164,6 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 id_headers.update(headers_in) merged_headers = self._merge_headers(id_headers) kwargs["headers"] = merged_headers - print( - "[DataverseSDK] request", - method.upper(), - url, - "corr:", - merged_headers.get("x-ms-correlation-request-id"), - "client:", - merged_headers.get("x-ms-client-request-id"), - ) r = self._raw_request(method, url, **kwargs) if r.status_code in expected: return r From 8b6fa848d81334ce17d6c2092f72b8a180d05709 Mon Sep 17 00:00:00 2001 From: zhaodongwang-msft Date: Wed, 19 Nov 2025 16:33:56 -0800 Subject: [PATCH 03/13] Update src/PowerPlatform/Dataverse/client.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/PowerPlatform/Dataverse/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 5c63340..9eed15a 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -697,7 +697,6 @@ def flush_cache(self, kind) -> int: """ with self._scoped_odata() as od: return od._flush_cache(kind) - # Other utilities @contextmanager def correlation_scope(self, correlation_id: str) -> Iterator["DataverseClient"]: From e611ded3b0eccc610ca290a5fc7883a0ce7630dc Mon Sep 17 00:00:00 2001 From: Max Wang Date: Wed, 19 Nov 2025 16:44:00 -0800 Subject: [PATCH 04/13] cleanup and move the ids to be optional --- src/PowerPlatform/Dataverse/core/errors.py | 12 ++++++------ src/PowerPlatform/Dataverse/data/_odata.py | 4 ++-- tests/unit/core/test_http_errors.py | 7 ++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/PowerPlatform/Dataverse/core/errors.py b/src/PowerPlatform/Dataverse/core/errors.py index 5b6027e..9ade2fd 100644 --- a/src/PowerPlatform/Dataverse/core/errors.py +++ b/src/PowerPlatform/Dataverse/core/errors.py @@ -141,11 +141,11 @@ class HttpError(DataverseError): :type subcode: :class:`str` | None :param service_error_code: Optional Dataverse-specific error code from the API response. :type service_error_code: :class:`str` | None - :param correlation_id: Client-generated Correlation ID for tracking requests within a SDK call. + :param correlation_id: Optional client-generated correlation ID for tracking requests within an SDK call. :type correlation_id: :class:`str` | None - :param client_request_id: Client-generated request ID injected into outbound headers. + :param client_request_id: Optional client-generated request ID injected into outbound headers. :type client_request_id: :class:`str` | None - :param service_request_id: ``x-ms-service-request-id`` returned by Dataverse servers. + :param service_request_id: Optional ``x-ms-service-request-id`` value returned by Dataverse servers. :type service_request_id: :class:`str` | None :param traceparent: Optional W3C trace context for distributed tracing. :type traceparent: :class:`str` | None @@ -161,12 +161,12 @@ def __init__( self, message: str, status_code: int, - correlation_id: Optional[str], - client_request_id: Optional[str], - service_request_id: Optional[str], is_transient: bool = False, subcode: Optional[str] = None, service_error_code: Optional[str] = None, + correlation_id: Optional[str] = None, + client_request_id: Optional[str] = None, + service_request_id: Optional[str] = None, traceparent: Optional[str] = None, body_excerpt: Optional[str] = None, retry_after: Optional[int] = None, diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 4e52961..99835d9 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -205,11 +205,11 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 raise HttpError( msg, status_code=sc, + subcode=subcode, + service_error_code=svc_code, correlation_id=merged_headers.get("x-ms-correlation-request-id"), client_request_id=merged_headers.get("x-ms-client-request-id"), service_request_id=request_id, - subcode=subcode, - service_error_code=svc_code, traceparent=traceparent, body_excerpt=body_excerpt, retry_after=retry_after, diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 145df1b..af43958 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -157,7 +157,12 @@ def test_correlation_id_diff_without_scope(): assert len(recorder.recorded_headers) == 2 h1, h2 = recorder.recorded_headers assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] - assert h1["x-ms-correlation-request-id"] != h2["x-ms-correlation-request-id"] + cid1 = h1.get("x-ms-correlation-request-id") + cid2 = h2.get("x-ms-correlation-request-id") + if cid1 is not None and cid2 is not None: + assert cid1 != cid2 + else: + assert cid1 is cid2 is None def test_correlation_id_shared_inside_call_scope(): From 87211a1f2564c55ef18a320b2844f74e4cb8ca41 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 20 Nov 2025 09:50:53 -0800 Subject: [PATCH 05/13] black formatting --- src/PowerPlatform/Dataverse/client.py | 1 + src/PowerPlatform/Dataverse/data/_odata.py | 10 ++-------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 9eed15a..68af39c 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -697,6 +697,7 @@ def flush_cache(self, kind) -> int: """ with self._scoped_odata() as od: return od._flush_cache(kind) + # Other utilities @contextmanager def correlation_scope(self, correlation_id: str) -> Iterator["DataverseClient"]: diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 99835d9..f4274f0 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -38,9 +38,7 @@ _USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" _GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") -_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar( - "_CALL_SCOPE_CORRELATION_ID", default=None -) +_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) class _ODataClient(_ODataFileUpload): @@ -188,11 +186,7 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 pass sc = r.status_code subcode = _http_subcode(sc) - request_id = ( - headers.get("x-ms-service-request-id") - or headers.get("req_id") - or headers.get("x-ms-request-id") - ) + request_id = headers.get("x-ms-service-request-id") or headers.get("req_id") or headers.get("x-ms-request-id") traceparent = headers.get("traceparent") ra = headers.get("Retry-After") retry_after = None From 333f80045a7823810fa60b830a198b2fa8268e8f Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 18 Dec 2025 16:35:11 -0800 Subject: [PATCH 06/13] clean up request and use correct correlation id header --- src/PowerPlatform/Dataverse/data/_odata.py | 66 ++++++++++++++++------ 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index f4274f0..ad73daa 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -5,8 +5,9 @@ from __future__ import annotations -from typing import Any, Dict, Optional, List, Union, Iterable +from typing import Any, Dict, Optional, List, Union, Iterable, Callable from enum import Enum +from dataclasses import dataclass, field import unicodedata import time import re @@ -39,7 +40,41 @@ _USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" _GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") _CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) - +_DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) + + +@dataclass +class _RequestContext: + """Structured request context used by ``_request`` to clarify payload and metadata.""" + + method: str + url: str + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES + headers: Optional[Dict[str, str]] = None + kwargs: Dict[str, Any] = field(default_factory=dict) + + @classmethod + def build( + cls, + method: str, + url: str, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + merge_headers: Optional[Callable[[Optional[Dict[str, str]]], Dict[str, str]]] = None, + **kwargs: Any, + ) -> "_RequestContext": + headers = kwargs.get("headers") + headers = merge_headers(headers) if merge_headers else (headers or {}) + headers.setdefault("x-ms-client-request-id", str(uuid.uuid4())) + headers.setdefault("x-ms-correlation-id", _CALL_SCOPE_CORRELATION_ID.get()) + kwargs["headers"] = headers + return cls( + method=method, + url=url, + expected=expected, + headers=headers, + kwargs=kwargs or {}, + ) class _ODataClient(_ODataFileUpload): """Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" @@ -152,18 +187,17 @@ def _merge_headers(self, headers: Optional[Dict[str, str]] = None) -> Dict[str, def _raw_request(self, method: str, url: str, **kwargs): return self._http._request(method, url, **kwargs) - def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 201, 202, 204), **kwargs): - headers_in = kwargs.pop("headers", None) - id_headers = { - "x-ms-client-request-id": str(uuid.uuid4()), - "x-ms-correlation-request-id": _CALL_SCOPE_CORRELATION_ID.get(), - } - if headers_in: - id_headers.update(headers_in) - merged_headers = self._merge_headers(id_headers) - kwargs["headers"] = merged_headers - r = self._raw_request(method, url, **kwargs) - if r.status_code in expected: + def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, **kwargs): + request_context = _RequestContext.build( + method, + url, + expected=expected, + merge_headers=self._merge_headers, + **kwargs, + ) + + r = self._raw_request(request_context.method, request_context.url, **request_context.kwargs) + if r.status_code in request_context.expected: return r headers = getattr(r, "headers", {}) or {} body_excerpt = (getattr(r, "text", "") or "")[:200] @@ -201,8 +235,8 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 status_code=sc, subcode=subcode, service_error_code=svc_code, - correlation_id=merged_headers.get("x-ms-correlation-request-id"), - client_request_id=merged_headers.get("x-ms-client-request-id"), + correlation_id=request_context.headers.get("x-ms-correlation-id"), # this is a value set on client side, although it's logged on server side too + client_request_id=request_context.headers.get("x-ms-client-request-id"), # this is a value set on client side, although it's logged on server side too service_request_id=request_id, traceparent=traceparent, body_excerpt=body_excerpt, From a507b787e03c870fe8d98868f779850e67b645f9 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 18 Dec 2025 16:36:11 -0800 Subject: [PATCH 07/13] rename --- src/PowerPlatform/Dataverse/data/_odata.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index ad73daa..31b6d3e 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -199,7 +199,7 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAUL r = self._raw_request(request_context.method, request_context.url, **request_context.kwargs) if r.status_code in request_context.expected: return r - headers = getattr(r, "headers", {}) or {} + response_headers = getattr(r, "headers", {}) or {} body_excerpt = (getattr(r, "text", "") or "")[:200] svc_code = None msg = f"HTTP {r.status_code}" @@ -220,9 +220,9 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAUL pass sc = r.status_code subcode = _http_subcode(sc) - request_id = headers.get("x-ms-service-request-id") or headers.get("req_id") or headers.get("x-ms-request-id") - traceparent = headers.get("traceparent") - ra = headers.get("Retry-After") + request_id = response_headers.get("x-ms-service-request-id") or response_headers.get("req_id") or response_headers.get("x-ms-request-id") + traceparent = response_headers.get("traceparent") + ra = response_headers.get("Retry-After") retry_after = None if ra: try: From ba5fefee62f66bd85a234ddd778928b85707517c Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 18 Dec 2025 17:04:12 -0800 Subject: [PATCH 08/13] enforce optional correlation id input to be guid --- src/PowerPlatform/Dataverse/data/_odata.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 31b6d3e..fc61011 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -156,7 +156,14 @@ def __init__( def _call_scope(self, correlation_id: Optional[str] = None): """Context manager to share a correlation id across nested SDK calls.""" existing = _CALL_SCOPE_CORRELATION_ID.get() - shared_id = correlation_id or existing or str(uuid.uuid4()) + if correlation_id is not None: + if not (_GUID_RE.fullmatch(correlation_id)): + raise ValueError("correlation_id provided must be a GUID string") + shared_id = correlation_id + elif existing is not None: + shared_id = existing + else: + shared_id = str(uuid.uuid4()) token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) try: yield shared_id From dac2b11e69d38409b987e469fc18e360f1052736 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 18 Dec 2025 17:08:39 -0800 Subject: [PATCH 09/13] black formatting --- src/PowerPlatform/Dataverse/data/_odata.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index fc61011..440d9b4 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -76,6 +76,7 @@ def build( kwargs=kwargs or {}, ) + class _ODataClient(_ODataFileUpload): """Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" @@ -227,7 +228,11 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAUL pass sc = r.status_code subcode = _http_subcode(sc) - request_id = response_headers.get("x-ms-service-request-id") or response_headers.get("req_id") or response_headers.get("x-ms-request-id") + request_id = ( + response_headers.get("x-ms-service-request-id") + or response_headers.get("req_id") + or response_headers.get("x-ms-request-id") + ) traceparent = response_headers.get("traceparent") ra = response_headers.get("Retry-After") retry_after = None @@ -242,8 +247,12 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = _DEFAUL status_code=sc, subcode=subcode, service_error_code=svc_code, - correlation_id=request_context.headers.get("x-ms-correlation-id"), # this is a value set on client side, although it's logged on server side too - client_request_id=request_context.headers.get("x-ms-client-request-id"), # this is a value set on client side, although it's logged on server side too + correlation_id=request_context.headers.get( + "x-ms-correlation-id" + ), # this is a value set on client side, although it's logged on server side too + client_request_id=request_context.headers.get( + "x-ms-client-request-id" + ), # this is a value set on client side, although it's logged on server side too service_request_id=request_id, traceparent=traceparent, body_excerpt=body_excerpt, From 8ebd08007eab2dbea7887fdd8a68504b0634eabb Mon Sep 17 00:00:00 2001 From: Max Wang Date: Thu, 18 Dec 2025 17:14:41 -0800 Subject: [PATCH 10/13] update test based on new update --- tests/unit/core/test_http_errors.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index af43958..1ae04c8 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -179,10 +179,10 @@ def test_correlation_id_shared_inside_call_scope(): assert len(recorder.recorded_headers) == 2 h1, h2 = recorder.recorded_headers assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] - assert h1["x-ms-correlation-request-id"] == h2["x-ms-correlation-request-id"] + assert h1["x-ms-correlation-id"] == h2["x-ms-correlation-id"] -def test_dataverse_client_correlation_scope_uses_user_id(): +def test_dataverse_client_correlation_scope_accepts_guid_and_sets_header(): responses = [ (200, {}, {"value": []}), (200, {}, {"value": []}), @@ -192,13 +192,14 @@ def test_dataverse_client_correlation_scope_uses_user_id(): recorder = RecordingHTTP(responses) mock._http = recorder client._odata = mock - with client.correlation_scope("custom-correlation-id"): + guid = "2f3cbe8f-2d3d-4f0a-9bb2-1c9a1f8f0b1b" + with client.correlation_scope(guid): mock._request("get", mock.api + "/accounts") mock._request("get", mock.api + "/accounts") assert len(recorder.recorded_headers) == 2 h1, h2 = recorder.recorded_headers - assert h1["x-ms-correlation-request-id"] == "custom-correlation-id" - assert h2["x-ms-correlation-request-id"] == "custom-correlation-id" + assert h1["x-ms-correlation-id"] == guid + assert h2["x-ms-correlation-id"] == guid def test_correlation_scope_rejects_blank_identifier(): @@ -206,3 +207,10 @@ def test_correlation_scope_rejects_blank_identifier(): with pytest.raises(ValueError): with client.correlation_scope(" "): pass + + +def test_correlation_scope_rejects_non_guid_identifier(): + client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) + with pytest.raises(ValueError): + with client.correlation_scope("not-a-guid"): + pass From 714953aab58f13c5de45397d094b259538d96ffc Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 19 Dec 2025 12:52:34 -0800 Subject: [PATCH 11/13] remove functionality for user to specify correlation scope --- README.md | 19 ------------- src/PowerPlatform/Dataverse/client.py | 33 ---------------------- src/PowerPlatform/Dataverse/data/_odata.py | 14 ++------- tests/unit/core/test_http_errors.py | 32 --------------------- 4 files changed, 3 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 5bc229a..edfd413 100644 --- a/README.md +++ b/README.md @@ -300,25 +300,6 @@ except ValidationError as e: print(f"Validation error: {e.message}") ``` -### Correlate multiple requests to debug - -Give your own identifier to every HTTP call by wrapping operations in -`DataverseClient.correlation_scope()`: - -```python -from uuid import uuid4 - -with client.correlation_scope(str(uuid4())): - client.create("account", {"name": "Scoped Request"}) - pages = client.get("account", filter="statecode eq 0") - for batch in pages: - ... -``` - -All nested SDK calls inside the block (including pagination and retries) reuse -the provided value for the `x-ms-correlation-request-id` header, which makes it -easy to align Dataverse traces. If you omit the context manager, the SDK automatically generates unique correlation IDs. - ### Authentication issues **Common fixes:** diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 68af39c..a9ba24c 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -698,37 +698,4 @@ def flush_cache(self, kind) -> int: with self._scoped_odata() as od: return od._flush_cache(kind) - # Other utilities - @contextmanager - def correlation_scope(self, correlation_id: str) -> Iterator["DataverseClient"]: - """Share a caller-specified correlation id across nested SDK calls. - - Use this context manager to stamp your own identifier on every Dataverse - request made within the ``with`` block. Nested SDK calls reuse the - existing correlation id, and concurrent scopes remain isolated. - - :param correlation_id: Non-empty identifier to propagate to - ``x-ms-correlation-request-id``. - :type correlation_id: :class:`str` - :raises TypeError: If ``correlation_id`` is not a string. - :raises ValueError: If ``correlation_id`` is empty after trimming. - - Example:: - - with client.correlation_scope("6f187988-5fb4-4bd2-9f25-4d7a1c9e24ce"): - client.create("account", {"name": "Scoped Run"}) - for batch in client.get("account", filter="statecode eq 0"): - ... - """ - - if not isinstance(correlation_id, str): - raise TypeError("correlation_id must be str") - trimmed = correlation_id.strip() - if not trimmed: - raise ValueError("correlation_id cannot be empty") - od = self._get_odata() - with od._call_scope(trimmed): - yield self - - __all__ = ["DataverseClient"] diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 440d9b4..ea31892 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -154,17 +154,9 @@ def __init__( self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL @contextmanager - def _call_scope(self, correlation_id: Optional[str] = None): - """Context manager to share a correlation id across nested SDK calls.""" - existing = _CALL_SCOPE_CORRELATION_ID.get() - if correlation_id is not None: - if not (_GUID_RE.fullmatch(correlation_id)): - raise ValueError("correlation_id provided must be a GUID string") - shared_id = correlation_id - elif existing is not None: - shared_id = existing - else: - shared_id = str(uuid.uuid4()) + def _call_scope(self): + """Context manager to generate a new correlation id for each SDK call scope.""" + shared_id = str(uuid.uuid4()) token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) try: yield shared_id diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 1ae04c8..392525c 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -182,35 +182,3 @@ def test_correlation_id_shared_inside_call_scope(): assert h1["x-ms-correlation-id"] == h2["x-ms-correlation-id"] -def test_dataverse_client_correlation_scope_accepts_guid_and_sets_header(): - responses = [ - (200, {}, {"value": []}), - (200, {}, {"value": []}), - ] - client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) - mock = MockClient([]) - recorder = RecordingHTTP(responses) - mock._http = recorder - client._odata = mock - guid = "2f3cbe8f-2d3d-4f0a-9bb2-1c9a1f8f0b1b" - with client.correlation_scope(guid): - mock._request("get", mock.api + "/accounts") - mock._request("get", mock.api + "/accounts") - assert len(recorder.recorded_headers) == 2 - h1, h2 = recorder.recorded_headers - assert h1["x-ms-correlation-id"] == guid - assert h2["x-ms-correlation-id"] == guid - - -def test_correlation_scope_rejects_blank_identifier(): - client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) - with pytest.raises(ValueError): - with client.correlation_scope(" "): - pass - - -def test_correlation_scope_rejects_non_guid_identifier(): - client = DataverseClient("https://org.example", DummyCredential(), DataverseConfig()) - with pytest.raises(ValueError): - with client.correlation_scope("not-a-guid"): - pass From e56e9bf9aceb097375b474a2573fca98e461c483 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 19 Dec 2025 13:16:01 -0800 Subject: [PATCH 12/13] remove extra whitespace --- tests/unit/core/test_http_errors.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 392525c..729ebae 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -180,5 +180,3 @@ def test_correlation_id_shared_inside_call_scope(): h1, h2 = recorder.recorded_headers assert h1["x-ms-client-request-id"] != h2["x-ms-client-request-id"] assert h1["x-ms-correlation-id"] == h2["x-ms-correlation-id"] - - From eb70efd507c2b5b66f84cda9344ab1d52b91643a Mon Sep 17 00:00:00 2001 From: Max Wang Date: Fri, 19 Dec 2025 13:19:13 -0800 Subject: [PATCH 13/13] black format --- src/PowerPlatform/Dataverse/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index a9ba24c..e59ad26 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -698,4 +698,5 @@ def flush_cache(self, kind) -> int: with self._scoped_odata() as od: return od._flush_cache(kind) + __all__ = ["DataverseClient"]