diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 47abe22..57fd173 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -28,7 +28,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 black build - if [ -f dev_dependencies.txt ]; then pip install -r dev_dependencies.txt; fi + python -m pip install -e .[dev] - name: Check format with black continue-on-error: true # TODO: fix detected formatting errors and remove this line. diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ade441..8828d4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,20 +2,42 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] +## [0.1.0b1] - 2025-11-14 ### Added -- Initial SDK implementation with CRUD operations -- Service principal authentication support -- Interactive browser authentication support -- SQL query execution via `query_sql()` -- File upload capabilities -- Pandas integration for query results -- Structured error handling with specific exception types -- GitHub Actions CI pipeline for automated testing +**Initial beta release** of Microsoft Dataverse SDK for Python + +**Core Client & Authentication:** +- Core `DataverseClient` with Azure Identity authentication support +- Secure authentication using Azure Identity credentials (Service Principal, Managed Identity, Interactive Browser) +- TLS 1.2+ encryption for all API communications +- Proper credential handling without exposing secrets in logs + +**Data Operations:** +- Complete CRUD operations (create, read, update, delete) for Dataverse records +- Advanced OData query support with filtering, sorting, and expansion +- SQL query execution via `query_sql()` method with result pagination +- Support for batch operations and transaction handling +- File upload capabilities for file and image columns + +**Table Management:** +- Table metadata operations (create, inspect, delete custom tables) + +**Integration & Analysis:** +- Pandas DataFrame integration for seamless data analysis workflows + +**Reliability & Error Handling:** +- Comprehensive error handling with specific exception types (`DataverseError`, `AuthenticationError`, etc.) +- HTTP retry logic with exponential backoff for resilient operations + +**Developer Experience:** +- Example scripts demonstrating common integration patterns +- Complete documentation with quickstart guides and API reference +- Modern Python packaging using `pyproject.toml` configuration + +**Quality Assurance:** +- Comprehensive test suite with unit and integration tests +- GitHub Actions CI/CD pipeline for automated testing and validation - Azure DevOps PR validation pipeline ### Changed @@ -32,42 +54,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security - N/A - -## [0.1.0] - TBD - -### Added -- First alpha release -- Core Dataverse client with authentication -- Basic CRUD operations (create, get, update, delete) -- OData query support -- SQL query support -- Error handling framework -- Example scripts for common scenarios - ---- - -## Release Notes Template - -When creating a new release, copy this template: - -```markdown -## [X.Y.Z] - YYYY-MM-DD - -### Added -- New features - -### Changed -- Changes in existing functionality - -### Deprecated -- Soon-to-be removed features - -### Removed -- Removed features - -### Fixed -- Bug fixes - -### Security -- Security improvements or fixes -``` diff --git a/README.md b/README.md index 909f2cd..055e31a 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A Python client library for Microsoft Dataverse that provides a unified interface for CRUD operations, SQL queries, table metadata management, and file uploads through the Dataverse Web API. -**[Source code](https://github.com/microsoft/PowerPlatform-DataverseClient-Python)** | **[Package (PyPI)](https://pypi.org/project/PowerPlatform-Dataverse-Client/)** | **[API reference documentation](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/tree/main/examples)** | **[Product documentation](https://learn.microsoft.com/power-apps/developer/data-platform/)** | **[Samples](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/tree/main/examples)** +**[Source code](https://github.com/microsoft/PowerPlatform-DataverseClient-Python)** | **[Package (PyPI)](https://pypi.org/project/PowerPlatform-Dataverse-Client/)** | **[API reference documentation](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/sdk-python/)** | **[Product documentation](https://learn.microsoft.com/en-us/python/api/dataverse-sdk-docs-python/dataverse-overview?view=dataverse-sdk-python-latest/)** | **[Samples](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/tree/main/examples)** > [!IMPORTANT] > This library is currently in **preview**. Preview versions are provided for early access to new features and may contain breaking changes. diff --git a/SUPPORT.md b/SUPPORT.md index 291d4d4..ea95202 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,25 +1,27 @@ -# TODO: The maintainer of this repo has not yet edited this file +# Support -**REPO OWNER**: Do you want Customer Service & Support (CSS) support for this product/project? +## How to file issues and get help -- **No CSS support:** Fill out this template with information about how to file issues and get help. -- **Yes CSS support:** Fill out an intake form at [aka.ms/onboardsupport](https://aka.ms/onboardsupport). CSS will work with/help you to determine next steps. -- **Not sure?** Fill out an intake as though the answer were "Yes". CSS will help you decide. +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or +feature request as a new Issue. -*Then remove this first heading from this SUPPORT.MD file before publishing your repo.* +### Getting Help -# Support +For help and questions about using the Microsoft Dataverse SDK for Python: -## How to file issues and get help +- **Documentation**: Check the [README](README.md) for quickstart guides and examples +- **GitHub Issues**: [File an issue](https://github.com/microsoft/PowerPlatform-DataverseClient-Python/issues) for bugs or feature requests -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or -feature request as a new Issue. +### Reporting Security Issues + +Security issues should be reported privately via the [Microsoft Security Response Center (MSRC)](https://aka.ms/opensource/security/msrc) or by emailing [secure@microsoft.com](mailto:secure@microsoft.com). Please do not report security vulnerabilities through public GitHub issues. + +## Microsoft Support Policy -For help and questions about using this project, please **REPO MAINTAINER: INSERT INSTRUCTIONS HERE -FOR HOW TO ENGAGE REPO OWNERS OR COMMUNITY FOR HELP. COULD BE A STACK OVERFLOW TAG OR OTHER -CHANNEL. WHERE WILL YOU HELP PEOPLE?**. +This is a community-supported project. Support for the Microsoft Dataverse SDK for Python is provided on a best-effort basis through: -## Microsoft Support Policy +- Community contributions via GitHub Issues and Pull Requests +- Documentation and examples in this repository -Support for this **PROJECT or PRODUCT** is limited to the resources listed above. +This project is not covered by Microsoft's standard product support services. For issues with Microsoft Dataverse itself (not this SDK), please use the official Microsoft support channels. diff --git a/dev_dependencies.txt b/dev_dependencies.txt deleted file mode 100644 index 1c10927..0000000 --- a/dev_dependencies.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Development dependencies for dataverse-client-python -# Install with: pip install -r dev_dependencies.txt - -pytest>=8.3.1 -pytest-asyncio>=0.21 -pytest-mock>=3.0 -black>=23.0 -flake8>=6.0 -build>=0.10 -twine>=4.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 50faeb0..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -azure-identity>=1.17.0 -azure-core>=1.30.2 -msal>=1.28.0 -requests>=2.32.0 -pyodbc>=5.1.0 -pytest>=8.3.1 diff --git a/src/PowerPlatform/Dataverse/core/__init__.py b/src/PowerPlatform/Dataverse/core/__init__.py index ccf2a6e..1a136b3 100644 --- a/src/PowerPlatform/Dataverse/core/__init__.py +++ b/src/PowerPlatform/Dataverse/core/__init__.py @@ -8,25 +8,4 @@ configuration, HTTP client, and error handling. """ -from .auth import AuthManager, TokenPair -from .config import DataverseConfig -from .errors import ( - DataverseError, - HttpError, - ValidationError, - MetadataError, - SQLParseError, -) -from .http import HttpClient - -__all__ = [ - "AuthManager", - "TokenPair", - "DataverseConfig", - "DataverseError", - "HttpError", - "ValidationError", - "MetadataError", - "SQLParseError", - "HttpClient", -] \ No newline at end of file +__all__ = [] \ No newline at end of file diff --git a/src/PowerPlatform/Dataverse/core/errors.py b/src/PowerPlatform/Dataverse/core/errors.py index 28c535f..ef810a8 100644 --- a/src/PowerPlatform/Dataverse/core/errors.py +++ b/src/PowerPlatform/Dataverse/core/errors.py @@ -53,7 +53,7 @@ def __init__( self.details = details or {} self.source = source or "client" self.is_transient = is_transient - self.timestamp = _dt.datetime.utcnow().isoformat() + "Z" + self.timestamp = _dt.datetime.now(_dt.timezone.utc).isoformat().replace('+00:00', 'Z') def to_dict(self) -> Dict[str, Any]: """ diff --git a/src/PowerPlatform/Dataverse/data/__init__.py b/src/PowerPlatform/Dataverse/data/__init__.py index a5854b8..86a3659 100644 --- a/src/PowerPlatform/Dataverse/data/__init__.py +++ b/src/PowerPlatform/Dataverse/data/__init__.py @@ -8,7 +8,4 @@ SQL query functionality, and file upload capabilities. """ -from .odata import ODataClient -from .upload import ODataFileUpload - -__all__ = ["ODataClient", "ODataFileUpload"] \ No newline at end of file +__all__ = [] \ No newline at end of file diff --git a/src/PowerPlatform/Dataverse/data/odata.py b/src/PowerPlatform/Dataverse/data/odata.py index bdcff06..17b2194 100644 --- a/src/PowerPlatform/Dataverse/data/odata.py +++ b/src/PowerPlatform/Dataverse/data/odata.py @@ -17,7 +17,18 @@ from ..core.http import HttpClient from .upload import ODataFileUpload from ..core.errors import * -from ..core import error_codes as ec +from ..core.error_codes import ( + http_subcode, + is_transient_status, + VALIDATION_SQL_NOT_STRING, + VALIDATION_SQL_EMPTY, + METADATA_ENTITYSET_NOT_FOUND, + METADATA_ENTITYSET_NAME_MISSING, + METADATA_TABLE_NOT_FOUND, + METADATA_TABLE_ALREADY_EXISTS, + METADATA_COLUMN_NOT_FOUND, + VALIDATION_UNSUPPORTED_CACHE_KIND, +) from ..__version__ import __version__ as _SDK_VERSION @@ -121,7 +132,7 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 except Exception: pass sc = r.status_code - subcode = ec.http_subcode(sc) + subcode = http_subcode(sc) correlation_id = headers.get("x-ms-correlation-request-id") or headers.get("x-ms-correlation-id") request_id = headers.get("x-ms-client-request-id") or headers.get("request-id") or headers.get("x-ms-request-id") traceparent = headers.get("traceparent") @@ -132,7 +143,7 @@ def _request(self, method: str, url: str, *, expected: tuple[int, ...] = (200, 2 retry_after = int(ra) except Exception: retry_after = None - is_transient = ec.is_transient_status(sc) + is_transient = is_transient_status(sc) raise HttpError( msg, status_code=sc, @@ -558,9 +569,9 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]: Endpoint form: ``GET /{entity_set}?sql=``. The client extracts the logical table name, resolves the entity set (metadata cached), then issues the request. Only a constrained SELECT subset is supported by the platform. """ if not isinstance(sql, str): - raise ValidationError("sql must be a string", subcode=ec.VALIDATION_SQL_NOT_STRING) + raise ValidationError("sql must be a string", subcode=VALIDATION_SQL_NOT_STRING) if not sql.strip(): - raise ValidationError("sql must be a non-empty string", subcode=ec.VALIDATION_SQL_EMPTY) + raise ValidationError("sql must be a non-empty string", subcode=VALIDATION_SQL_EMPTY) sql = sql.strip() # Extract logical table name via helper (robust to identifiers ending with 'from') @@ -631,14 +642,14 @@ def _entity_set_from_logical(self, logical: str) -> str: plural_hint = " (did you pass a plural entity set name instead of the singular logical name?)" if logical.endswith("s") and not logical.endswith("ss") else "" raise MetadataError( f"Unable to resolve entity set for logical name '{logical}'. Provide the singular logical name.{plural_hint}", - subcode=ec.METADATA_ENTITYSET_NOT_FOUND, + subcode=METADATA_ENTITYSET_NOT_FOUND, ) md = items[0] es = md.get("EntitySetName") if not es: raise MetadataError( f"Metadata response missing EntitySetName for logical '{logical}'.", - subcode=ec.METADATA_ENTITYSET_NAME_MISSING, + subcode=METADATA_ENTITYSET_NAME_MISSING, ) self._logical_to_entityset_cache[logical] = es primary_id_attr = md.get("PrimaryIdAttribute") @@ -1150,7 +1161,7 @@ def _delete_table(self, tablename: str) -> None: if not ent or not ent.get("MetadataId"): raise MetadataError( f"Table '{entity_schema}' not found.", - subcode=ec.METADATA_TABLE_NOT_FOUND, + subcode=METADATA_TABLE_NOT_FOUND, ) metadata_id = ent["MetadataId"] url = f"{self.api}/EntityDefinitions({metadata_id})" @@ -1191,7 +1202,7 @@ def _create_table( if ent: raise MetadataError( f"Table '{entity_schema}' already exists.", - subcode=ec.METADATA_TABLE_ALREADY_EXISTS, + subcode=METADATA_TABLE_ALREADY_EXISTS, ) created_cols: List[str] = [] @@ -1254,7 +1265,7 @@ def _create_columns( if not ent or not ent.get("MetadataId"): raise MetadataError( f"Table '{entity_schema}' not found.", - subcode=ec.METADATA_TABLE_NOT_FOUND, + subcode=METADATA_TABLE_NOT_FOUND, ) metadata_id = ent.get("MetadataId") @@ -1317,7 +1328,7 @@ def _delete_columns( if not ent or not ent.get("MetadataId"): raise MetadataError( f"Table '{entity_schema}' not found.", - subcode=ec.METADATA_TABLE_NOT_FOUND, + subcode=METADATA_TABLE_NOT_FOUND, ) metadata_id = ent.get("MetadataId") @@ -1330,7 +1341,7 @@ def _delete_columns( if not attr_meta: raise MetadataError( f"Column '{schema_name}' not found on table '{entity_schema}'.", - subcode=ec.METADATA_COLUMN_NOT_FOUND, + subcode=METADATA_COLUMN_NOT_FOUND, ) attr_metadata_id = attr_meta.get("MetadataId") @@ -1372,7 +1383,7 @@ def _flush_cache( if k != "picklist": raise ValidationError( f"Unsupported cache kind '{kind}' (only 'picklist' is implemented)", - subcode=ec.VALIDATION_UNSUPPORTED_CACHE_KIND, + subcode=VALIDATION_UNSUPPORTED_CACHE_KIND, ) removed = len(self._picklist_label_cache) diff --git a/src/PowerPlatform/Dataverse/utils/__init__.py b/src/PowerPlatform/Dataverse/utils/__init__.py index 2f02dfb..d12c4e0 100644 --- a/src/PowerPlatform/Dataverse/utils/__init__.py +++ b/src/PowerPlatform/Dataverse/utils/__init__.py @@ -7,6 +7,4 @@ This module contains adapters (like Pandas integration). """ -from .pandas_adapter import PandasODataClient - -__all__ = ["PandasODataClient"] \ No newline at end of file +__all__ = [] \ No newline at end of file diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index e1fffa9..f3fd29f 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -3,7 +3,7 @@ import pytest from PowerPlatform.Dataverse.core.errors import HttpError -from PowerPlatform.Dataverse.core import error_codes as ec +from PowerPlatform.Dataverse.core.error_codes import HTTP_404, HTTP_429, HTTP_500 from PowerPlatform.Dataverse.data.odata import ODataClient class DummyAuth: @@ -34,7 +34,7 @@ def json_fail(): raise ValueError("non-json") r.json = json_fail return r -class TestClient(ODataClient): +class MockClient(ODataClient): def __init__(self, responses): super().__init__(DummyAuth(), "https://org.example", None) self._http = DummyHTTP(responses) @@ -47,11 +47,11 @@ def test_http_404_subcode_and_service_code(): {"x-ms-correlation-request-id": "cid1"}, {"error": {"code": "0x800404", "message": "Not found"}}, )] - c = TestClient(responses) + c = MockClient(responses) with pytest.raises(HttpError) as ei: c._request("get", c.api + "/accounts(abc)") err = ei.value.to_dict() - assert err["subcode"] == ec.HTTP_404 + assert err["subcode"] == HTTP_404 assert err["details"]["service_error_code"] == "0x800404" @@ -61,12 +61,12 @@ def test_http_429_transient_and_retry_after(): {"Retry-After": "7"}, {"error": {"message": "Throttle"}}, )] - c = TestClient(responses) + c = MockClient(responses) with pytest.raises(HttpError) as ei: c._request("get", c.api + "/accounts") err = ei.value.to_dict() assert err["is_transient"] is True - assert err["subcode"] == ec.HTTP_429 + assert err["subcode"] == HTTP_429 assert err["details"]["retry_after"] == 7 @@ -76,11 +76,11 @@ def test_http_500_body_excerpt(): {}, "Internal failure XYZ stack truncated", )] - c = TestClient(responses) + c = MockClient(responses) with pytest.raises(HttpError) as ei: c._request("get", c.api + "/accounts") err = ei.value.to_dict() - assert err["subcode"] == ec.HTTP_500 + assert err["subcode"] == HTTP_500 assert "XYZ stack" in err["details"]["body_excerpt"] @@ -90,7 +90,7 @@ def test_http_non_mapped_status_code_subcode_fallback(): {}, {"error": {"message": "Teapot"}}, )] - c = TestClient(responses) + c = MockClient(responses) with pytest.raises(HttpError) as ei: c._request("get", c.api + "/accounts") err = ei.value.to_dict() diff --git a/tests/unit/data/test_logical_crud.py b/tests/unit/data/test_logical_crud.py index 78280d0..ccebefc 100644 --- a/tests/unit/data/test_logical_crud.py +++ b/tests/unit/data/test_logical_crud.py @@ -34,7 +34,7 @@ def json_func(): resp.json = json_func return resp -class TestableClient(ODataClient): +class MockableClient(ODataClient): def __init__(self, responses): super().__init__(DummyAuth(), "https://org.example", None) self._http = DummyHTTPClient(responses) @@ -76,7 +76,7 @@ def test_single_create_update_delete_get(): (204, {}, {}), # update (no body) (204, {}, {}), # delete ] - c = TestableClient(responses) + c = MockableClient(responses) entity_set = c._entity_set_from_logical("account") rid = c._create(entity_set, "account", {"name": "Acme"}) assert rid == guid @@ -96,7 +96,7 @@ def test_bulk_create_and_update(): (204, {}, {}), # UpdateMultiple broadcast (204, {}, {}), # UpdateMultiple 1:1 ] - c = TestableClient(responses) + c = MockableClient(responses) entity_set = c._entity_set_from_logical("account") ids = c._create_multiple(entity_set, "account", [{"name": "A"}, {"name": "B"}]) assert ids == [g1, g2] @@ -111,7 +111,7 @@ def test_get_multiple_paging(): (200, {}, {"value": [{"accountid": "1"}], "@odata.nextLink": "https://org.example/api/data/v9.2/accounts?$skip=1"}), (200, {}, {"value": [{"accountid": "2"}]}), ] - c = TestableClient(responses) + c = MockableClient(responses) pages = list(c._get_multiple("account", select=["accountid"], page_size=1)) assert pages == [[{"accountid": "1"}], [{"accountid": "2"}]] @@ -120,6 +120,6 @@ def test_unknown_logical_name_raises(): responses = [ (200, {}, {"value": []}), # metadata lookup returns empty ] - c = TestableClient(responses) + c = MockableClient(responses) with pytest.raises(MetadataError): c._entity_set_from_logical("nonexistent") \ No newline at end of file