diff --git a/src/_example/django/django_demo/app/flask_models.py b/src/_example/django/django_demo/app/flask_models.py index 367c56a13..01dc28a17 100644 --- a/src/_example/django/django_demo/app/flask_models.py +++ b/src/_example/django/django_demo/app/flask_models.py @@ -23,7 +23,7 @@ class Meta: class FlaskCustomer(models.Model): - id = models.BinaryField(primary_key=True, db_column="pk") + id = models.UUIDField(primary_key=True, db_column="pk") first_name = models.CharField(max_length=255) last_name = models.CharField(max_length=255) birthday_date = models.DateTimeField(blank=True, null=True) diff --git a/src/_example/django/django_demo/app/forest/custom_datasources/typicode.py b/src/_example/django/django_demo/app/forest/custom_datasources/typicode.py index a353ee223..6d80413d0 100644 --- a/src/_example/django/django_demo/app/forest/custom_datasources/typicode.py +++ b/src/_example/django/django_demo/app/forest/custom_datasources/typicode.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Self +from typing import Any, Dict, List from aiohttp import ClientSession from forestadmin.agent_toolkit.utils.context import User @@ -10,6 +10,7 @@ from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter from forestadmin.datasource_toolkit.interfaces.query.filter.unpaginated import Filter from forestadmin.datasource_toolkit.interfaces.query.projections import Projection +from typing_extensions import Self class TypicodeCollection(Collection): diff --git a/src/_example/django/django_demo/app/forest/customer.py b/src/_example/django/django_demo/app/forest/customer.py index 3a32ca7cb..9879778cd 100644 --- a/src/_example/django/django_demo/app/forest/customer.py +++ b/src/_example/django/django_demo/app/forest/customer.py @@ -314,12 +314,12 @@ async def time_order_number_chart( async def order_details(context: CollectionChartContext, result_builder: ResultBuilderChart, ids: CompositeIdAlias): - orders = await context.datasource.get_collection("order").list( + orders = await context.datasource.get_collection("app_order").list( context.caller, PaginatedFilter( {"condition_tree": ConditionTreeLeaf("customer_id", "in", ids)}, ), - Projection("id", "customer_full_name"), + Projection("id", "customer_full_name", "amount"), ) return result_builder.smart(orders) diff --git a/src/_example/django/django_demo/app/forest/order.py b/src/_example/django/django_demo/app/forest/order.py index ae9a2dc0a..5663ad1d1 100644 --- a/src/_example/django/django_demo/app/forest/order.py +++ b/src/_example/django/django_demo/app/forest/order.py @@ -1,5 +1,6 @@ import io import json +from datetime import date from typing import List from app.models import Order @@ -184,14 +185,18 @@ async def refund_order_execute(context: ActionContextSingle, result_builder: Res # charts async def total_order_chart(context: AgentCustomizationContext, result_builder: ResultBuilderChart): - records = await context.datasource.get_collection("app_order").list(context.caller, PaginatedFilter({}), ["id"]) - return result_builder.value(len(records)) + # records = await context.datasource.get_collection("app_order").list(context.caller, PaginatedFilter({}), ["id"]) + # return result_builder.value(len(records)) + record = await context.datasource.get_collection("app_order").aggregate( + context.caller, Filter({}), Aggregation({"operation": "Count"}) + ) + return result_builder.value(record[0]["value"]) async def nb_order_per_week(context: AgentCustomizationContext, result_builder: ResultBuilderChart): records = await context.datasource.get_collection("app_order").aggregate( context.caller, - Filter({"condition_tree": ConditionTreeLeaf("created_at", "before", "2022-01-01")}), + Filter({"condition_tree": ConditionTreeLeaf("created_at", "before", date.today())}), Aggregation( { "field": "created_at", diff --git a/src/_example/django/django_demo/app/management/commands/sqlalchemy_init.py b/src/_example/django/django_demo/app/management/commands/sqlalchemy_init.py index 6e3c95367..f398af0aa 100644 --- a/src/_example/django/django_demo/app/management/commands/sqlalchemy_init.py +++ b/src/_example/django/django_demo/app/management/commands/sqlalchemy_init.py @@ -78,7 +78,7 @@ def _populate_addresses(session, customers: List[Customer]) -> List[Address]: for _ in range(0, 1000): address = Address( - street=fake.street_address(), city=fake.city(), country=fake.country(), zip_code=fake.postcode() + street=fake.street_address(), city=fake.city(), country=fake.country(), zip_code=fr_fake.postcode() ) known_customer: Set[Customer] = set() for _ in range(1, random.randint(2, 4)): diff --git a/src/_example/django/poetry.lock b/src/_example/django/poetry.lock index 8d431ca83..6abe6b532 100644 --- a/src/_example/django/poetry.lock +++ b/src/_example/django/poetry.lock @@ -733,7 +733,7 @@ pyflakes = ">=2.5.0,<2.6.0" [[package]] name = "forestadmin-agent-django" -version = "1.22.3" +version = "1.22.5" description = "django agent for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -744,8 +744,8 @@ develop = true "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} django = ">=3.2,<5.2" django-cors-headers = ">=3.8" -forestadmin-agent-toolkit = "1.22.3" -forestadmin-datasource-django = "1.22.3" +forestadmin-agent-toolkit = "1.22.5" +forestadmin-datasource-django = "1.22.5" typing-extensions = "~=4.2" [package.source] @@ -754,7 +754,7 @@ url = "../../django_agent" [[package]] name = "forestadmin-agent-toolkit" -version = "1.22.3" +version = "1.22.5" description = "agent toolkit for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -765,11 +765,10 @@ develop = true aiohttp = "~=3.9" "backports.zoneinfo" = {version = "~0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} cachetools = "~=5.2" -forestadmin-datasource-toolkit = "1.22.3" -marshmallow-jsonapi = ">=0.24.0, <1.0" +forestadmin-datasource-toolkit = "1.22.5" numpy = [ - {version = ">=1.26.0,<2.0.0", markers = "python_full_version >= \"3.12.0\""}, - {version = "<2.0.0", markers = "python_full_version < \"3.12.0\""}, + {version = ">=1.24.0", markers = "python_full_version >= \"3.8.0\" and python_version < \"3.12\""}, + {version = ">=1.3.0", markers = "python_version >= \"3.13\""}, ] oic = "~=1.4" pandas = [ @@ -786,7 +785,7 @@ url = "../../agent_toolkit" [[package]] name = "forestadmin-datasource-django" -version = "1.22.3" +version = "1.22.5" description = "django datasource for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -796,8 +795,8 @@ develop = true [package.dependencies] "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} django = ">=3.2,<5.2" -forestadmin-agent-toolkit = "1.22.3" -forestadmin-datasource-toolkit = "1.22.3" +forestadmin-agent-toolkit = "1.22.5" +forestadmin-datasource-toolkit = "1.22.5" typing-extensions = "~=4.2" [package.source] @@ -806,7 +805,7 @@ url = "../../datasource_django" [[package]] name = "forestadmin-datasource-sqlalchemy" -version = "1.22.3" +version = "1.22.5" description = "sqlalchemy datasource for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -815,8 +814,8 @@ develop = true [package.dependencies] "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} -forestadmin-agent-toolkit = "1.22.3" -forestadmin-datasource-toolkit = "1.22.3" +forestadmin-agent-toolkit = "1.22.5" +forestadmin-datasource-toolkit = "1.22.5" sqlalchemy = ">=1.4.0" typing-extensions = "~=4.2" @@ -826,7 +825,7 @@ url = "../../datasource_sqlalchemy" [[package]] name = "forestadmin-datasource-toolkit" -version = "1.22.3" +version = "1.22.5" description = "datasource toolkit for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -837,8 +836,8 @@ develop = true "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} filetype = "^1.0.0" numpy = [ - {version = ">=1.26.0,<2.0.0", markers = "python_full_version >= \"3.12.0\""}, - {version = "<2.0.0", markers = "python_full_version < \"3.12.0\""}, + {version = ">=1.24.0", markers = "python_full_version >= \"3.8.0\" and python_version < \"3.12\""}, + {version = ">=1.3.0", markers = "python_version >= \"3.13\""}, ] pandas = [ {version = ">=1.4.0", markers = "python_full_version < \"3.13.0\""}, @@ -1190,44 +1189,6 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] -[[package]] -name = "marshmallow" -version = "3.22.0" -description = "A lightweight library for converting complex datatypes to and from native Python datatypes." -optional = false -python-versions = ">=3.8" -files = [ - {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, - {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, -] - -[package.dependencies] -packaging = ">=17.0" - -[package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] -tests = ["pytest", "pytz", "simplejson"] - -[[package]] -name = "marshmallow-jsonapi" -version = "0.24.0" -description = "JSON API 1.0 (https://jsonapi.org) formatting with marshmallow" -optional = false -python-versions = ">=3.6" -files = [ - {file = "marshmallow-jsonapi-0.24.0.tar.gz", hash = "sha256:bd88c0ac0e2ddeb0a3ceb86229963b9f828d898041f29d92a68f585a1feb37b5"}, - {file = "marshmallow_jsonapi-0.24.0-py2.py3-none-any.whl", hash = "sha256:b7403688297dfe8b89173582811989badbe1328ac36447c5a151c006fbe34d24"}, -] - -[package.dependencies] -marshmallow = ">=2.15.2" - -[package.extras] -dev = ["Flask (==1.1.2)", "faker (==4.18.0)", "flake8 (==3.8.4)", "flake8-bugbear (==20.11.1)", "mock", "pre-commit (>=2.0,<3.0)", "pytest", "tox"] -lint = ["flake8 (==3.8.4)", "flake8-bugbear (==20.11.1)", "pre-commit (>=2.0,<3.0)"] -tests = ["Flask (==1.1.2)", "faker (==4.18.0)", "mock", "pytest"] - [[package]] name = "mccabe" version = "0.7.0" @@ -1527,9 +1488,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.20.3", markers = "python_version < \"3.10\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" diff --git a/src/_example/flask_sqlalchemy_package/poetry.lock b/src/_example/flask_sqlalchemy_package/poetry.lock index b1c5e7e24..58f6f990f 100644 --- a/src/_example/flask_sqlalchemy_package/poetry.lock +++ b/src/_example/flask_sqlalchemy_package/poetry.lock @@ -759,7 +759,7 @@ sqlalchemy = ">=2.0.16" [[package]] name = "forestadmin-agent-flask" -version = "1.20.0" +version = "1.22.5" description = "flask agent for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -770,8 +770,8 @@ develop = true "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} flask = {version = ">=2.0.0", extras = ["async"]} flask-cors = ">=3.0.0" -forestadmin-agent-toolkit = "1.20.0" -forestadmin-datasource-toolkit = "1.20.0" +forestadmin-agent-toolkit = "1.22.5" +forestadmin-datasource-toolkit = "1.22.5" typing-extensions = "~=4.2" [package.source] @@ -780,7 +780,7 @@ url = "../../flask_agent" [[package]] name = "forestadmin-agent-toolkit" -version = "1.20.0" +version = "1.22.5" description = "agent toolkit for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -791,11 +791,10 @@ develop = true aiohttp = "~=3.9" "backports.zoneinfo" = {version = "~0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} cachetools = "~=5.2" -forestadmin-datasource-toolkit = "1.20.0" -marshmallow-jsonapi = ">=0.24.0, <1.0" +forestadmin-datasource-toolkit = "1.22.5" numpy = [ - {version = "<2.0.0", markers = "python_full_version < \"3.12.0\""}, - {version = ">=1.26.0,<2.0.0", markers = "python_full_version >= \"3.12.0\""}, + {version = ">=1.24.0", markers = "python_full_version >= \"3.8.0\" and python_version < \"3.12\""}, + {version = ">=1.3.0", markers = "python_version >= \"3.13\""}, ] oic = "~=1.4" pandas = [ @@ -812,7 +811,7 @@ url = "../../agent_toolkit" [[package]] name = "forestadmin-datasource-sqlalchemy" -version = "1.20.0" +version = "1.22.5" description = "sqlalchemy datasource for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -821,8 +820,8 @@ develop = true [package.dependencies] "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} -forestadmin-agent-toolkit = "1.20.0" -forestadmin-datasource-toolkit = "1.20.0" +forestadmin-agent-toolkit = "1.22.5" +forestadmin-datasource-toolkit = "1.22.5" sqlalchemy = ">=1.4.0" typing-extensions = "~=4.2" @@ -832,7 +831,7 @@ url = "../../datasource_sqlalchemy" [[package]] name = "forestadmin-datasource-toolkit" -version = "1.20.0" +version = "1.22.5" description = "datasource toolkit for forestadmin python agent" optional = false python-versions = ">=3.8,<3.14" @@ -843,8 +842,8 @@ develop = true "backports.zoneinfo" = {version = "~=0.2.1", extras = ["tzdata"], markers = "python_version < \"3.9\""} filetype = "^1.0.0" numpy = [ - {version = "<2.0.0", markers = "python_full_version < \"3.12.0\""}, - {version = ">=1.26.0,<2.0.0", markers = "python_full_version >= \"3.12.0\""}, + {version = ">=1.24.0", markers = "python_full_version >= \"3.8.0\" and python_version < \"3.12\""}, + {version = ">=1.3.0", markers = "python_version >= \"3.13\""}, ] pandas = [ {version = ">=1.4.0", markers = "python_full_version < \"3.13.0\""}, @@ -1210,44 +1209,6 @@ files = [ {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, ] -[[package]] -name = "marshmallow" -version = "3.22.0" -description = "A lightweight library for converting complex datatypes to and from native Python datatypes." -optional = false -python-versions = ">=3.8" -files = [ - {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, - {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, -] - -[package.dependencies] -packaging = ">=17.0" - -[package.extras] -dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] -tests = ["pytest", "pytz", "simplejson"] - -[[package]] -name = "marshmallow-jsonapi" -version = "0.24.0" -description = "JSON API 1.0 (https://jsonapi.org) formatting with marshmallow" -optional = false -python-versions = ">=3.6" -files = [ - {file = "marshmallow-jsonapi-0.24.0.tar.gz", hash = "sha256:bd88c0ac0e2ddeb0a3ceb86229963b9f828d898041f29d92a68f585a1feb37b5"}, - {file = "marshmallow_jsonapi-0.24.0-py2.py3-none-any.whl", hash = "sha256:b7403688297dfe8b89173582811989badbe1328ac36447c5a151c006fbe34d24"}, -] - -[package.dependencies] -marshmallow = ">=2.15.2" - -[package.extras] -dev = ["Flask (==1.1.2)", "faker (==4.18.0)", "flake8 (==3.8.4)", "flake8-bugbear (==20.11.1)", "mock", "pre-commit (>=2.0,<3.0)", "pytest", "tox"] -lint = ["flake8 (==3.8.4)", "flake8-bugbear (==20.11.1)", "pre-commit (>=2.0,<3.0)"] -tests = ["Flask (==1.1.2)", "faker (==4.18.0)", "mock", "pytest"] - [[package]] name = "mccabe" version = "0.7.0" @@ -1558,8 +1519,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/agent.py b/src/agent_toolkit/forestadmin/agent_toolkit/agent.py index 4c20df3f2..199ab8fa5 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/agent.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/agent.py @@ -15,7 +15,6 @@ from forestadmin.agent_toolkit.services.permissions.ip_whitelist_service import IpWhiteListService from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService from forestadmin.agent_toolkit.services.permissions.sse_cache_invalidation import SSECacheInvalidation -from forestadmin.agent_toolkit.services.serializers.json_api import create_json_api_schema from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder from forestadmin.agent_toolkit.utils.forest_schema.emitter import SchemaEmitter from forestadmin.agent_toolkit.utils.forest_schema.type import AgentMeta @@ -235,9 +234,6 @@ async def _start(self): else: ForestLogger.log("warning", 'Schema update was skipped (caused by options["skip_schema_update"]=True)') - for collection in (await self.customizer.get_datasource()).collections: - create_json_api_schema(collection) - if self.options["instant_cache_refresh"]: self._sse_thread.start() diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_collection.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_collection.py index 5caf3aac8..061fa95d4 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_collection.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_collection.py @@ -4,7 +4,7 @@ from forestadmin.agent_toolkit.resources.collections.base_collection_resource import BaseCollectionResource from forestadmin.agent_toolkit.resources.collections.decorators import authenticate, check_method, ip_white_list from forestadmin.agent_toolkit.resources.collections.requests import RequestCollection, RequestCollectionException -from forestadmin.agent_toolkit.services.serializers import json_api +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import render_chart from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response from forestadmin.agent_toolkit.utils.id import unpack_id from forestadmin.datasource_toolkit.exceptions import ForestException @@ -40,7 +40,7 @@ async def handle_api_chart(self, request: RequestCollection) -> Response: ids = unpack_id(request.collection.schema, request.body.get("record_id") or request.query.get("record_id")) chart = await request.collection.render_chart(request.user, request.query["chart_name"], ids) - return {"data": json_api.render_chart(chart)} + return {"data": render_chart(chart)} @check_method(RequestMethod.GET) @authenticate diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_datasource.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_datasource.py index bafba8683..0d25c8c5e 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_datasource.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/charts_datasource.py @@ -3,7 +3,7 @@ from forestadmin.agent_toolkit.forest_logger import ForestLogger from forestadmin.agent_toolkit.resources.collections.base_collection_resource import BaseCollectionResource from forestadmin.agent_toolkit.resources.collections.decorators import authenticate, check_method, ip_white_list -from forestadmin.agent_toolkit.services.serializers import json_api +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import render_chart from forestadmin.agent_toolkit.utils.context import FileResponse, HttpResponseBuilder, Request, RequestMethod, Response from forestadmin.datasource_toolkit.exceptions import ForestException @@ -30,7 +30,7 @@ async def dispatch(self, request: Request, method_name: Literal["add"]) -> Union @authenticate async def handle_api_chart(self, request: Request) -> Response: chart = await self.datasource.render_chart(request.user, request.query["chart_name"]) - return {"data": json_api.render_chart(chart)} + return {"data": render_chart(chart)} @check_method(RequestMethod.GET) @authenticate diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py index 8a3a09e7d..2157700db 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py @@ -26,10 +26,12 @@ from forestadmin.agent_toolkit.services.permissions.ip_whitelist_service import IpWhiteListService from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService from forestadmin.agent_toolkit.services.serializers import add_search_metadata -from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiException +from forestadmin.agent_toolkit.services.serializers.json_api_deserializer import JsonApiDeserializer +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import JsonApiSerializer from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response, User from forestadmin.agent_toolkit.utils.csv import Csv, CsvException -from forestadmin.agent_toolkit.utils.id import unpack_id +from forestadmin.agent_toolkit.utils.id import IdException, unpack_id from forestadmin.agent_toolkit.utils.sql_query_checker import SqlQueryChecker from forestadmin.datasource_toolkit.collections import Collection from forestadmin.datasource_toolkit.datasource_customizer.collection_customizer import CollectionCustomizer @@ -124,7 +126,7 @@ async def get(self, request: RequestCollection) -> Response: return HttpResponseBuilder.build_unknown_response() try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projections, many=False ) except JsonApiException as e: @@ -138,9 +140,8 @@ async def get(self, request: RequestCollection) -> Response: @authorize("add") async def add(self, request: RequestCollection) -> Response: collection = request.collection - schema = JsonApiSerializer.get(collection) try: - data: RecordsDataAlias = schema().load(request.body) # type: ignore + data = JsonApiDeserializer(self.datasource).deserialize(request.body, collection) except JsonApiException as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) @@ -166,7 +167,7 @@ async def add(self, request: RequestCollection) -> Response: return HttpResponseBuilder.build_client_error_response([e]) return HttpResponseBuilder.build_success_response( - CrudResource._serialize_records_with_relationships( + self._serialize_records_with_relationships( records, request.collection, Projection(*list(records[0].keys())), many=False ) ) @@ -193,7 +194,7 @@ async def list(self, request: RequestCollection) -> Response: records = await request.collection.list(request.user, paginated_filter, projections) try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projections, many=True ) except JsonApiException as e: @@ -263,21 +264,21 @@ async def update(self, request: RequestCollection) -> Response: collection = request.collection try: ids = unpack_id(collection.schema, request.pks) - except (FieldValidatorException, CollectionResourceException) as e: + except (FieldValidatorException, CollectionResourceException, IdException) as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) if request.body and "data" in request.body and "relationships" in request.body["data"]: del request.body["data"]["relationships"] - schema = JsonApiSerializer.get(collection) try: # if the id change it will be in 'data.attributes', otherwise, we get the id by from the request url. request.body["data"].pop("id", None) # type: ignore - data: RecordsDataAlias = schema().load(request.body) # type: ignore + data = JsonApiDeserializer(self.datasource).deserialize(request.body, collection) except JsonApiException as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) + trees: List[ConditionTree] = [ConditionTreeFactory.match_ids(collection.schema, [ids])] scope_tree = await self.permission.get_scope(request.user, request.collection) if scope_tree: @@ -290,7 +291,7 @@ async def update(self, request: RequestCollection) -> Response: records = await collection.list(request.user, PaginatedFilter.from_base_filter(filter), projection) try: - dumped: Dict[str, Any] = CrudResource._serialize_records_with_relationships( + dumped: Dict[str, Any] = self._serialize_records_with_relationships( records, request.collection, projection, many=False ) except JsonApiException as e: @@ -428,14 +429,15 @@ async def extract_data( return record, one_to_one_relations - @staticmethod def _serialize_records_with_relationships( + self, records: List[RecordsDataAlias], collection: Union[Collection, CollectionCustomizer], projection: Projection, many: bool, ) -> Dict[str, Any]: relations_to_set = [] + projection = Projection(*projection) for name, schema in collection.schema["fields"].items(): if is_many_to_many(schema) or is_one_to_many(schema) or is_polymorphic_one_to_many(schema): pks = SchemaUtils.get_primary_keys( @@ -449,8 +451,10 @@ def _serialize_records_with_relationships( for name in relations_to_set: record[name] = None - schema = JsonApiSerializer.get(collection) - return schema(projections=projection).dump(records if many is True else records[0], many=many) + ret = JsonApiSerializer(self.datasource, projection).serialize( + records if many is True else records[0], collection + ) + return ret async def _handle_live_query_segment( self, request: RequestCollection, condition_tree: Optional[ConditionTree] diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py index 29ee9e181..f325ed5a6 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud_related.py @@ -26,8 +26,9 @@ RequestCollectionException, RequestRelationCollection, ) -from forestadmin.agent_toolkit.services.serializers import DumpedResult, add_search_metadata -from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException, JsonApiSerializer +from forestadmin.agent_toolkit.services.serializers import add_search_metadata +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiException +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import JsonApiSerializer from forestadmin.agent_toolkit.utils.context import HttpResponseBuilder, Request, RequestMethod, Response from forestadmin.agent_toolkit.utils.csv import Csv, CsvException from forestadmin.agent_toolkit.utils.id import unpack_id @@ -103,9 +104,8 @@ async def list(self, request: RequestRelationCollection) -> Response: paginated_filter, projection, ) - schema = JsonApiSerializer.get(request.foreign_collection) try: - dumped: DumpedResult = schema(projections=projection).dump(records, many=True) # type: ignore + dumped = JsonApiSerializer(self.datasource, projection).serialize(records, request.foreign_collection) except JsonApiException as e: ForestLogger.log("exception", e) return HttpResponseBuilder.build_client_error_response([e]) diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/filter.py b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/filter.py index d3815a467..e01c460bb 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/filter.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/resources/collections/filter.py @@ -57,7 +57,7 @@ class FilterException(AgentToolkitException): def _get_collection( - request: Union[RequestCollection, RequestRelationCollection] + request: Union[RequestCollection, RequestRelationCollection], ) -> Union[CollectionCustomizer, Collection]: collection = request.collection if isinstance(request, RequestRelationCollection): diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py index 9ad33ccfa..140a6114f 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/__init__.py @@ -1,18 +1,28 @@ -from typing import Any, Dict, List, Optional, TypedDict +from typing import Any, Dict, List, Union + +from typing_extensions import NotRequired, TypedDict class Data(TypedDict): type: str - relationships: Dict[str, Any] + id: int attributes: Dict[str, Any] + relationships: Dict[str, Any] + links: Dict[str, Any] + + +class IncludedData(TypedDict): + type: str id: int links: Dict[str, Any] + attributes: NotRequired[Dict[str, Any]] + relationships: NotRequired[Dict[str, Any]] class DumpedResult(TypedDict): - data: List[Data] - included: Dict[str, Any] - meta: Optional[Dict[str, Any]] + data: Union[List[Data], Data] + included: NotRequired[List[IncludedData]] + meta: NotRequired[Dict[str, Any]] def add_search_metadata(dumped: DumpedResult, search_value: str): diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py new file mode 100644 index 000000000..452e1e1c4 --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/exceptions.py @@ -0,0 +1,13 @@ +from forestadmin.agent_toolkit.exceptions import AgentToolkitException + + +class JsonApiException(AgentToolkitException): + pass + + +class JsonApiSerializerException(JsonApiException): + pass + + +class JsonApiDeserializerException(JsonApiException): + pass diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api.py deleted file mode 100644 index 63a539d0c..000000000 --- a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api.py +++ /dev/null @@ -1,355 +0,0 @@ -from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union, cast -from uuid import uuid4 - -from forestadmin.agent_toolkit.exceptions import AgentToolkitException -from forestadmin.agent_toolkit.forest_logger import ForestLogger -from forestadmin.agent_toolkit.utils.id import pack_id -from forestadmin.datasource_toolkit.collections import Collection -from forestadmin.datasource_toolkit.datasource_customizer.collection_customizer import CollectionCustomizer -from forestadmin.datasource_toolkit.interfaces.chart import Chart -from forestadmin.datasource_toolkit.interfaces.fields import ( - FieldAlias, - Operator, - PolymorphicManyToOne, - PrimitiveType, - RelationAlias, - is_column, - is_many_to_many, - is_one_to_many, - is_polymorphic_many_to_one, - is_polymorphic_one_to_many, -) -from forestadmin.datasource_toolkit.interfaces.query.projections import Projection -from forestadmin.datasource_toolkit.utils.schema import SchemaUtils -from marshmallow.exceptions import MarshmallowError -from marshmallow.schema import SchemaMeta -from marshmallow_jsonapi import Schema, fields # type: ignore - -CollectionAlias = Union[Collection, CollectionCustomizer] - - -class IntOrFloat(fields.Field): - def _serialize(self, value, attr, obj, **kwargs): # type: ignore - if value is None: - return value - if isinstance(value, int) or isinstance(value, float): - return value - try: - return int(value) - except ValueError: - return float(value) - - def _deserialize(self, value, attr, data, **kwargs): # type: ignore - if value is None: - return value - if isinstance(value, int) or isinstance(value, float): - return value - try: - return int(value) - except ValueError: - return float(value) - - -def schema_name(collection: CollectionAlias): - return f"{collection.name.lower()}_schema" - - -class JsonApiException(AgentToolkitException): - pass - - -def _map_primitive_type(_type: PrimitiveType): - TYPES = { - PrimitiveType.STRING: fields.Str, - PrimitiveType.NUMBER: IntOrFloat, - PrimitiveType.BOOLEAN: fields.Boolean, - PrimitiveType.DATE_ONLY: fields.Date, - PrimitiveType.DATE: fields.DateTime, - PrimitiveType.TIME_ONLY: fields.Time, - PrimitiveType.JSON: fields.Raw, - } - return TYPES.get(_type, fields.Str) - - -def _map_attribute_to_marshmallow(column_alias: FieldAlias): - if isinstance(column_alias["column_type"], PrimitiveType): - type_ = _map_primitive_type(column_alias["column_type"]) - else: - type_ = fields.Raw - - is_nullable = column_alias.get("is_read_only", False) is True or ( - column_alias.get("validations") is not None - and {"operator": Operator.PRESENT} not in column_alias["validations"] - ) - return type_(allow_none=is_nullable) - - -def _create_relationship(collection: CollectionAlias, field_name: str, relation: RelationAlias): - many = is_one_to_many(relation) or is_many_to_many(relation) or is_polymorphic_one_to_many(relation) - kwargs = { - "many": many, - "related_url": f"/forest/{collection.name}/{{{collection.name.lower()}_id}}/relationships/{field_name}", - "related_url_kwargs": {f"{collection.name.lower()}_id": "<__forest_id__>"}, - "collection": collection, - "forest_is_polymorphic": False, - } - if is_many_to_many(relation): - type_ = relation["foreign_collection"] - kwargs["id_field"] = SchemaUtils.get_primary_keys(collection.datasource.get_collection(type_).schema)[0] - elif is_polymorphic_many_to_one(relation): - kwargs["forest_is_polymorphic"] = True - kwargs["forest_relation"] = relation - type_ = relation["foreign_collections"] - else: - type_ = relation["foreign_collection"] - kwargs["id_field"] = SchemaUtils.get_primary_keys(collection.datasource.get_collection(type_).schema)[0] - kwargs.update( - { - "type_": type_, - "schema": f"{type_}_schema", - } - ) - return ForestRelationShip(**kwargs) - - -def _create_schema_attributes(collection: CollectionAlias) -> Dict[str, Any]: - attributes: Dict[str, Any] = {} - pks = SchemaUtils.get_primary_keys(collection.schema) - pk_field = pks[0] - attributes["id_field"] = pk_field - attributes["default_id_field"] = pk_field - - for name, field_schema in collection.schema["fields"].items(): - if is_column(field_schema): - attributes[name] = _map_attribute_to_marshmallow(field_schema) - else: - attributes[name] = _create_relationship(collection, name, cast(RelationAlias, field_schema)) - if "id" not in attributes: - attributes["id"] = _map_primitive_type(collection.get_field(pk_field)["column_type"])() - return attributes - - -class JsonApiSerializer(type): - schema: Dict[str, Type["ForestSchema"]] = {} - attributes: Dict[str, Any] = {} - - def __new__(cls: Any, collection_name: str, bases: Tuple[Any], attrs: Dict[str, Any]): - cls.attributes[collection_name] = attrs.copy() # attrs is removed by the parent init - klass = super(JsonApiSerializer, cls).__new__(cls, collection_name, bases, attrs) - cls.schema[collection_name] = klass - return klass - - @classmethod - def get(cls, collection: CollectionAlias) -> Type["ForestSchema"]: - json_schema_name = schema_name(collection) - try: - json_schema = cls.schema[json_schema_name] - except KeyError: - raise JsonApiException(f"The serializer for the collection {collection.name} is not built") - - current_attr = _create_schema_attributes(collection) - - if cls.attributes[json_schema_name].keys() != current_attr.keys(): - json_schema = refresh_json_api_schema(collection) - return json_schema # type: ignore - - -class JsonApiSchemaType(JsonApiSerializer, SchemaMeta): - pass - - -class ForestRelationShip(fields.Relationship): - def __init__(self, *args, **kwargs): # type: ignore - self.collection: Collection = kwargs.pop("collection") # type: ignore - self.related_collection_name = kwargs["type_"] - self.forest_is_polymorphic = kwargs.pop("forest_is_polymorphic", None) - self.forest_relation = kwargs.pop("forest_relation", None) - self._forest_current_obj = None - super(ForestRelationShip, self).__init__(*args, **kwargs) # type: ignore - - @property - def schema(self) -> "ForestSchema": - if self.forest_is_polymorphic: - target_collection_field = cast(PolymorphicManyToOne, self.forest_relation)["foreign_key_type_field"] - target_collection = self._forest_current_obj[target_collection_field] - related_collection = self.collection.datasource.get_collection(target_collection) - else: - related_collection = self.collection.datasource.get_collection(self.related_collection_name) - - SchemaClass: Type[Schema] = JsonApiSerializer.get(related_collection) - return SchemaClass( - only=getattr(self, "only", None), exclude=getattr(self, "exclude", ()), context=getattr(self, "context", {}) - ) - - def handle_polymorphism(self, attr): - target_collection_field = cast(PolymorphicManyToOne, self.forest_relation)["foreign_key_type_field"] - target_collection = self._forest_current_obj[target_collection_field] - - if target_collection is not None and ( - target_collection not in self.forest_relation["foreign_collections"] - or target_collection not in [c.name for c in self.collection.datasource.collections] - ): - ForestLogger.log( - "warning", - f"Trying to serialize a polymorphic relationship ({self.collection.name}.{attr} for record " - f"{self._forest_current_obj['id']}) of type {target_collection}; but this type is not known by forest." - " Ignoring and setting this relationship to None.", - ) - self._forest_current_obj[attr] = None - - self.type_ = target_collection - if getattr(self, "only", False): - self._old_only = self.only - self.only = None - - def teardown_polymorphism(self): - self.__schema = None # this is a cache variable, so it's preferable to clean it after - if getattr(self, "only", False): - self.only = self._old_only - - def serialize(self, attr, obj, accessor=None): - self._forest_current_obj = obj - if self.forest_is_polymorphic: - self.handle_polymorphism(attr) - ret = super(ForestRelationShip, self).serialize(attr, obj, accessor) - if self.forest_is_polymorphic: - self.teardown_polymorphism() - return ret - - def _get_id(self, value): - if self.forest_is_polymorphic: - type_field = self.forest_relation["foreign_key_type_field"] - type_value = self._forest_current_obj[type_field] - return value.get( - self.forest_relation["foreign_key_targets"][type_value], - value, - ) - else: - return super()._get_id(value) - - def get_related_url(self, obj: Any): - if "id" in obj: - obj["__forest_id__"] = obj["id"] - elif "data" in obj: - obj["__forest_id__"] = obj["data"]["id"] - else: - raise JsonApiException("Cannot find json api 'id' in given obj.") - res: Any = super(ForestRelationShip, self).get_related_url(obj) # type: ignore - del obj["__forest_id__"] - return {"href": res} - - -class ForestSchema(Schema): - def __init__(self, *args, **kwargs): # type: ignore - if "projections" in kwargs: - only, include_data = self._build_only_included_data(kwargs.pop("projections")) # type: ignore - kwargs["only"] = only - if "include_data" not in kwargs: - kwargs["include_data"] = include_data - if kwargs.get("only") is not None and "id" not in kwargs["only"]: - kwargs["only"].add("id") - super(ForestSchema, self).__init__(*args, **kwargs) # type: ignore - - def _build_only_included_data(self, projections: Projection): - only: Set[str] = set() - include_data: Set[str] = set() - for projection in cast(List[str], projections): - if ":" in projection: - only.add(projection.replace(":*", "").replace(":", ".")) - include_data.add(projection.split(":")[0]) - else: - only.add(projection) - return only, include_data - - def get_resource_links(self, item: Any): - item["__forest_id__"] = pack_id(self.Meta.fcollection.schema, item) # type: ignore - res = super(ForestSchema, self).get_resource_links(item) # type: ignore - del item["__forest_id__"] - return res - - def _populate_id(self, obj: Union[Dict[str, Any], List[Dict[str, Any]]]): - if isinstance(obj, list): - for o in obj: - self._populate_id(o) - return obj - if "id" not in obj: - obj["id"] = pack_id(self.Meta.fcollection.schema, obj) # type: ignore - return obj - - def dump(self, obj: Any, *, many: Optional[bool] = None) -> Any: - self._populate_id(obj) - try: - res = super().dump(obj, many=many) # type: ignore - except MarshmallowError as e: - raise JsonApiException(str(e)) - return res # type: ignore - - def load(self, data, *, many=None, partial=None, unknown=None): # type: ignore - try: - return super().load(data, many=many, partial=partial, unknown=unknown) # type: ignore - except MarshmallowError as e: - raise JsonApiException(str(e)) - - def unwrap_item(self, item): # type: ignore - """needed to avoid an issue introduced by the front (type are pluralize for add and update)""" - relationship_to_del = [] - item["type"] = self.opts.type_ # type: ignore - - for name, relationships in item.get("relationships", {}).items(): - relation_field = self.Meta.fcollection.get_field(name) # type: ignore - - if isinstance(relationships["data"], list): - # for many to many and one to many relations - for relationship_data in relationships["data"]: - relationship_data["type"] = relation_field["foreign_collection"] - else: - # for many to one and one to one relations - if relationships is None or relationships.get("data") in [None, {}]: - relationship_to_del.append(name) - continue - - # if polymorphic is sent in relationships, lets put the relation in the attributes - if is_polymorphic_many_to_one(relation_field): - item["attributes"][relation_field["foreign_key"]] = relationships["data"]["id"] - item["attributes"][relation_field["foreign_key_type_field"]] = relationships["data"]["type"] - relationship_to_del.append(name) - else: - relationships["data"]["type"] = relation_field["foreign_collection"] # type: ignore - item["relationships"][name] = relationships - - for name in relationship_to_del: - del item["relationships"][name] - return super(ForestSchema, self).unwrap_item(item) - - -def refresh_json_api_schema(collection: CollectionAlias, ignores: Optional[List[CollectionAlias]] = None): - if ignores is None: - ignores = [] - if collection in ignores: - return - ignores.append(collection) - if schema_name(collection) not in JsonApiSerializer.schema: - raise JsonApiException("The schema doesn't exist") - del JsonApiSerializer.schema[schema_name(collection)] - return create_json_api_schema(collection) - - -def create_json_api_schema(collection: CollectionAlias): - if schema_name(collection) in JsonApiSerializer.schema: - raise JsonApiException("The schema has already been created for this collection") - - attributes: Dict[str, Any] = _create_schema_attributes(collection) - - class JsonApiSchema(ForestSchema): - class Meta: # type: ignore - type_ = collection.name - self_url = f"/forest/{collection.name}/{{{collection.name.lower()}_id}}" - self_url_kwargs = {f"{collection.name.lower()}_id": "<__forest_id__>"} - strict = True - fcollection: CollectionAlias = collection - - return JsonApiSchemaType(schema_name(collection), (JsonApiSchema,), attributes) - - -def render_chart(chart: Chart): - return {"id": str(uuid4()), "type": "stats", "attributes": {"value": chart}} diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py new file mode 100644 index 000000000..9d574feda --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_deserializer.py @@ -0,0 +1,107 @@ +from datetime import date, datetime, time +from typing import Any, Callable, Dict, Union, cast +from uuid import UUID + +from forestadmin.agent_toolkit.forest_logger import ForestLogger +from forestadmin.agent_toolkit.services.serializers import Data, DumpedResult +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiDeserializerException +from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.datasources import Datasource +from forestadmin.datasource_toolkit.interfaces.fields import ( + Column, + PrimitiveType, + is_many_to_many, + is_many_to_one, + is_one_to_many, + is_one_to_one, + is_polymorphic_many_to_one, + is_polymorphic_one_to_many, + is_polymorphic_one_to_one, +) +from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias + + +class JsonApiDeserializer: + def __init__(self, datasource: Datasource) -> None: + self.datasource = datasource + + def deserialize(self, data: DumpedResult, collection: Collection) -> RecordsDataAlias: + ret = {} + data["data"] = cast(Data, data["data"]) + + for key, value in data["data"]["attributes"].items(): + if key not in collection.schema["fields"]: + raise JsonApiDeserializerException(f"Field {key} doesn't exists in collection {collection.name}.") + ret[key] = self._deserialize_value(value, cast(Column, collection.schema["fields"][key])) + + # PK is never sent to deserialize. It's used to identify record. No need to handle it. + # If it's sent to update the PK value, the new value is in 'attributes' + + for key, value in data["data"].get("relationships", {}).items(): + if key not in collection.schema["fields"]: + raise JsonApiDeserializerException(f"Field {key} doesn't exists in collection {collection.name}.") + schema = collection.schema["fields"][key] + + if is_one_to_many(schema) or is_many_to_many(schema) or is_polymorphic_one_to_many(schema): + raise JsonApiDeserializerException("We shouldn't deserialize toMany relations") + + if value.get("data") is None or "id" not in value["data"]: + ret[key] = None + continue + + if is_polymorphic_many_to_one(schema): + ret[schema["foreign_key_type_field"]] = self._deserialize_value( + value["data"]["type"], cast(Column, collection.schema["fields"][schema["foreign_key_type_field"]]) + ) + ret[schema["foreign_key"]] = self._deserialize_value( + value["data"]["id"], cast(Column, collection.schema["fields"][schema["foreign_key"]]) + ) + continue + + elif is_many_to_one(schema): + ret[key] = self._deserialize_value( + value["data"]["id"], cast(Column, collection.schema["fields"][schema["foreign_key"]]) + ) + elif is_one_to_one(schema): + ret[key] = self._deserialize_value( + value["data"]["id"], cast(Column, collection.schema["fields"][schema["origin_key_target"]]) + ) + elif is_polymorphic_one_to_one(schema): + ret[key] = self._deserialize_value( + value["data"]["id"], cast(Column, collection.schema["fields"][schema["origin_key_target"]]) + ) + return ret + + def _deserialize_value(self, value: Union[str, int, float, bool, None], schema: Column) -> Any: + if value is None: + return None + + def number_parser(val): + if isinstance(val, int) or isinstance(val, float): + return val + try: + return int(value) + except ValueError: + return float(value) + + parser_map: Dict[PrimitiveType, Callable] = { + PrimitiveType.STRING: str, + PrimitiveType.ENUM: str, + PrimitiveType.BOOLEAN: bool, + PrimitiveType.NUMBER: number_parser, + PrimitiveType.UUID: lambda v: UUID(v) if isinstance(v, str) else v, + PrimitiveType.DATE_ONLY: lambda v: date.fromisoformat(v) if isinstance(v, str) else v, + PrimitiveType.TIME_ONLY: lambda v: time.fromisoformat(v) if isinstance(v, str) else v, + PrimitiveType.DATE: lambda v: datetime.fromisoformat(v) if isinstance(v, str) else v, + PrimitiveType.POINT: lambda v: [int(v_) for v_ in cast(str, v).split(",")], + PrimitiveType.BINARY: lambda v: v, # should not be called + PrimitiveType.JSON: lambda v: v, + } + + if isinstance(schema["column_type"], PrimitiveType): + return parser_map[cast(PrimitiveType, schema["column_type"])](value) + elif isinstance(schema["column_type"], dict) or isinstance(schema["column_type"], list): + return value + else: + ForestLogger.log("error", f"Unknown column type {schema['column_type']}") + raise JsonApiDeserializerException(f"Unknown column type {schema['column_type']}") diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py new file mode 100644 index 000000000..f015a6ad9 --- /dev/null +++ b/src/agent_toolkit/forestadmin/agent_toolkit/services/serializers/json_api_serializer.py @@ -0,0 +1,268 @@ +from ast import literal_eval +from datetime import date, datetime, time +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from uuid import uuid4 + +from forestadmin.agent_toolkit.forest_logger import ForestLogger +from forestadmin.agent_toolkit.services.serializers import Data, DumpedResult, IncludedData +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiSerializerException +from forestadmin.agent_toolkit.utils.id import pack_id +from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.datasources import Datasource, DatasourceException +from forestadmin.datasource_toolkit.interfaces.chart import Chart +from forestadmin.datasource_toolkit.interfaces.fields import ( + Column, + ManyToOne, + OneToOne, + PolymorphicManyToOne, + PolymorphicOneToOne, + PrimitiveType, + RelationAlias, + is_column, + is_many_to_many, + is_many_to_one, + is_one_to_many, + is_one_to_one, + is_polymorphic_many_to_one, + is_polymorphic_one_to_one, +) +from forestadmin.datasource_toolkit.interfaces.query.projections import Projection +from forestadmin.datasource_toolkit.interfaces.query.projections.factory import ProjectionFactory +from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias + + +def render_chart(chart: Chart): + return {"id": str(uuid4()), "type": "stats", "attributes": {"value": chart}} + + +class JsonApiSerializer: + def __init__(self, datasource: Datasource, projection: Projection) -> None: + self.datasource = datasource + self.projection = projection + + def serialize(self, data, collection: Collection) -> DumpedResult: + if isinstance(data, list): + ret = self._serialize_many(data, collection) + else: + ret = self._serialize_one(data, collection) + + if ret.get("included") == []: + del ret["included"] + return cast(DumpedResult, ret) + + @classmethod + def _get_id(cls, collection: Collection, data: RecordsDataAlias) -> Union[int, str]: + pk = pack_id(collection.schema, data) + try: + pk = int(pk) + except ValueError: + pass + return pk + + @classmethod + def _is_in_included(cls, included: List[Dict[str, Any]], item: IncludedData) -> bool: + if "id" not in item or "type" not in item: + raise JsonApiSerializerException("Included item must have an id and a type") + for included_item in included: + if included_item["id"] == item["id"] and included_item["type"] == item["type"]: + return True + return False + + def _serialize_many(self, data, collection: Collection) -> DumpedResult: + ret = {"data": [], "included": []} + for item in data: + serialized = self._serialize_one(item, collection) + ret["data"].append(serialized["data"]) + for included in serialized.get("included", []): + if not self._is_in_included(ret["included"], included): + ret["included"].append(included) + + return cast(DumpedResult, ret) + + def _serialize_one( + self, data: RecordsDataAlias, collection: Collection, projection: Optional[Projection] = None + ) -> DumpedResult: + projection = projection if projection is not None else self.projection + pk_value = self._get_id(collection, data) + ret = { + "data": { + "id": pk_value, + "attributes": {}, + "links": {"self": f"/forest/{collection.name}/{pk_value}"}, + "relationships": {}, + "type": collection.name, + }, + "included": [], + "links": {"self": f"/forest/{collection.name}/{pk_value}"}, + } + + first_level_projection = [*projection.relations.keys(), *projection.columns] + for key, value in data.items(): + if key not in first_level_projection or key not in collection.schema["fields"]: + continue + if is_column(collection.schema["fields"][key]) and key in first_level_projection: + ret["data"]["attributes"][key] = self._serialize_value( + value, cast(Column, collection.schema["fields"][key]) + ) + elif not is_column(collection.schema["fields"][key]): + relation, included = self._serialize_relation( + key, + data, + cast(RelationAlias, collection.schema["fields"][key]), + f"/forest/{collection.name}/{pk_value}", + ) + ret["data"]["relationships"][key] = relation + if included is not None and not self._is_in_included(ret["included"], included): + ret["included"].append(included) + + if ret["data"].get("attributes") == {}: + del ret["data"]["attributes"] + if ret["data"].get("relationships") == {}: + del ret["data"]["relationships"] + return cast(DumpedResult, ret) + + def _serialize_value(self, value: Any, schema: Column) -> Union[str, int, float, bool, None]: + if value is None: + return None + + def number_dump(val): + if isinstance(val, int) or isinstance(val, float): + return val + elif isinstance(val, str): + return literal_eval(str(value)) + + parser_map: Dict[PrimitiveType, Callable] = { + PrimitiveType.STRING: str, + PrimitiveType.ENUM: str, + PrimitiveType.BOOLEAN: bool, + PrimitiveType.NUMBER: number_dump, + PrimitiveType.UUID: str, + PrimitiveType.DATE_ONLY: lambda v: v if isinstance(v, str) else date.isoformat(v), + PrimitiveType.TIME_ONLY: lambda v: v if isinstance(v, str) else time.isoformat(v), + PrimitiveType.DATE: lambda v: v if isinstance(v, str) else datetime.isoformat(v), + PrimitiveType.POINT: lambda v: v, + PrimitiveType.BINARY: lambda v: v, # should not be called, because of binary decorator this type + # is transformed to string + PrimitiveType.JSON: lambda v: v, + } + + if isinstance(schema["column_type"], PrimitiveType): + return parser_map[cast(PrimitiveType, schema["column_type"])](value) + elif isinstance(schema["column_type"], dict) or isinstance(schema["column_type"], list): + return value + else: + ForestLogger.log("error", f"Unknown column type {schema['column_type']}") + raise JsonApiSerializerException(f"Unknown column type {schema['column_type']}") + + def _serialize_relation( + self, name: str, data: Any, schema: RelationAlias, current_link: str + ) -> Tuple[Dict[str, Any], Optional[IncludedData]]: + relation, included = {}, None + sub_data = data[name] + if sub_data is None: + return { + "data": ( + None + if is_polymorphic_many_to_one(schema) or is_polymorphic_one_to_one(schema) or is_one_to_one(schema) + else [] + ), + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + }, included + + if is_polymorphic_many_to_one(schema): + relation, included = self._serialize_polymorphic_many_to_one_relationship(name, data, schema, current_link) + elif is_many_to_one(schema) or is_one_to_one(schema) or is_polymorphic_one_to_one(schema): + relation, included = self._serialize_to_one_relationships(name, sub_data, schema, current_link) + elif is_many_to_many(schema) or is_one_to_many(schema): + relation = { + "data": [], + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + + return relation, included + + def _serialize_to_one_relationships( + self, + name: str, + data: Any, + schema: Union[PolymorphicOneToOne, OneToOne, ManyToOne], + current_link: str, + ) -> Tuple[Dict[str, Any], IncludedData]: + """return (relationships, included)""" + foreign_collection = self.datasource.get_collection(schema["foreign_collection"]) + + relation = { + "data": { + "id": pack_id(foreign_collection.schema, data), + # "id": self._get_id(foreign_collection, data), + "type": schema["foreign_collection"], + }, + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + + sub_projection = self.projection.relations[name] + included_attributes = {} + for key, value in data.items(): + if key not in sub_projection: + continue + included_attributes[key] = self._serialize_value(value, foreign_collection.schema["fields"][key]) + + included = { + "id": self._get_id(foreign_collection, data), + "links": { + "self": f"/forest/{foreign_collection.name}/{self._get_id(foreign_collection, data)}", + }, + "type": foreign_collection.name, + } + if included_attributes != {}: + included["attributes"] = included_attributes + return relation, cast(IncludedData, included) + + def _serialize_polymorphic_many_to_one_relationship( + self, + name: str, + data: Any, + schema: PolymorphicManyToOne, + current_link: str, + ) -> Tuple[Dict[str, Any], Optional[IncludedData]]: + """return (relationships, included)""" + sub_data = data[name] + try: + foreign_collection = self.datasource.get_collection(data[schema["foreign_key_type_field"]]) + except DatasourceException: + return {"data": None, "links": {"related": {"href": f"{current_link}/relationships/{name}"}}}, None + + relation = { + "data": { + "id": pack_id(foreign_collection.schema, sub_data), # TODO: validate + # "id": self._get_id(foreign_collection, sub_data), + "type": data[schema["foreign_key_type_field"]], + }, + "links": {"related": {"href": f"{current_link}/relationships/{name}"}}, + } + included = self._serialize_one( + sub_data, foreign_collection, ProjectionFactory.all(foreign_collection, allow_nested=True) + ) + included["data"] = cast(Data, included["data"]) + included = { + "type": included["data"]["type"], + "id": included["data"]["id"], + "attributes": included["data"]["attributes"], + # **included.get("data", {}), # type: ignore for serialize_one it's a dict + "links": included.get("links", {}), + "relationships": {}, + } + + # add relationships key in included + for foreign_relation_name, foreign_relation_schema in foreign_collection.schema["fields"].items(): + if not is_column(foreign_relation_schema): + included["relationships"][foreign_relation_name] = { + "links": { + "related": { + "href": f"/forest/{foreign_collection.name}/{self._get_id(foreign_collection, sub_data)}" + f"/relationships/{foreign_relation_name}" + } + } + } + + return relation, cast(IncludedData, included) diff --git a/src/agent_toolkit/forestadmin/agent_toolkit/utils/id.py b/src/agent_toolkit/forestadmin/agent_toolkit/utils/id.py index 56f131b87..b01fb86c0 100644 --- a/src/agent_toolkit/forestadmin/agent_toolkit/utils/id.py +++ b/src/agent_toolkit/forestadmin/agent_toolkit/utils/id.py @@ -2,6 +2,7 @@ from typing import Any, List, cast from uuid import UUID +from forestadmin.agent_toolkit.exceptions import AgentToolkitException from forestadmin.datasource_toolkit.collections import CollectionSchema from forestadmin.datasource_toolkit.interfaces.fields import Column, PrimitiveType from forestadmin.datasource_toolkit.interfaces.records import CompositeIdAlias, RecordsDataAlias @@ -9,7 +10,7 @@ from forestadmin.datasource_toolkit.validations.field import FieldValidator, FieldValidatorException -class IdException(BaseException): +class IdException(AgentToolkitException): pass diff --git a/src/agent_toolkit/pyproject.toml b/src/agent_toolkit/pyproject.toml index 09fb60e71..f517f5c56 100644 --- a/src/agent_toolkit/pyproject.toml +++ b/src/agent_toolkit/pyproject.toml @@ -21,8 +21,6 @@ aiohttp = "~=3.9" oic = "~=1.4" pyjwt = "^2" cachetools = "~=5.2" -marshmallow-jsonapi = ">=0.24.0, <1.0" -marshmallow = "<3.24.0" sseclient-py = "^1.5" forestadmin-datasource-toolkit = "1.22.5" [[tool.poetry.dependencies.pandas]] diff --git a/src/agent_toolkit/tests/resources/collections/test_crud.py b/src/agent_toolkit/tests/resources/collections/test_crud.py index 70a0abe89..063fe348d 100644 --- a/src/agent_toolkit/tests/resources/collections/test_crud.py +++ b/src/agent_toolkit/tests/resources/collections/test_crud.py @@ -17,22 +17,20 @@ from forestadmin.agent_toolkit.resources.collections.requests import RequestCollection, RequestCollectionException from forestadmin.agent_toolkit.services.permissions.ip_whitelist_service import IpWhiteListService from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService -from forestadmin.agent_toolkit.services.serializers.json_api import ( - JsonApiException, - JsonApiSerializer, - create_json_api_schema, -) +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiSerializerException +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import JsonApiSerializer from forestadmin.agent_toolkit.utils.context import Request, RequestMethod, User from forestadmin.agent_toolkit.utils.csv import CsvException -from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.collections import Collection, CollectionException from forestadmin.datasource_toolkit.datasource_customizer.datasource_composite import CompositeDatasource from forestadmin.datasource_toolkit.datasources import Datasource, DatasourceException from forestadmin.datasource_toolkit.exceptions import ForbiddenError, NativeQueryException, ValidationError from forestadmin.datasource_toolkit.interfaces.fields import FieldType, Operator, PrimitiveType from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.branch import ConditionTreeBranch from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf +from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter +from forestadmin.datasource_toolkit.interfaces.query.filter.unpaginated import Filter from forestadmin.datasource_toolkit.interfaces.query.projections import Projection -from forestadmin.datasource_toolkit.validations.records import RecordValidatorException FAKE_USER = User( rendering_id=1, @@ -118,11 +116,17 @@ def _create_collections(cls): "origin_key_target": "id", "type": FieldType.ONE_TO_MANY, }, + "status_id": { + "column_type": PrimitiveType.NUMBER, + "is_primary_key": False, + "type": FieldType.COLUMN, + "filter_operators": set([Operator.IN, Operator.EQUAL]), + }, "status": { "type": FieldType.MANY_TO_ONE, "foreign_collection": "status", "foreign_key_target": "id", - "foreign_key": "status", + "foreign_key": "status_id", }, "cart": { "type": FieldType.ONE_TO_ONE, @@ -213,14 +217,13 @@ def _create_collections(cls): "foreign_collections": ["product", "order"], "foreign_key_target": {"order": "id", "product": "id"}, "foreign_key": "taggable_id", - "foreign_type_field": "taggable_type", + "foreign_key_type_field": "taggable_type", }, }, ) @classmethod def setUpClass(cls) -> None: - JsonApiSerializer.schema = {} cls.loop = asyncio.new_event_loop() cls.options = Options( auth_secret="fake_secret", @@ -242,8 +245,6 @@ def setUpClass(cls) -> None: "tag": cls.collection_tag, } cls.datasource_composite.add_datasource(cls.datasource) - for collection in cls.datasource.collections: - create_json_api_schema(collection) def setUp(self): self.ip_white_list_service = Mock(IpWhiteListService) @@ -254,11 +255,6 @@ def setUp(self): self.permission_service.can = AsyncMock(return_value=None) self.permission_service.can_live_query_segment = AsyncMock(return_value=None) - @classmethod - def tearDownClass(cls) -> None: - JsonApiSerializer.schema = {} - return super().tearDownClass() - # dispatch def test_dispatch(self): request = Request( @@ -347,32 +343,13 @@ def test_dispatch_error(self, mock_request_collection: Mock): assert body["errors"][0] == {"name": "ValidationError", "detail": "test exception", "status": 400, "data": {}} # get - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ProjectionFactory.all", - return_value=Projection("id", "cost"), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_get( - self, - mocked_json_serializer_get: Mock, - mocked_projection_factory_all: Mock, - mocked_match_ids: Mock, - mocked_unpack_id: Mock, - ): - mock_order = {"id": 10, "cost": 200} - self.collection_order.list = AsyncMock(return_value=[mock_order]) + def test_get_should_return_simple_data(self): + mock_order = {"id": 10, "cost": 0, "important": True} + request = RequestCollection( RequestMethod.GET, self.collection_order, - query={"collection_name": "order", "pks": "10"}, + query={"collection_name": "order", "pks": "10", "fields[order]": "id,cost,important"}, headers={}, client_ip="127.0.0.1", ) @@ -383,75 +360,86 @@ def test_get( self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={"data": {"type": "order", "attributes": mock_order}} - ) - - response = self.loop.run_until_complete(crud_resource.get(request)) - + with patch.object( + self.collection_order, "list", new_callable=AsyncMock, return_value=[mock_order] + ) as mock_list: + response = self.loop.run_until_complete(crud_resource.get(request)) + mock_list.assert_any_await( + request.user, + PaginatedFilter( + { + "condition_tree": ConditionTreeBranch( + "and", [ConditionTreeLeaf("id", "equal", 10), ConditionTreeLeaf("id", "greater_than", 0)] + ) + } + ), + ["id", "cost", "important"], + ) self.permission_service.can.assert_any_await(request.user, request.collection, "read") self.permission_service.can.reset_mock() response_content = json.loads(response.body) - assert response.status == 200 - assert isinstance(response_content["data"], dict) - assert response_content["data"]["attributes"]["cost"] == mock_order["cost"] - assert response_content["data"]["attributes"]["id"] == mock_order["id"] - mocked_unpack_id.assert_called_once() - self.collection_order.list.assert_awaited() - + self.assertEqual(response.status, 200) + self.assertTrue(isinstance(response_content["data"], dict)) + self.assertEqual(response_content["data"]["attributes"]["cost"], mock_order["cost"]) + self.assertEqual(response_content["data"]["attributes"]["important"], mock_order["important"]) + self.assertEqual(response_content["data"]["id"], mock_order["id"]) # relations - mocked_unpack_id.reset_mock() - mock_order = {"id": 10, "cost": 0} - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": { - "type": "order", - "attributes": mock_order, - "relationships": { - "products": {"data": [], "links": {"related": "/forest/order/10/relationships/products"}} - }, - } - } + def test_get_should_return_simple_data_with_relation(self): + mock_order = {"id": 10, "cost": 200.3, "cart": {"id": 11}, "important": True} + request = RequestCollection( + RequestMethod.GET, + self.collection_order, + headers={}, + client_ip="127.0.0.1", + query={"collection_name": "order", "pks": "10", "fields[order]": "id,cost,important", "fields[cart]": "id"}, + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, ) - response = self.loop.run_until_complete(crud_resource.get(request)) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer", wraps=JsonApiSerializer + ) as spy_jsonapi: + with patch.object( + self.collection_order, "list", new_callable=AsyncMock, return_value=[mock_order] + ) as mock_list: + response = self.loop.run_until_complete(crud_resource.get(request)) + mock_list.assert_any_await( + request.user, + PaginatedFilter( + { + "condition_tree": ConditionTreeBranch( + "and", + [ConditionTreeLeaf("id", "equal", 10), ConditionTreeLeaf("id", "greater_than", 0)], + ) + } + ), + ["id", "cost", "important"], + ) + spy_jsonapi.assert_called_once_with(ANY, ["id", "cost", "important", "products:id"]) self.permission_service.can.assert_any_await(request.user, request.collection, "read") self.permission_service.can.reset_mock() response_content = json.loads(response.body) - assert response.status == 200 - assert isinstance(response_content["data"], dict) - assert response_content["data"]["attributes"]["cost"] == mock_order["cost"] - assert response_content["data"]["attributes"]["id"] == mock_order["id"] - mocked_unpack_id.assert_called_once() - self.collection_order.list.assert_awaited() + self.assertEqual(response.status, 200) + self.assertTrue(isinstance(response_content["data"], dict)) + self.assertEqual(response_content["data"]["attributes"]["cost"], mock_order["cost"]) + self.assertEqual( + response_content["data"]["relationships"]["products"], + {"data": [], "links": {"related": {"href": "/forest/order/10/relationships/products"}}}, + ) + self.assertEqual(response_content["data"]["id"], mock_order["id"]) - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ProjectionFactory.all", - return_value=Projection("id", "cost"), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_get_no_data( - self, - mocked_json_serializer_get: Mock, - mocked_projection_factory_all: Mock, - mocked_match_ids: Mock, - mocked_unpack_id: Mock, - ): - self.collection_order.list = AsyncMock(return_value=[]) + def test_get_no_data(self): request = RequestCollection( RequestMethod.GET, self.collection_order, - query={"collection_name": "order", "pks": "10"}, + query={"collection_name": "order", "pks": "10", "fields[order]": "id,cost"}, headers={}, client_ip="127.0.0.1", ) @@ -463,41 +451,22 @@ def test_get_no_data( self.options, ) - response = self.loop.run_until_complete(crud_resource.get(request)) + with patch.object(self.collection_order, "list", new_callable=AsyncMock, return_value=[]) as mock_list: + response = self.loop.run_until_complete(crud_resource.get(request)) + mock_list.assert_any_await(request.user, ANY, Projection("id", "cost")) self.permission_service.can.assert_any_await(request.user, request.collection, "read") self.permission_service.can.reset_mock() - assert response.status == 404 - assert response.body is None - mocked_unpack_id.assert_called_once() - self.collection_order.list.assert_awaited() + self.assertEqual(response.status, 404) + self.assertIsNone(response.body) - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ProjectionFactory.all", - return_value=Projection("id", "cost"), - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_get_errors( - self, - mocked_json_serializer_get: Mock, - mocked_projection_factory_all: Mock, - mocked_match_ids: Mock, - mocked_unpack_id: Mock, - ): + def test_get_projection_error(self): mock_order = {"id": 10, "costt": 200} self.collection_order.list = AsyncMock(return_value=[mock_order]) request = RequestCollection( RequestMethod.GET, self.collection_order, - query={"collection_name": "order", "pks": "10"}, + query={"collection_name": "order", "pks": "10", "fields[order]": "id,costt"}, headers={}, client_ip="127.0.0.1", ) @@ -508,35 +477,50 @@ def test_get_errors( self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock(side_effect=JsonApiException) - - response = self.loop.run_until_complete(crud_resource.get(request)) + with patch.object(self.collection_order, "list", new_callable=AsyncMock, return_value=[]) as mock_list: + self.assertRaisesRegex( + CollectionException, + r"🌳🌳🌳Field not found 'order\.costt'", + self.loop.run_until_complete, + crud_resource.get(request), + ) + mock_list.assert_not_awaited() self.permission_service.can.assert_any_await(request.user, request.collection, "read") self.permission_service.can.reset_mock() - assert response.status == 500 - response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "JsonApiException", - "detail": "🌳🌳🌳", - "status": 500, - } - mocked_unpack_id.assert_called_once() - self.collection_order.list.assert_awaited() - mocked_unpack_id.reset_mock() + def test_get_error_on_unpacking_id(self): + request = RequestCollection( + RequestMethod.GET, + self.collection_order, + query={"collection_name": "order", "pks": "10", "fields[order]": "id,costt"}, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) - mocked_unpack_id.side_effect = CollectionResourceException - response = self.loop.run_until_complete(crud_resource.get(request)) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud.unpack_id", side_effect=CollectionResourceException + ) as mocked_unpack_id: + response = self.loop.run_until_complete(crud_resource.get(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "read") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "CollectionResourceException", - "detail": "🌳🌳🌳", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "CollectionResourceException", + "detail": "🌳🌳🌳", + "status": 500, + }, + ) mocked_unpack_id.assert_called_once() def test_get_should_return_to_many_relations_as_link(self): @@ -576,11 +560,7 @@ def test_get_should_return_to_many_relations_as_link(self): }, ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_get_with_polymorphic_relation_should_add_projection_star(self, mocked_json_serializer_get: Mock): + def test_get_with_polymorphic_relation_should_add_projection_star(self): request = RequestCollection( RequestMethod.GET, self.collection_tag, @@ -595,12 +575,6 @@ def test_get_with_polymorphic_relation_should_add_projection_star(self, mocked_j self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": {"type": "tag", "attributes": {"id": 10, "taggable_id": 10, "taggable_type": "product"}}, - "included": [{"id": 10, "attributes": {"name": "my product"}}], - } - ) with patch.object( self.collection_tag, "list", @@ -609,27 +583,50 @@ def test_get_with_polymorphic_relation_should_add_projection_star(self, mocked_j {"id": 10, "taggable_id": 10, "taggable_type": "product", "taggable": {"id": 10, "name": "my product"}} ], ) as mock_list: - self.loop.run_until_complete(crud_resource.get(request)) + response = self.loop.run_until_complete(crud_resource.get(request)) + mock_list.assert_awaited_with( + request.user, ANY, ["id", "tag", "taggable_id", "taggable_type", "taggable:*"] + ) - mock_list.assert_awaited_with(ANY, ANY, ["id", "tag", "taggable_id", "taggable_type", "taggable:*"]) + response_content = json.loads(response.body) + self.assertEqual( + response_content, + { + "data": { + "id": 10, + "attributes": {"id": 10, "taggable_id": 10, "taggable_type": "product"}, + "links": {"self": "/forest/tag/10"}, + "relationships": { + "taggable": { + "data": {"id": "10", "type": "product"}, + "links": {"related": {"href": "/forest/tag/10/relationships/taggable"}}, + } + }, + "type": "tag", + }, + "links": {"self": "/forest/tag/10"}, + "included": [ + { + "type": "product", + "id": 10, + "attributes": {"id": 10, "name": "my product"}, + "links": {"self": "/forest/product/10"}, + "relationships": { + "tags": {"links": {"related": {"href": "/forest/product/10/relationships/tags"}}} + }, + } + ], + }, + ) # add - @patch("forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate") - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add( - self, - mocked_json_serializer_get: Mock, - mocked_record_validator_validate: Mock, - ): - mock_order = {"id": 10, "cost": 200} + def test_simple_add(self): + mock_order = {"cost": 200} request = RequestCollection( RequestMethod.POST, self.collection_order, - body=mock_order, + body={"data": {"attributes": mock_order}, "type": "order"}, query={"collection_name": "order"}, headers={}, client_ip="127.0.0.1", @@ -641,45 +638,28 @@ def test_add( self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.load = Mock(return_value=mock_order) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={"data": {"type": "order", "attributes": mock_order}} - ) - - crud_resource.extract_data = AsyncMock(return_value=(mock_order, [])) - # with patch.object(self.collection_order, "get_field", new_callable=AsyncMock, return_value=(mock_order, [])): with patch.object( - self.collection_order, "create", new_callable=AsyncMock, return_value=[mock_order] + self.collection_order, "create", new_callable=AsyncMock, return_value=[{**mock_order, "id": 10}] ) as mock_collection_create: response = self.loop.run_until_complete(crud_resource.add(request)) - mock_collection_create.assert_awaited() + mock_collection_create.assert_any_await(request.user, [mock_order]) self.permission_service.can.assert_any_await(request.user, request.collection, "add") self.permission_service.can.reset_mock() - mocked_record_validator_validate.assert_called() response_content = json.loads(response.body) - assert response.status == 200 - assert isinstance(response_content["data"], dict) - assert response_content["data"]["attributes"]["cost"] == mock_order["cost"] - assert response_content["data"]["attributes"]["id"] == mock_order["id"] - assert response_content["data"]["type"] == "order" - - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add_errors( - self, - mocked_json_serializer_get: Mock, - ): - mock_order = {"id": 10, "cost": 200} + self.assertEqual(response.status, 200) + self.assertTrue(isinstance(response_content["data"], dict)) + self.assertEqual(response_content["data"]["attributes"]["cost"], mock_order["cost"]) + self.assertEqual(response_content["data"]["id"], 10) + self.assertEqual(response_content["data"]["type"], "order") + def test_add_error_on_json_api(self): request = RequestCollection( RequestMethod.POST, self.collection_order, - body=mock_order, + body={"data": {"attributes": {"costtt": 399}}, "type": "order"}, query={"collection_name": "order"}, headers={}, client_ip="127.0.0.1", @@ -691,97 +671,59 @@ def test_add_errors( self.ip_white_list_service, self.options, ) - crud_resource.extract_data = AsyncMock(return_value=(mock_order, [])) # JsonApiException - mocked_json_serializer_get.return_value.load = Mock(side_effect=JsonApiException) - response = self.loop.run_until_complete(crud_resource.add(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "add") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "JsonApiException", - "detail": "🌳🌳🌳", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "JsonApiDeserializerException", + "detail": "🌳🌳🌳Field costtt doesn't exists in collection order.", + "status": 500, + }, + ) - mocked_json_serializer_get.return_value.load = Mock(return_value=mock_order) + def test_add_error_record_validation(self): + request = RequestCollection( + RequestMethod.POST, + self.collection_order, + body={"data": {"attributes": {}}, "type": "order"}, + query={"collection_name": "order"}, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) # RecordValidatorException - with patch( - "forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate", - side_effect=RecordValidatorException, - ): - response = self.loop.run_until_complete(crud_resource.add(request)) - assert response.status == 500 - response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "RecordValidatorException", - "detail": "🌳🌳🌳", - "status": 500, - } - - # DatasourceException - with patch( - "forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate", - ): - with patch.object(self.collection_order, "create", new_callable=AsyncMock, side_effect=DatasourceException): - response = self.loop.run_until_complete(crud_resource.add(request)) - self.permission_service.can.assert_any_await(request.user, request.collection, "add") - self.permission_service.can.reset_mock() - assert response.status == 500 - response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "DatasourceException", - "detail": "🌳🌳🌳", - "status": 500, - } - - # CollectionResourceException - with patch( - "forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate", - ): - with patch.object(self.collection_order, "create", new_callable=AsyncMock, return_value=[mock_order]): - with patch.object(crud_resource, "_link_one_to_one_relations", side_effect=CollectionResourceException): - response = self.loop.run_until_complete(crud_resource.add(request)) - self.permission_service.can.assert_any_await(request.user, request.collection, "add") - self.permission_service.can.reset_mock() - assert response.status == 500 + response = self.loop.run_until_complete(crud_resource.add(request)) + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "CollectionResourceException", - "detail": "🌳🌳🌳", - "status": 500, - } - - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 1), - ) - @patch("forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate") - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add_with_relation( - self, - mocked_json_serializer_get: Mock, - mocked_record_validator_validate: Mock, - mock_match_ids: Mock, - ): - mock_order = {"id": 10, "cost": 200, "status": 1, "cart": 1} + self.assertEqual( + response_content["errors"][0], + { + "name": "RecordValidatorException", + "detail": "🌳🌳🌳The record data is empty", + "status": 500, + }, + ) + def test_add_error_on_datasource_create(self): request = RequestCollection( RequestMethod.POST, self.collection_order, - body=mock_order, - query={ - "collection_name": "order", - "timezone": "Europe/Paris", - }, + body={"data": {"attributes": {"cost": 399}}, "type": "order"}, + query={"collection_name": "order"}, headers={}, client_ip="127.0.0.1", ) @@ -792,83 +734,91 @@ def test_add_with_relation( self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.load = Mock(return_value=mock_order) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={"data": {"type": "order", "attributes": mock_order}} - ) - with patch.object( - self.collection_order, "create", new_callable=AsyncMock, return_value=[mock_order] - ) as mock_collection_create: + # DatasourceException + with patch.object(self.collection_order, "create", new_callable=AsyncMock, side_effect=DatasourceException): response = self.loop.run_until_complete(crud_resource.add(request)) - - mock_collection_create.assert_awaited() self.permission_service.can.assert_any_await(request.user, request.collection, "add") self.permission_service.can.reset_mock() - - mocked_record_validator_validate.assert_called() + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response.status == 200 - assert isinstance(response_content["data"], dict) - assert response_content["data"]["attributes"]["cost"] == mock_order["cost"] - assert response_content["data"]["attributes"]["id"] == mock_order["id"] - assert response_content["data"]["attributes"]["status"] == mock_order["status"] - assert response_content["data"]["attributes"]["cart"] == mock_order["cart"] - assert response_content["data"]["type"] == "order" - self.collection_cart.update.assert_awaited() + self.assertEqual( + response_content["errors"][0], + { + "name": "DatasourceException", + "detail": "🌳🌳🌳", + "status": 500, + }, + ) + def test_add_error_on_link_one_to_one_relations(self): request = RequestCollection( RequestMethod.POST, self.collection_order, - body=mock_order, - query={ - "collection_name": "order", + body={ + "data": { + "attributes": {"cost": 399}, + "relationships": { + "cart": {"data": {"type": "cart", "id": "11"}}, + }, + }, + "type": "order", }, + query={"collection_name": "order"}, headers={}, client_ip="127.0.0.1", ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) + # CollectionResourceException with patch.object( - self.collection_order, "create", new_callable=AsyncMock, return_value=[mock_order] - ) as mock_collection_create: - response = self.loop.run_until_complete(crud_resource.add(request)) - mock_collection_create.assert_awaited() + self.collection_order, "create", new_callable=AsyncMock, return_value=[{"cost": 399, "id": 1}] + ): + with patch.object( + self.collection_cart, + "update", + new_callable=AsyncMock, + ): + response = self.loop.run_until_complete(crud_resource.add(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "add") self.permission_service.can.reset_mock() - - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "CollectionResourceException", - "detail": "🌳🌳🌳Missing timezone", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "CollectionResourceException", + "detail": "🌳🌳🌳Missing timezone", + "status": 500, + }, + ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add_should_create_and_associate_polymorphic_many_to_one(self, mocked_json_serializer_get: Mock): + def test_add_with_relation(self): request = RequestCollection( RequestMethod.POST, - self.collection_tag, + self.collection_order, body={ "data": { - "attributes": {"tag": "Good"}, - "relationships": {"taggable": {"data": [{"type": "order", "id": "14"}]}}, + "attributes": {"cost": 200}, + "relationships": { + "cart": {"data": {"type": "cart", "id": "11"}}, + "status": {"data": {"type": "status", "id": "11"}}, + }, }, - }, # body + "type": "order", + }, query={ "collection_name": "order", - "relation_name": "tags", "timezone": "Europe/Paris", - }, # query + }, headers={}, client_ip="127.0.0.1", ) - mocked_json_serializer_get.return_value.load = Mock( - return_value={"taggable_id": 14, "taggable_type": "order", "tag": "aaaaa"} - ) - mocked_json_serializer_get.return_value.dump = Mock(return_value={}) crud_resource = CrudResource( self.datasource_composite, self.datasource, @@ -877,40 +827,78 @@ def test_add_should_create_and_associate_polymorphic_many_to_one(self, mocked_js self.options, ) + mock_order = {"id": 10, "cost": 200, "status_id": 11} with patch.object( - self.collection_tag, "create", new_callable=AsyncMock, return_value=[{}] - ) as mock_collection_create: - self.loop.run_until_complete(crud_resource.add(request)) - mock_collection_create.assert_awaited_once_with( - ANY, [{"taggable_id": 14, "taggable_type": "order", "tag": "aaaaa"}] - ) + self.collection_order, + "create", + new_callable=AsyncMock, + return_value=[mock_order], + ) as mock_collection_order_create: + with patch.object( + self.collection_cart, + "update", + new_callable=AsyncMock, + ) as mock_collection_cart_update: + response = self.loop.run_until_complete(crud_resource.add(request)) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add_should_create_and_associate_polymorphic_one_to_one(self, mocked_json_serializer_get: Mock): + mock_collection_order_create.assert_any_await(request.user, [{"cost": 200, "status_id": 11}]) + mock_collection_cart_update.assert_any_await( + request.user, + Filter( + { + "condition_tree": ConditionTreeBranch( + "and", + [ + ConditionTreeLeaf("order_id", "equal", 10), + ConditionTreeLeaf("id", "greater_than", 0), + ], + ), + "timezone": zoneinfo.ZoneInfo(key="Europe/Paris"), + } + ), + {"order_id": None}, + ) + mock_collection_cart_update.assert_any_await( + request.user, + Filter( + { + "condition_tree": ConditionTreeBranch( + "and", + [ConditionTreeLeaf("id", "equal", 11), ConditionTreeLeaf("id", "greater_than", 0)], + ), + "timezone": zoneinfo.ZoneInfo(key="Europe/Paris"), + } + ), + {"order_id": 10}, + ) + self.permission_service.can.assert_any_await(request.user, request.collection, "add") + self.permission_service.can.reset_mock() + + response_content = json.loads(response.body) + self.assertEqual(response.status, 200) + self.assertTrue(isinstance(response_content["data"], dict)) + self.assertEqual(response_content["data"]["attributes"]["cost"], mock_order["cost"]) + self.assertEqual(response_content["data"]["id"], mock_order["id"]) + self.assertEqual(response_content["data"]["type"], "order") + self.assertEqual(response_content["data"]["attributes"]["status_id"], mock_order["status_id"]) + + def test_add_should_create_and_associate_polymorphic_many_to_one(self): request = RequestCollection( RequestMethod.POST, - self.collection_order, + self.collection_tag, body={ "data": { - "attributes": {"cost": 12.3, "important": True}, - "relationships": {"tags": {"data": {"type": "tag", "id": "22"}}}, + "attributes": {"tag": "Good"}, + "relationships": {"taggable": {"data": {"type": "order", "id": "14"}}}, }, }, # body query={ - "collection_name": "order", - "relation_name": "tags", + "collection_name": "tag", "timezone": "Europe/Paris", }, # query headers={}, client_ip="127.0.0.1", ) - - mocked_json_serializer_get.return_value.load = Mock(return_value={"cost": 12.3, "important": True, "tags": 22}) - mocked_json_serializer_get.return_value.dump = Mock(return_value={}) - crud_resource = CrudResource( self.datasource_composite, self.datasource, @@ -918,36 +906,17 @@ def test_add_should_create_and_associate_polymorphic_one_to_one(self, mocked_jso self.ip_white_list_service, self.options, ) + with patch.object( - self.collection_order, + self.collection_tag, "create", new_callable=AsyncMock, - return_value=[{"cost": 12.3, "important": True, "id": 12}], - ) as mock_collection_order_create: - with patch.object(self.collection_tag, "update", new_callable=AsyncMock) as mock_collection_tag_update: - self.loop.run_until_complete(crud_resource.add(request)) - mock_collection_order_create.assert_awaited_once_with(ANY, [{"cost": 12.3, "important": True}]) - - # first update to break potential old link to current record (should do nothing) - first_call_update_args = mock_collection_tag_update.await_args_list[0].args - self.assertIn( - ConditionTreeLeaf("taggable_id", "equal", 12), - first_call_update_args[1].condition_tree.conditions, - ) - self.assertIn( - ConditionTreeLeaf("taggable_type", "equal", "order"), - first_call_update_args[1].condition_tree.conditions, - ) - self.assertEqual(first_call_update_args[2], {"taggable_id": None, "taggable_type": None}) - - # second update to link the 1 to 1 - second_call_update_args = mock_collection_tag_update.await_args_list[1].args - - self.assertIn( - ConditionTreeLeaf("id", "equal", 22), - second_call_update_args[1].condition_tree.conditions, - ) - self.assertEqual(second_call_update_args[2], {"taggable_id": 12, "taggable_type": "order"}) + return_value=[{"taggable_id": 14, "taggable_type": "order", "tag": "Good", "id": 1}], + ) as mock_collection_create: + self.loop.run_until_complete(crud_resource.add(request)) + mock_collection_create.assert_awaited_once_with( + ANY, [{"taggable_id": 14, "taggable_type": "order", "tag": "Good"}] + ) def test_add_should_return_to_many_relations_as_link(self): mock_orders = [{"cost": 12.3, "important": True, "id": 10}] @@ -961,7 +930,6 @@ def test_add_should_return_to_many_relations_as_link(self): }, # body query={ "collection_name": "order", - "relation_name": "tags", "timezone": "Europe/Paris", }, headers={}, @@ -990,11 +958,7 @@ def test_add_should_return_to_many_relations_as_link(self): ) # list - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list(self, mocked_json_serializer_get: Mock): + def test_list(self): mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] request = RequestCollection( @@ -1016,33 +980,26 @@ def test_list(self, mocked_json_serializer_get: Mock): self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": [ - {"type": "order", "attributes": mock_order, "id": mock_order["id"]} for mock_order in mock_orders - ] - } - ) self.collection_order.list = AsyncMock(return_value=mock_orders) response = self.loop.run_until_complete(crud_resource.list(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "browse") self.permission_service.can.reset_mock() - assert response.status == 200 + self.assertEqual(response.status, 200) response_content = json.loads(response.body) - assert isinstance(response_content["data"], list) - assert len(response_content["data"]) == 2 - assert response_content["data"][0]["type"] == "order" - assert response_content["data"][0]["attributes"]["cost"] == mock_orders[0]["cost"] - assert response_content["data"][0]["attributes"]["id"] == mock_orders[0]["id"] - assert response_content["data"][1]["type"] == "order" - assert response_content["data"][1]["attributes"]["cost"] == mock_orders[1]["cost"] - assert response_content["data"][1]["attributes"]["id"] == mock_orders[1]["id"] + self.assertTrue(isinstance(response_content["data"], list)) + self.assertEqual(len(response_content["data"]), 2) + self.assertEqual(response_content["data"][0]["type"], "order") + self.assertEqual(response_content["data"][0]["attributes"]["cost"], mock_orders[0]["cost"]) + self.assertEqual(response_content["data"][0]["id"], mock_orders[0]["id"]) + self.assertEqual(response_content["data"][1]["type"], "order") + self.assertEqual(response_content["data"][1]["attributes"]["cost"], mock_orders[1]["cost"]) + self.assertEqual(response_content["data"][1]["id"], mock_orders[1]["id"]) self.collection_order.list.assert_awaited() - assert response_content["meta"]["decorators"]["0"] == {"id": 10, "search": ["cost"]} - assert response_content["meta"]["decorators"]["1"] == {"id": 11, "search": ["cost"]} + self.assertEqual(response_content["meta"]["decorators"]["0"], {"id": 10, "search": ["cost"]}) + self.assertEqual(response_content["meta"]["decorators"]["1"], {"id": 11, "search": ["cost"]}) def test_list_should_return_to_many_relations_as_link(self): mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] @@ -1080,13 +1037,7 @@ def test_list_should_return_to_many_relations_as_link(self): }, ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list_with_polymorphic_many_to_one_should_query_all_relation_record_columns( - self, mocked_json_serializer_get: Mock - ): + def test_list_with_polymorphic_many_to_one_should_query_all_relation_record_columns(self): crud_resource = CrudResource( self.datasource_composite, self.datasource, @@ -1094,48 +1045,6 @@ def test_list_with_polymorphic_many_to_one_should_query_all_relation_record_colu self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": [ - { - "type": "tag", - "attributes": { - "taggable_id": 11, - "taggable_type": "order", - }, - "id": 1, - "relationships": { - "taggable": { - "data": { - "id": 10, - "type": "order", - }, - "links": {"related": {"href": "/forest/tag/1/relationships/taggable"}}, - } - }, - }, - ], - "included": [ - { - "type": "order", - "id": 11, - "attributes": { - "id": 11, - "cost": 201, - "important": True, - }, - "relationships": [ - { - "tags": {"link": {"related": {"href": "/forest/order/11/relationships/tags"}}}, - "cart": {"link": {"related": {"href": "/forest/order/11/relationships/cart"}}}, - "status": {"link": {"related": {"href": "/forest/order/11/relationships/status"}}}, - "products": {"link": {"related": {"href": "/forest/order/11/relationships/products"}}}, - } - ], - } - ], - } - ) request = RequestCollection( RequestMethod.GET, self.collection_tag, @@ -1153,7 +1062,7 @@ def test_list_with_polymorphic_many_to_one_should_query_all_relation_record_colu "list", new_callable=AsyncMock, return_value=[ - {"id": 1, "taggable_id": 11, "taggable_type": "order", "taggable": {}}, + {"id": 1, "taggable_id": 11, "taggable_type": "order", "taggable": {"id": 12, "cost": 12.3}}, ], ) as mock_list: self.loop.run_until_complete(crud_resource.list(request)) @@ -1161,11 +1070,7 @@ def test_list_with_polymorphic_many_to_one_should_query_all_relation_record_colu list_args = mock_list.await_args_list[0].args self.assertIn("taggable:*", list_args[2]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list_should_parse_multi_field_sorting(self, mocked_json_serializer_get: Mock): + def test_list_should_parse_multi_field_sorting(self): mock_orders = [ {"id": 10, "cost": 200, "important": True}, {"id": 11, "cost": 201, "important": True}, @@ -1191,13 +1096,6 @@ def test_list_should_parse_multi_field_sorting(self, mocked_json_serializer_get: self.ip_white_list_service, self.options, ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": [ - {"type": "order", "attributes": mock_order, "id": mock_order["id"]} for mock_order in mock_orders - ] - } - ) self.collection_order.list = AsyncMock(return_value=mock_orders) self.loop.run_until_complete(crud_resource.list(request)) @@ -1241,12 +1139,7 @@ def test_list_should_handle_live_query_segment(self): self.loop.run_until_complete(crud_resource.list(request)) mock_handle_live_queries.assert_awaited_once_with(request, ConditionTreeLeaf("id", "greater_than", 0)) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list_errors(self, mocked_json_serializer_get: Mock): - mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] + def test_list_errors_filter_error(self): request = RequestCollection( RequestMethod.GET, self.collection_order, @@ -1278,6 +1171,24 @@ def test_list_errors(self, mocked_json_serializer_get: Mock): "status": 500, } + def test_list_errors_datasource_error(self): + request = RequestCollection( + RequestMethod.GET, + self.collection_order, + query={ + "collection_name": "order", + "fields[order]": "id,cost", + }, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) # DatasourceException request = RequestCollection( RequestMethod.GET, @@ -1304,18 +1215,42 @@ def test_list_errors(self, mocked_json_serializer_get: Mock): "status": 500, } + def test_list_errors_jsonapi_error(self): + mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] + request = RequestCollection( + RequestMethod.GET, + self.collection_order, + query={ + "collection_name": "order", + "fields[order]": "id,cost", + "timezone": "Europe/Paris", + }, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) # JsonApiException - self.collection_order.list = AsyncMock(return_value=mock_orders) - mocked_json_serializer_get.return_value.dump = Mock(side_effect=JsonApiException) - response = self.loop.run_until_complete(crud_resource.list(request)) + self.collection_order.list = AsyncMock(return_value=mock_orders) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.serialize", + side_effect=JsonApiSerializerException, + ) as mock_serialize: + response = self.loop.run_until_complete(crud_resource.list(request)) + mock_serialize.assert_called() self.permission_service.can.assert_any_await(request.user, request.collection, "browse") self.permission_service.can.reset_mock() assert response.status == 500 response_content = json.loads(response.body) assert response_content["errors"][0] == { - "name": "JsonApiException", + "name": "JsonApiSerializerException", "detail": "🌳🌳🌳", "status": 500, } @@ -1416,23 +1351,7 @@ def test_deactivate_count(self): assert response_content["meta"]["count"] == "deactivated" # edit - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch("forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate") - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_edit( - self, - mocked_json_serializer_get: Mock, - mocked_unpack_id: Mock, - mocked_record_validator_validate: Mock, - mocked_match_ids: Mock, - ): + def test_edit(self): mock_order = {"id": 10, "cost": 201} request = RequestCollection( RequestMethod.PUT, @@ -1456,39 +1375,18 @@ def test_edit( ) self.collection_order.list = AsyncMock(return_value=[mock_order]) self.collection_order.update = AsyncMock() - mocked_json_serializer_get.return_value.load = Mock(return_value=mock_order) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={"data": {"type": "order", "attributes": mock_order}} - ) - response = self.loop.run_until_complete(crud_resource.update(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 200 + self.assertEqual(response.status, 200) response_content = json.loads(response.body) - assert isinstance(response_content["data"], dict) - assert response_content["data"]["attributes"]["id"] == 10 - assert response_content["data"]["attributes"]["cost"] == 201 + self.assertTrue(isinstance(response_content["data"], dict)) + self.assertEqual(response_content["data"]["id"], 10) + self.assertEqual(response_content["data"]["attributes"]["cost"], 201) self.collection_order.update.assert_awaited() - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch("forestadmin.agent_toolkit.resources.collections.crud.RecordValidator.validate") - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - def test_edit_errors( - self, - mocked_json_serializer_get: Mock, - mocked_unpack_id: Mock, - mocked_record_validator_validate: Mock, - mocked_match_ids: Mock, - ): + def test_edit_error_no_pk(self): mock_order = {"id": 10, "cost": 201} self.collection_order.update = AsyncMock() self.collection_order.list = AsyncMock(return_value=[mock_order]) @@ -1499,7 +1397,7 @@ def test_edit_errors( query={ "collection_name": "order", "timezone": "Europe/Paris", - "pks": "10", + "pks": "10|é", "fields[order]": "id,cost", }, headers={}, @@ -1513,38 +1411,103 @@ def test_edit_errors( self.options, ) - # CollectionResourceException - with patch( - "forestadmin.agent_toolkit.resources.collections.crud.unpack_id", side_effect=CollectionResourceException - ): - response = self.loop.run_until_complete(crud_resource.update(request)) + # IdException + response = self.loop.run_until_complete(crud_resource.update(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "detail": "🌳🌳🌳", - "name": "CollectionResourceException", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "detail": "🌳🌳🌳Unable to unpack the id", + "name": "IdException", + "status": 500, + }, + ) + + def test_edit_error_jsonapi_deserialization(self): + self.collection_order.update = AsyncMock() + request = RequestCollection( + RequestMethod.PUT, + self.collection_order, + body={"data": {"attributes": {"cost": 201}, "relationships": {}}}, + query={ + "collection_name": "order", + "timezone": "Europe/Paris", + "pks": "10", + "fields[order]": "id,cost", + }, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) # JsonApiException - mocked_json_serializer_get.return_value.load = Mock(side_effect=JsonApiException) - response = self.loop.run_until_complete(crud_resource.update(request)) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud.JsonApiDeserializer.deserialize", + side_effect=JsonApiSerializerException, + ) as mock_deserialize: + response = self.loop.run_until_complete(crud_resource.update(request)) + mock_deserialize.assert_called() + self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == {"detail": "🌳🌳🌳", "name": "JsonApiException", "status": 500} + self.assertEqual( + response_content["errors"][0], + { + "detail": "🌳🌳🌳", + "name": "JsonApiSerializerException", + "status": 500, + }, + ) + + def test_edit_error_jsonapi_serialization(self): + mock_order = {"id": 10, "cost": 201} + self.collection_order.list = AsyncMock(return_value=[mock_order]) + request = RequestCollection( + RequestMethod.PUT, + self.collection_order, + body={"data": {"attributes": {"cost": 201}, "relationships": {}}}, + query={ + "collection_name": "order", + "timezone": "Europe/Paris", + "pks": "10", + "fields[order]": "id,cost", + }, + headers={}, + client_ip="127.0.0.1", + ) + crud_resource = CrudResource( + self.datasource_composite, + self.datasource, + self.permission_service, + self.ip_white_list_service, + self.options, + ) # JsonApiException - mocked_json_serializer_get.return_value.dump = Mock(side_effect=JsonApiException) - response = self.loop.run_until_complete(crud_resource.update(request)) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.serialize", + side_effect=JsonApiSerializerException, + ) as mock_serialize: + response = self.loop.run_until_complete(crud_resource.update(request)) + mock_serialize.assert_called() self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == {"detail": "🌳🌳🌳", "name": "JsonApiException", "status": 500} + self.assertEqual( + response_content["errors"][0], {"detail": "🌳🌳🌳", "name": "JsonApiSerializerException", "status": 500} + ) def test_edit_should_not_throw_and_do_nothing_on_empty_record(self): request = RequestCollection( @@ -1678,21 +1641,7 @@ def test_update_should_return_to_many_relations_as_link(self): ) # delete - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.JsonApiSerializer.get", - return_value=Mock, - ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.EQUAL, 10), - ) - @patch("forestadmin.agent_toolkit.resources.collections.crud.unpack_id", return_value=[10]) - def test_delete( - self, - mocked_json_serializer_get: Mock, - mocked_unpack_id: Mock, - mocked_match_ids: Mock, - ): + def test_delete(self): request = RequestCollection( RequestMethod.DELETE, self.collection_order, @@ -1747,14 +1696,7 @@ def test_delete_error(self): "status": 500, } - @patch( - "forestadmin.agent_toolkit.resources.collections.crud.ConditionTreeFactory.match_ids", - return_value=ConditionTreeLeaf("id", Operator.NOT_EQUAL, 10), - ) - def test_delete_list( - self, - mocked_match_ids: Mock, - ): + def test_delete_list(self): request = RequestCollection( RequestMethod.DELETE, self.collection_order, @@ -2142,7 +2084,6 @@ def test_handle_native_query_should_raise_error_if_live_query_params_are_incorre ) def test_handle_native_query_should_raise_error_if_not_permission(self): - request = RequestCollection( RequestMethod.GET, self.collection_order, diff --git a/src/agent_toolkit/tests/resources/collections/test_crud_related.py b/src/agent_toolkit/tests/resources/collections/test_crud_related.py index c5f4f035a..830aced18 100644 --- a/src/agent_toolkit/tests/resources/collections/test_crud_related.py +++ b/src/agent_toolkit/tests/resources/collections/test_crud_related.py @@ -20,7 +20,7 @@ ) from forestadmin.agent_toolkit.services.permissions.ip_whitelist_service import IpWhiteListService from forestadmin.agent_toolkit.services.permissions.permission_service import PermissionService -from forestadmin.agent_toolkit.services.serializers.json_api import JsonApiException +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiSerializerException from forestadmin.agent_toolkit.utils.context import Request, RequestMethod, User from forestadmin.agent_toolkit.utils.csv import CsvException from forestadmin.agent_toolkit.utils.id import unpack_id @@ -386,11 +386,7 @@ def test_dispatch_error(self, mock_request_relation_collection: Mock): self.assertEqual(response.status, 500) # -- list - @patch( - "forestadmin.agent_toolkit.resources.collections.crud_related.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list(self, mocked_json_serializer_get: Mock): + def test_list(self): mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] request = RequestRelationCollection( @@ -414,14 +410,6 @@ def test_list(self, mocked_json_serializer_get: Mock): self.datasource, self.permission_service, self.ip_white_list_service, self.options ) - mocked_json_serializer_get.return_value.dump = Mock( - return_value={ - "data": [ - {"type": "order", "attributes": mock_order, "id": mock_order["id"]} for mock_order in mock_orders - ] - } - ) - with patch.object( self.collection_order, "list", new_callable=AsyncMock, return_value=mock_orders ) as mocked_collection_list: @@ -431,26 +419,21 @@ def test_list(self, mocked_json_serializer_get: Mock): self.permission_service.can.assert_any_await(request.user, request.foreign_collection, "browse") self.permission_service.can.reset_mock() - assert response.status == 200 + self.assertEqual(response.status, 200) response_content = json.loads(response.body) - assert isinstance(response_content["data"], list) - assert len(response_content["data"]) == 2 - assert response_content["data"][0]["attributes"]["cost"] == mock_orders[0]["cost"] - assert response_content["data"][0]["id"] == mock_orders[0]["id"] - assert response_content["data"][0]["type"] == "order" - assert response_content["data"][1]["attributes"]["cost"] == mock_orders[1]["cost"] - assert response_content["data"][1]["id"] == mock_orders[1]["id"] - assert response_content["data"][1]["type"] == "order" - - assert response_content["meta"]["decorators"]["0"] == {"id": 10, "search": ["cost"]} - assert response_content["meta"]["decorators"]["1"] == {"id": 11, "search": ["cost"]} - - @patch( - "forestadmin.agent_toolkit.resources.collections.crud_related.JsonApiSerializer.get", - return_value=Mock, - ) - def test_list_errors(self, mocked_json_serializer_get: Mock): - mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] + self.assertTrue(isinstance(response_content["data"], list)) + self.assertEqual(len(response_content["data"]), 2) + self.assertEqual(response_content["data"][0]["attributes"]["cost"], mock_orders[0]["cost"]) + self.assertEqual(response_content["data"][0]["id"], mock_orders[0]["id"]) + self.assertEqual(response_content["data"][0]["type"], "order") + self.assertEqual(response_content["data"][1]["attributes"]["cost"], mock_orders[1]["cost"]) + self.assertEqual(response_content["data"][1]["id"], mock_orders[1]["id"]) + self.assertEqual(response_content["data"][1]["type"], "order") + + self.assertEqual(response_content["meta"]["decorators"]["0"], {"id": 10, "search": ["cost"]}) + self.assertEqual(response_content["meta"]["decorators"]["1"], {"id": 11, "search": ["cost"]}) + + def test_list_error_on_relation_type(self): crud_related_resource = CrudRelatedResource( self.datasource, self.permission_service, self.ip_white_list_service, self.options ) @@ -479,14 +462,21 @@ def test_list_errors(self, mocked_json_serializer_get: Mock): self.permission_service.can.assert_any_await(request.user, request.foreign_collection, "browse") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "ForestException", - "detail": "🌳🌳🌳Unhandled relation type", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "ForestException", + "detail": "🌳🌳🌳Unhandled relation type", + "status": 500, + }, + ) + def test_list_error_on_instance_pk(self): + crud_related_resource = CrudRelatedResource( + self.datasource, self.permission_service, self.ip_white_list_service, self.options + ) # collectionResourceException request_get_params = { "collection_name": "customer", @@ -508,16 +498,32 @@ def test_list_errors(self, mocked_json_serializer_get: Mock): self.permission_service.can.assert_any_await(request.user, request.foreign_collection, "browse") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "CollectionResourceException", - "detail": "🌳🌳🌳primary keys are missing", - "status": 500, + self.assertEqual( + response_content["errors"][0], + { + "name": "CollectionResourceException", + "detail": "🌳🌳🌳primary keys are missing", + "status": 500, + }, + ) + + def test_list_error_on_jsonapi_serialize(self): + mock_orders = [{"id": 10, "cost": 200}, {"id": 11, "cost": 201}] + crud_related_resource = CrudRelatedResource( + self.datasource, self.permission_service, self.ip_white_list_service, self.options + ) + # collectionResourceException + request_get_params = { + "collection_name": "customer", + "relation_name": "order", + "timezone": "Europe/Paris", + "fields[order]": "id,cost", + "pks": "2", } # # JsonApiException - request_get_params["pks"] = "2" request = RequestRelationCollection( RequestMethod.GET, *self.mk_request_customer_order_one_to_many(), @@ -526,20 +532,27 @@ def test_list_errors(self, mocked_json_serializer_get: Mock): client_ip="127.0.0.1", ) self.collection_order.list = AsyncMock(return_value=mock_orders) - mocked_json_serializer_get.return_value.dump = Mock(side_effect=JsonApiException) - response = self.loop.run_until_complete(crud_related_resource.list(request)) + with patch( + "forestadmin.agent_toolkit.resources.collections.crud_related.JsonApiSerializer.serialize", + side_effect=JsonApiSerializerException, + ) as mock_serialize: + response = self.loop.run_until_complete(crud_related_resource.list(request)) + mock_serialize.assert_called() self.permission_service.can.assert_any_await(request.user, request.foreign_collection, "browse") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "JsonApiException", - "detail": "🌳🌳🌳", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "JsonApiSerializerException", + "detail": "🌳🌳🌳", + "status": 500, + }, + ) # CSV def test_csv(self): @@ -743,7 +756,7 @@ def test_add(self): self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 204 + self.assertEqual(response.status, 204) # many to Many request = RequestRelationCollection( @@ -770,7 +783,7 @@ def test_add(self): self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 204 + self.assertEqual(response.status, 204) def test_add_should_associate_polymorphic_one_to_many(self): request = RequestRelationCollection( @@ -804,14 +817,7 @@ def test_add_should_associate_polymorphic_one_to_many(self): mock_collection_update.await_args_list[0].args[2], {"taggable_id": 2, "taggable_type": "order"} ) - @patch( - "forestadmin.agent_toolkit.resources.collections.crud_related.JsonApiSerializer.get", - return_value=Mock, - ) - def test_add_errors( - self, - mocked_json_serializer_get: Mock, - ): + def test_add_error_no_id(self): request_get_params = { "collection_name": "customer", "relation_name": "order", @@ -833,15 +839,34 @@ def test_add_errors( self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "CollectionResourceException", - "detail": "🌳🌳🌳primary keys are missing", - "status": 500, + self.assertEqual( + response_content["errors"][0], + { + "name": "CollectionResourceException", + "detail": "🌳🌳🌳primary keys are missing", + "status": 500, + }, + ) + + def test_add_error_no_id_in_body(self): + request_get_params = { + "collection_name": "customer", + "relation_name": "order", + "timezone": "Europe/Paris", + "pks": "2", # customer id } - # no date body id - request_get_params["pks"] = "2" + request = RequestRelationCollection( + RequestMethod.POST, + *self.mk_request_customer_order_one_to_many(), + body={"data": [{"id": "201", "type": "order"}]}, # body + query=request_get_params, # query + headers={}, + client_ip="127.0.0.1", + ) + + # no body id request = RequestRelationCollection( RequestMethod.POST, *self.mk_request_customer_order_one_to_many(), @@ -852,15 +877,25 @@ def test_add_errors( response = self.loop.run_until_complete(self.crud_related_resource.add(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "ForestException", - "detail": "🌳🌳🌳missing target's id", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "ForestException", + "detail": "🌳🌳🌳missing target's id", + "status": 500, + }, + ) + def test_add_error_unpack_foreign_id(self): # unpack foreign id + request_get_params = { + "collection_name": "customer", + "relation_name": "order", + "timezone": "Europe/Paris", + "pks": "2", # customer id + } request = RequestRelationCollection( RequestMethod.POST, *self.mk_request_customer_order_one_to_many(), @@ -888,7 +923,14 @@ def mocked_unpack_id(schema, pk): "status": 500, } + def test_add_error_bad_relation_type(self): # Unhandled relation type + request_get_params = { + "collection_name": "customer", + "relation_name": "order", + "timezone": "Europe/Paris", + "pks": "2", # customer id + } request = RequestRelationCollection( RequestMethod.POST, self.collection_customer, @@ -910,13 +952,16 @@ def mocked_unpack_id(schema, pk): response = self.loop.run_until_complete(self.crud_related_resource.add(request)) self.permission_service.can.assert_any_await(request.user, request.collection, "edit") self.permission_service.can.reset_mock() - assert response.status == 500 + self.assertEqual(response.status, 500) response_content = json.loads(response.body) - assert response_content["errors"][0] == { - "name": "ForestException", - "detail": "🌳🌳🌳Unhandled relation type", - "status": 500, - } + self.assertEqual( + response_content["errors"][0], + { + "name": "ForestException", + "detail": "🌳🌳🌳Unhandled relation type", + "status": 500, + }, + ) @patch( "forestadmin.agent_toolkit.resources.collections.crud_related.ConditionTreeFactory.match_ids", @@ -1985,7 +2030,3 @@ def test_update_many_to_one(self, mock_match_ids: Mock): ) self.permission_service.can.assert_not_awaited() - self.permission_service.can.assert_not_awaited() - self.permission_service.can.assert_not_awaited() - self.permission_service.can.assert_not_awaited() - self.permission_service.can.assert_not_awaited() diff --git a/src/agent_toolkit/tests/services/serializers/test_jsonapi.py b/src/agent_toolkit/tests/services/serializers/test_jsonapi.py index eb70be8a3..3343ea7ed 100644 --- a/src/agent_toolkit/tests/services/serializers/test_jsonapi.py +++ b/src/agent_toolkit/tests/services/serializers/test_jsonapi.py @@ -1,17 +1,10 @@ -from datetime import date, datetime, timezone -from typing import cast -from unittest import TestCase -from unittest.mock import patch +from datetime import date, datetime, time, timezone +from unittest import TestCase, skip +from uuid import UUID -from forestadmin.agent_toolkit.services.serializers.json_api import ( - JsonApiException, - JsonApiSerializer, - _create_relationship, - _map_attribute_to_marshmallow, - create_json_api_schema, - refresh_json_api_schema, - schema_name, -) +from forestadmin.agent_toolkit.services.serializers.exceptions import JsonApiDeserializerException +from forestadmin.agent_toolkit.services.serializers.json_api_deserializer import JsonApiDeserializer +from forestadmin.agent_toolkit.services.serializers.json_api_serializer import JsonApiSerializer from forestadmin.datasource_toolkit.collections import Collection from forestadmin.datasource_toolkit.datasources import Datasource from forestadmin.datasource_toolkit.interfaces.fields import ( @@ -26,7 +19,6 @@ PolymorphicOneToMany, PolymorphicOneToOne, PrimitiveType, - RelationAlias, ) from forestadmin.datasource_toolkit.interfaces.query.projections import Projection from forestadmin.datasource_toolkit.interfaces.query.projections.factory import ProjectionFactory @@ -36,9 +28,9 @@ class TestJsonApi(TestCase): @classmethod def setUpClass(cls) -> None: cls.datasource: Datasource = Datasource() - Collection.__abstractmethods__ = set() # to instantiate abstract class + Collection.__abstractmethods__ = set() # to instantiate abstract class # type:ignore - cls.collection_person = Collection("Person", cls.datasource) + cls.collection_person = Collection("Person", cls.datasource) # type:ignore cls.collection_person.add_fields( { "person_pk": Column( @@ -100,7 +92,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_profile = Collection("Profile", cls.datasource) + cls.collection_profile = Collection("Profile", cls.datasource) # type:ignore cls.collection_profile.add_fields( { "profile_pk": Column( @@ -161,7 +153,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_order = Collection("Order", cls.datasource) + cls.collection_order = Collection("Order", cls.datasource) # type:ignore cls.collection_order.add_fields( { "order_pk": Column( @@ -211,7 +203,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_order_products = Collection("OrderProducts", cls.datasource) + cls.collection_order_products = Collection("OrderProducts", cls.datasource) # type:ignore cls.collection_order_products.add_fields( { "order_id": Column( @@ -245,7 +237,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_product = Collection("Product", cls.datasource) + cls.collection_product = Collection("Product", cls.datasource) # type:ignore cls.collection_product.add_fields( { "product_pk": Column( @@ -321,7 +313,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_picture = Collection("Picture", cls.datasource) + cls.collection_picture = Collection("Picture", cls.datasource) # type:ignore cls.collection_picture.add_fields( { "picture_pk": Column( @@ -378,7 +370,7 @@ def setUpClass(cls) -> None: } ) - cls.collection_comment = Collection("Comment", cls.datasource) + cls.collection_comment = Collection("Comment", cls.datasource) # type:ignore cls.collection_comment.add_fields( { "comment_pk": Column( @@ -435,6 +427,35 @@ def setUpClass(cls) -> None: } ) + cls.collection_all_types = Collection("AllTypes", cls.datasource) # type:ignore + cls.collection_all_types.add_fields( + { + "all_types_pk": Column( + column_type=PrimitiveType.UUID, + is_primary_key=True, + type=FieldType.COLUMN, + is_read_only=False, + default_value=None, + enum_values=None, + filter_operators=set([Operator.EQUAL, Operator.IN]), + is_sortable=True, + validations=[], + ), + "comment": Column(column_type=PrimitiveType.STRING, type=FieldType.COLUMN), + "enum": Column(column_type="Enum", type="Column", enum_values=["a", "b", "c"]), + "bool": Column(column_type="Boolean", type="Column"), + "int": Column(column_type="Number", type="Column"), + "float": Column(column_type="Number", type="Column"), + "datetime": Column(column_type="Date", type="Column"), + "dateonly": Column(column_type="Dateonly", type="Column"), + "time_only": Column(column_type="Timeonly", type="Column"), + "point": Column(column_type="Point", type="Column"), + "binary": Column(column_type="String", type="Column"), + "json": Column(column_type="Json", type="Column"), + "custom": Column(column_type=[{"id": "Number"}], type="Column"), + } + ) + cls.datasource.add_collection(cls.collection_order) cls.datasource.add_collection(cls.collection_order_products) cls.datasource.add_collection(cls.collection_product) @@ -442,218 +463,302 @@ def setUpClass(cls) -> None: cls.datasource.add_collection(cls.collection_profile) cls.datasource.add_collection(cls.collection_picture) cls.datasource.add_collection(cls.collection_comment) + cls.datasource.add_collection(cls.collection_all_types) -class TestJsonApiSchemaCreation(TestJsonApi): - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_should_create_and_register_schema(self): - create_json_api_schema(self.collection_product) - self.assertIn(schema_name(self.collection_product), JsonApiSerializer.schema.keys()) - schema = JsonApiSerializer.schema[schema_name(self.collection_product)] - self.assertIsNotNone(schema) - self.assertEqual(schema.Meta.type_, self.collection_product.name) - self.assertEqual(schema.Meta.fcollection, self.collection_product) - self.assertEqual(schema.Meta.fcollection, self.collection_product) - self.assertEqual(schema.Meta.self_url, "/forest/Product/{product_id}") - - for name, field_schema in self.collection_product.schema["fields"].items(): - if name == "pk": - self.assertIsNotNone(schema.attributes["product_schema"].get("id")) - # pk is name as original and also "id" - self.assertIsNotNone(schema.attributes["product_schema"].get(name)) - - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_should_raise_if_schema_already_exists(self): - create_json_api_schema(self.collection_product) +class TestJsonApiDeserializer(TestJsonApi): + def test_should_correctly_load_attributes(self): + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": { + "price": 2.23, + "label": "strawberries", + "date_online": "2023-10-10T10:10:10+00:00", + }, + "type": "Product", + } + } - self.assertRaisesRegex( - JsonApiException, - r"The schema has already been created for this collection", - create_json_api_schema, - self.collection_product, + data = JsonApiDeserializer(self.datasource).deserialize(request_body, self.collection_product) + self.assertEqual( + data, + { + "price": 2.23, + "label": "strawberries", + "date_online": datetime(2023, 10, 10, 10, 10, 10, tzinfo=timezone.utc), + }, ) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_refresh_json_api_schema_should_replace_current_schema(self): - with patch.object(JsonApiSerializer, "schema", dict()): - create_json_api_schema(self.collection_product) - existing_schema = JsonApiSerializer.schema[schema_name(self.collection_product)] - refresh_json_api_schema(self.collection_product) - replaced_schema = JsonApiSerializer.schema[schema_name(self.collection_product)] - self.assertNotEqual(existing_schema, replaced_schema) + def test_should_correctly_load_all_types_of_data(self): + deserializer = JsonApiDeserializer(self.datasource) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_refresh_json_api_schema_should_raise_if_schema_does_not_exists(self): - self.assertRaisesRegex( - JsonApiException, r"The schema doesn't exist", refresh_json_api_schema, self.collection_product + request_body = { + "data": { + "id": "b2f47557-8518-4e55-a02b-ed92d113d42d", + "attributes": { + "all_types_pk": "b2f47557-8518-4e55-a02b-ed92d113d42f", + "comment": "record 1", + "enum": "a", + "bool": True, + "int": 10, + "float": 22, + "datetime": "2025-02-03T14:54:56.000255+00:00", + "dateonly": "2025-02-01", + "time_only": "15:35:25", + "point": "12,14", + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElE" + "QVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "custom": [{"id": 1}], + "json": {"a": "a", "b": 2, "c": []}, + }, + "type": "AllTypes", + } + } + data = deserializer.deserialize(request_body, self.collection_all_types) + self.assertEqual( + data, + { + "all_types_pk": UUID("b2f47557-8518-4e55-a02b-ed92d113d42f"), + "comment": "record 1", + "enum": "a", + "bool": True, + "int": 10, + "float": 22, + "datetime": datetime(2025, 2, 3, 14, 54, 56, 255, timezone.utc), + "dateonly": date(2025, 2, 1), + "time_only": time(15, 35, 25), + "point": [12, 14], + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElE" + "QVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "custom": [{"id": 1}], + "json": {"a": "a", "b": 2, "c": []}, + }, ) - def test_map_attribute_to_marshmallow_should_correctly_handled_allow_none(self): - res = _map_attribute_to_marshmallow( - Column( - column_type=PrimitiveType.STRING, - type=FieldType.COLUMN, - is_sortable=True, - default_value=None, - enum_values=None, - is_primary_key=False, - filter_operators=set([Operator.EQUAL, Operator.IN]), - is_read_only=False, - validations=[], - ), + def test_should_correctly_load_int_or_float_from_string_value(self): + deserializer = JsonApiDeserializer(self.datasource) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": { + "price": "2.23", + }, + "type": "Product", + } + } + data = deserializer.deserialize(request_body, self.collection_product) + self.assertEqual( + data, + { + "price": 2.23, + }, ) - self.assertEqual(res.allow_none, True) - res = _map_attribute_to_marshmallow( - Column( - column_type=PrimitiveType.STRING, - type=FieldType.COLUMN, - is_sortable=True, - default_value=None, - enum_values=None, - is_primary_key=False, - filter_operators=set([Operator.EQUAL, Operator.IN]), - is_read_only=True, - validations=[], - ), - ) - self.assertEqual(res.allow_none, True) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": { + "price": "10", + "label": None, + }, + "type": "Product", + } + } - res = _map_attribute_to_marshmallow( - Column( - column_type=PrimitiveType.STRING, - type=FieldType.COLUMN, - is_sortable=True, - default_value=None, - enum_values=None, - is_primary_key=False, - filter_operators=set([Operator.EQUAL, Operator.IN]), - is_read_only=False, - validations=[{"operator": Operator.PRESENT}], - ), + data = deserializer.deserialize(request_body, self.collection_product) + self.assertEqual( + data, + { + "price": 10, + "label": None, + }, ) - self.assertEqual(res.allow_none, False) - def test_create_relationship_should_handle_many_to_one(self): - ret = _create_relationship( - self.collection_order, "customer", cast(RelationAlias, self.collection_order.schema["fields"]["customer"]) + def test_should_correctly_load_many_to_one_relationship(self): + deserializer = JsonApiDeserializer(self.datasource) + + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Orders", + "relationships": {"customer": {"data": {"id": "12", "type": "Persons"}}}, + } + } + data = deserializer.deserialize(request_body, self.collection_order) + self.assertEqual( + data, + { + "customer": 12, + }, ) - self.assertEqual(ret.collection, self.collection_order) - self.assertEqual(ret.related_collection_name, "Person") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "person_pk") - self.assertEqual(ret.many, False) - self.assertEqual(ret.type_, "Person") - self.assertEqual(ret._Relationship__schema, "Person_schema") + @skip("Front end never send toMany relationships") + def test_should_correctly_load_to_many_relations(self): + deserializer = JsonApiDeserializer(self.datasource) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_many_to_many(self): - create_json_api_schema(self.collection_product) - ret = _create_relationship( - self.collection_order, "products", cast(RelationAlias, self.collection_order.schema["fields"]["products"]) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Orders", + "relationships": { + "products": { + "data": [ + {"type": "Products", "id": "0086ebe0-3452-4779-91de-26d14850998c"}, + {"type": "Products", "id": "68dcab0f-2dec-468f-8ebd-ff2752d24b81"}, + ] + }, + "order_products": { + "data": [ + { + "type": "OrderProducts", + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7|833a6308-da81-4363-9448-d101eb593d94", + }, + { + "type": "OrderProducts", + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7|9f9348fe-3d2d-43be-b0be-ff58313b137e", + }, + ] + }, + }, + } + } + data = deserializer.deserialize(request_body, self.collection_order) + self.assertEqual( + data, + { + "products": ["0086ebe0-3452-4779-91de-26d14850998c", "68dcab0f-2dec-468f-8ebd-ff2752d24b81"], + "order_products": ["833a6308-da81-4363-9448-d101eb593d94", "9f9348fe-3d2d-43be-b0be-ff58313b137e"], + "order_pk": UUID("43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7"), + # "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + }, ) - self.assertEqual(ret.collection, self.collection_order) - self.assertEqual(ret.related_collection_name, "Product") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "product_pk") # it should be pk; but jsonapi always use id - self.assertEqual(ret.many, True) - self.assertEqual(ret.type_, "Product") - self.assertEqual(ret._Relationship__schema, "Product_schema") + def test_should_not_deserialize_toMany_relations(self): + deserializer = JsonApiDeserializer(self.datasource) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_one_to_many(self): - create_json_api_schema(self.collection_order) - ret = _create_relationship( - self.collection_person, "orders", cast(RelationAlias, self.collection_person.schema["fields"]["orders"]) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Orders", + "relationships": { + "products": { + "data": [ + {"type": "Products", "id": "0086ebe0-3452-4779-91de-26d14850998c"}, + ] + } + }, + } + } + self.assertRaisesRegex( + JsonApiDeserializerException, + "We shouldn't deserialize toMany relations", + deserializer.deserialize, + request_body, + self.collection_order, ) - self.assertEqual(ret.collection, self.collection_person) - self.assertEqual(ret.related_collection_name, "Order") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "order_pk") - self.assertEqual(ret.many, True) - self.assertEqual(ret.type_, "Order") - self.assertEqual(ret._Relationship__schema, "Order_schema") + def test_should_correctly_load_polymorphic_many_to_one_relation(self): + deserializer = JsonApiDeserializer(self.datasource) + request_body = { + "data": { + "type": "Comments", + "attributes": { + "comment": "I like it a lot.", + }, + "relationships": { + "target_object": { + "data": { + "type": "Product", + "id": "1806bdb7-5db4-46a1-acca-9a00f8a670dd", + }, + } + }, + } + } - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_one_to_one(self): - create_json_api_schema(self.collection_profile) - ret = _create_relationship( - self.collection_person, "profile", cast(RelationAlias, self.collection_person.schema["fields"]["profile"]) + data = deserializer.deserialize(request_body, self.collection_comment) + self.assertEqual( + data, + { + "target_id": UUID("1806bdb7-5db4-46a1-acca-9a00f8a670dd"), + "target_type": "Product", + "comment": "I like it a lot.", + }, ) - self.assertEqual(ret.collection, self.collection_person) - self.assertEqual(ret.related_collection_name, "Profile") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "profile_pk") - self.assertEqual(ret.many, False) - self.assertEqual(ret.type_, "Profile") - self.assertEqual(ret._Relationship__schema, "Profile_schema") + def test_should_ignore_null_many_to_one_relations(self): + deserializer = JsonApiDeserializer(self.datasource) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Orders", + "relationships": {"customer": {"data": None}}, + } + } + data = deserializer.deserialize(request_body, self.collection_order) + self.assertEqual(data, {"customer": None}) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_polymorphic_many_to_one(self): - create_json_api_schema(self.collection_profile) - create_json_api_schema(self.collection_product) - ret = _create_relationship( - self.collection_picture, - "target_object", - cast(RelationAlias, self.collection_picture.schema["fields"]["target_object"]), - ) + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Orders", + "relationships": {"customer": {"data": {}}}, + } + } + data = deserializer.deserialize(request_body, self.collection_order) + self.assertEqual(data, {"customer": None}) - self.assertEqual(ret.collection, self.collection_picture) - self.assertEqual(ret.related_collection_name, ["Product", "Profile"]) - self.assertEqual(ret.forest_is_polymorphic, True) - # self.assertEqual(ret.id_field, "id") - self.assertEqual(ret.many, False) - self.assertEqual(ret.forest_relation, self.collection_picture.schema["fields"]["target_object"]) - self.assertEqual(ret.type_, ["Product", "Profile"]) - self.assertEqual(ret._Relationship__schema, "['Product', 'Profile']_schema") + def test_should_correctly_load_polymorphic_one_to_one_relation(self): + deserializer = JsonApiDeserializer(self.datasource) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_polymorphic_one_to_one(self): - create_json_api_schema(self.collection_picture) - create_json_api_schema(self.collection_product) - ret = _create_relationship( - self.collection_profile, - "picture", - cast(RelationAlias, self.collection_profile.schema["fields"]["picture"]), + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "Profile", + "relationships": { + "picture": {"data": {"id": "34d0adfe-823d-4fb4-9c3e-ac241887aa1c", "type": "Picture"}} + }, + } + } + data = deserializer.deserialize(request_body, self.collection_profile) + self.assertEqual( + data, + { + "picture": UUID("34d0adfe-823d-4fb4-9c3e-ac241887aa1c"), + }, ) - self.assertEqual(ret.collection, self.collection_profile) - self.assertEqual(ret.related_collection_name, "Picture") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "picture_pk") - self.assertEqual(ret.many, False) - self.assertEqual(ret.type_, "Picture") - self.assertEqual(ret._Relationship__schema, "Picture_schema") + def test_should_correctly_load_one_to_one_relation(self): + deserializer = JsonApiDeserializer(self.datasource) - @patch.object(JsonApiSerializer, "schema", dict()) - def test_create_relationship_should_handle_polymorphic_one_to_many(self): - create_json_api_schema(self.collection_picture) - create_json_api_schema(self.collection_product) - ret = _create_relationship( - self.collection_profile, - "comments", - cast(RelationAlias, self.collection_profile.schema["fields"]["comments"]), + request_body = { + "data": { + "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", + "attributes": {}, + "type": "person", + "relationships": {"profile": {"data": {"id": "12", "type": "Profile"}}}, + } + } + data = deserializer.deserialize(request_body, self.collection_person) + self.assertEqual( + data, + { + "profile": 12, + }, ) - self.assertEqual(ret.collection, self.collection_profile) - self.assertEqual(ret.related_collection_name, "Comment") - self.assertEqual(ret.forest_is_polymorphic, False) - self.assertEqual(ret.id_field, "comment_pk") - self.assertEqual(ret.many, True) - self.assertEqual(ret.type_, "Comment") - self.assertEqual(ret._Relationship__schema, "Comment_schema") - -class TestJsonApiSchemaDump(TestJsonApi): +class TestJsonApiSerializer(TestJsonApi): @classmethod - def setUpClass(cls): + def setUpClass(cls) -> None: super().setUpClass() - for collection in cls.datasource.collections: - create_json_api_schema(collection) cls.person_records = [ { @@ -713,17 +818,45 @@ def setUpClass(cls): }, }, ] - - @classmethod - def tearDownClass(cls) -> None: - JsonApiSerializer.schema = dict() - return super().tearDownClass() + cls.all_types_records = [ + { + "all_types_pk": UUID("b2f47557-8518-4e55-a02b-ed92d113d42f"), + "comment": "record 1", + "enum": "a", + "bool": True, + "int": 10, + "float": 22.3, + "datetime": datetime(2025, 2, 3, 14, 54, 56, 255, timezone.utc), + "dateonly": date(2025, 2, 1), + "time_only": time(15, 35, 25), + "point": [12, 14], + "json": [{"a": "a", "b": 2, "c": []}], + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/" + "w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "custom": [{"id": 1}], + }, + { + "all_types_pk": "c578ccd6-3dd0-4315-87f3-e200d80dd6f9", + "comment": "record 1", + "enum": "b", + "bool": False, + "int": None, + "float": None, + "datetime": datetime(2025, 2, 3, 14, 54, 56, 255, timezone.utc).isoformat(), + "dateonly": date(2025, 2, 1).isoformat(), + "time_only": time(15, 35, 25).isoformat(), + "point": (12, 14), + "json": {"a": "a", "b": 2, "c": []}, + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4//8/" + "w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "custom": [{"id": 2}], + }, + ] def test_should_correctly_dump_attributes_according_to_projection(self): projection = ProjectionFactory.all(self.collection_product, allow_nested=False) - schema = JsonApiSerializer.get(self.collection_product)(projections=projection) + dumped = JsonApiSerializer(self.datasource, projection).serialize(self.product_records, self.collection_product) - dumped = schema.dump(self.product_records, many=True) self.assertEqual( dumped, { @@ -754,9 +887,63 @@ def test_should_correctly_dump_attributes_according_to_projection(self): }, ) + def test_should_correctly_dump_all_data_types(self): + projection = ProjectionFactory.all(self.collection_all_types, allow_nested=False) + serializer = JsonApiSerializer(self.datasource, projection) + dumped = serializer.serialize(self.all_types_records, self.collection_all_types) + self.assertEqual( + dumped, + { + "data": [ + { + "type": "AllTypes", + "id": "b2f47557-8518-4e55-a02b-ed92d113d42f", + "attributes": { + "all_types_pk": "b2f47557-8518-4e55-a02b-ed92d113d42f", + "comment": "record 1", + "enum": "a", + "bool": True, + "int": 10, + "float": 22.3, + "datetime": "2025-02-03T14:54:56.000255+00:00", + "time_only": "15:35:25", + "dateonly": "2025-02-01", + "point": [12, 14], + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElE" + "QVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "json": [{"a": "a", "b": 2, "c": []}], + "custom": [{"id": 1}], + }, + "links": {"self": "/forest/AllTypes/b2f47557-8518-4e55-a02b-ed92d113d42f"}, + }, + { + "type": "AllTypes", + "id": "c578ccd6-3dd0-4315-87f3-e200d80dd6f9", + "attributes": { + "all_types_pk": "c578ccd6-3dd0-4315-87f3-e200d80dd6f9", + "comment": "record 1", + "enum": "b", + "bool": False, + "int": None, + "float": None, + "datetime": "2025-02-03T14:54:56.000255+00:00", + "time_only": "15:35:25", + "dateonly": "2025-02-01", + "point": (12, 14), + "binary": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHE" + "lEQVQI12P4//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg==", + "json": {"a": "a", "b": 2, "c": []}, + "custom": [{"id": 2}], + }, + "links": {"self": "/forest/AllTypes/c578ccd6-3dd0-4315-87f3-e200d80dd6f9"}, + }, + ] + }, + ) + def test_should_correctly_dump_int_or_float_from_string_value(self): projection = ProjectionFactory.all(self.collection_product, allow_nested=False) - schema = JsonApiSerializer.get(self.collection_product)(projections=projection) + serializer = JsonApiSerializer(self.datasource, projection) records = [ {**self.product_records[0]}, @@ -765,7 +952,7 @@ def test_should_correctly_dump_int_or_float_from_string_value(self): records[0]["price"] = "2.23" records[1]["price"] = "10" - dumped = schema.dump(records, many=True) + dumped = serializer.serialize(records, self.collection_product) self.assertEqual( dumped, { @@ -797,15 +984,13 @@ def test_should_correctly_dump_int_or_float_from_string_value(self): ) def test_should_correctly_dump_many_to_one_according_to_projection(self): - schema = JsonApiSerializer.get(self.collection_order)( - projections=Projection("order_pk", "customer_id", "customer:person_pk", "customer:first_name") - ) - record = {**self.order_records[0]} record["customer"] = {**self.person_records[0]} record["customer_id"] = record["customer"]["person_pk"] + dumped = JsonApiSerializer( + self.datasource, Projection("order_pk", "customer_id", "customer:person_pk", "customer:first_name") + ).serialize(record, self.collection_order) - dumped = schema.dump(record, many=False) self.assertEqual( dumped, { @@ -845,23 +1030,24 @@ def test_should_correctly_dump_to_many_according_to_projection(self): toMany relations should not be serialize (and it's not by the datasource) the only think to do is to fill [data/relationships/$relation/links/related/href] """ - schema = JsonApiSerializer.get(self.collection_order)( - projections=Projection( + + record = {**self.order_records[0]} + record["customer_id"] = self.person_records[0]["person_pk"] + record["products"] = None + record["order_products"] = None + + dumped = JsonApiSerializer( + self.datasource, + Projection( "order_pk", "customer_id", "products:label", "products:price", "products:product_pk", "order_products:product_id", - ) - ) - - record = {**self.order_records[0]} - record["customer_id"] = self.person_records[0]["person_pk"] - record["products"] = None - record["order_products"] = None + ), + ).serialize(record, self.collection_order) - dumped = schema.dump(record, many=False) self.assertEqual( dumped, { @@ -898,10 +1084,10 @@ def test_should_correctly_dump_to_many_according_to_projection(self): ) def test_should_correctly_dump_polymorphic_many_to_one(self): - schema = JsonApiSerializer.get(self.collection_comment)( - projections=Projection("comment_pk", "comment", "target_id", "target_type", "target_object:*") - ) - dumped = schema.dump(self.comments_records, many=True) + dumped = JsonApiSerializer( + self.datasource, Projection("comment_pk", "comment", "target_id", "target_type", "target_object:*") + ).serialize(self.comments_records, self.collection_comment) + self.assertEqual( dumped, { @@ -1033,18 +1219,10 @@ def test_should_correctly_dump_polymorphic_many_to_one(self): ) def test_should_ignore_polymorphic_many_to_one_if_type_is_unknown(self): - schema = JsonApiSerializer.get(self.collection_comment)( - projections=Projection("comment_pk", "comment", "target_id", "target_type", "target_object:*") - ) records = [{**self.comments_records[0], "target_type": "Unknown"}] - with patch("forestadmin.agent_toolkit.services.serializers.json_api.ForestLogger.log") as log_method: - dumped = schema.dump(records, many=True) - log_method.assert_called_once_with( - "warning", - "Trying to serialize a polymorphic relationship (Comment.target_object for record " - "0b622590-c823-4d2f-84e6-bbbdd31c8af8) of type Unknown; but this type is not known by forest. " - "Ignoring and setting this relationship to None.", - ) + dumped = JsonApiSerializer( + self.datasource, Projection("comment_pk", "comment", "target_id", "target_type", "target_object:*") + ).serialize(records, self.collection_comment) self.assertEqual( dumped, @@ -1075,198 +1253,3 @@ def test_should_ignore_polymorphic_many_to_one_if_type_is_unknown(self): ], }, ) - - -class TestJsonApiSchemaLoad(TestJsonApi): - @classmethod - def setUpClass(cls): - super().setUpClass() - for collection in cls.datasource.collections: - create_json_api_schema(collection) - - @classmethod - def tearDownClass(cls) -> None: - JsonApiSerializer.schema = dict() - return super().tearDownClass() - - def test_should_correctly_load_attributes(self): - schema = JsonApiSerializer.get(self.collection_product)() - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": { - "price": 2.23, - "label": "strawberries", - "date_online": "2023-10-10T10:10:10+00:00", - }, - "type": "Product", - } - } - - data = schema.load(request_body) - self.assertEqual( - data, - { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "price": 2.23, - "label": "strawberries", - "date_online": datetime(2023, 10, 10, 10, 10, 10, tzinfo=timezone.utc), - }, - ) - # as this line agent_toolkit/forestadmin/agent_toolkit/resources/collections/crud.py#L245C1-L246C1 - # it seems normal json api doesn't load the primary key in another field than id - - def test_should_correctly_load_int_or_float_from_string_value(self): - schema = JsonApiSerializer.get(self.collection_product)() - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": { - "price": "2.23", - }, - "type": "Product", - } - } - - data = schema.load(request_body) - self.assertEqual( - data, - { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "price": 2.23, - }, - ) - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": { - "price": "10", - }, - "type": "Product", - } - } - - data = schema.load(request_body) - self.assertEqual( - data, - { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "price": 10, - }, - ) - - def test_should_correctly_load_many_to_one_relationship(self): - schema = JsonApiSerializer.get(self.collection_order)() - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": {}, - "type": "Orders", - "relationships": { - "customer": {"data": {"id": "12", "type": "Persons"}}, - "products": {"data": []}, - "order_products": {"data": []}, - }, - } - } - data = schema.load(request_body) - self.assertEqual( - data, - { - "customer": 12, - "products": [], - "order_products": [], - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - }, - ) - - def test_should_correctly_load_to_many_relations(self): - schema = JsonApiSerializer.get(self.collection_order)() - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": {}, - "type": "Orders", - "relationships": { - "products": { - "data": [ - {"type": "Products", "id": "0086ebe0-3452-4779-91de-26d14850998c"}, - {"type": "Products", "id": "68dcab0f-2dec-468f-8ebd-ff2752d24b81"}, - ] - }, - "order_products": { - "data": [ - {"type": "OrderProducts", "id": "833a6308-da81-4363-9448-d101eb593d94"}, - {"type": "OrderProducts", "id": "9f9348fe-3d2d-43be-b0be-ff58313b137e"}, - ] - }, - }, - } - } - data = schema.load(request_body) - self.assertEqual( - data, - { - "products": ["0086ebe0-3452-4779-91de-26d14850998c", "68dcab0f-2dec-468f-8ebd-ff2752d24b81"], - "order_products": ["833a6308-da81-4363-9448-d101eb593d94", "9f9348fe-3d2d-43be-b0be-ff58313b137e"], - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - }, - ) - - def test_should_correctly_load_polymorphic_many_to_one_relation(self): - schema = JsonApiSerializer.get(self.collection_comment)() - request_body = { - "data": { - "type": "Comments", - "attributes": { - "comment": "I like it a lot.", - }, - "relationships": { - "target_object": { - "data": { - "type": "Product", - "id": "1806bdb7-5db4-46a1-acca-9a00f8a670dd", - }, - } - }, - } - } - - data = schema.load(request_body) - self.assertEqual( - data, - { - "target_id": "1806bdb7-5db4-46a1-acca-9a00f8a670dd", - "target_type": "Product", - "comment": "I like it a lot.", - }, - ) - - def test_should_ignore_null_many_to_one_relations(self): - schema = JsonApiSerializer.get(self.collection_order)() - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": {}, - "type": "Orders", - "relationships": {"customer": {"data": None}}, - } - } - data = schema.load(request_body) - self.assertEqual(data, {"id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7"}) - - request_body = { - "data": { - "id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7", - "attributes": {}, - "type": "Orders", - "relationships": {"customer": {"data": {}}}, - } - } - data = schema.load(request_body) - self.assertEqual(data, {"id": "43661dae-97c3-4ea9-bd43-a6d8ac3f4ca7"}) diff --git a/src/agent_toolkit/tests/test_agent_toolkit.py b/src/agent_toolkit/tests/test_agent_toolkit.py index e3036b7bc..f3cbedadd 100644 --- a/src/agent_toolkit/tests/test_agent_toolkit.py +++ b/src/agent_toolkit/tests/test_agent_toolkit.py @@ -252,10 +252,8 @@ def test_customize_datasource( agent.customizer.customize_collection.assert_called_once_with(collection_name) - @patch("forestadmin.agent_toolkit.agent.create_json_api_schema") def test_start( self, - mocked_create_json_api_schema, mocked_schema_emitter__get_serialized_schema, mocked_forest_http_api__send_schema, mocked_native_query_resource, @@ -283,12 +281,10 @@ def test_start( self.loop.run_until_complete(agent._start()) self.assertEqual(logger.output, ["DEBUG:forestadmin:Starting agent", "DEBUG:forestadmin:Agent started"]) - mocked_create_json_api_schema.assert_called_once_with("fake_collection") mocked_schema_emitter__get_serialized_schema.assert_called_once() mocked_forest_http_api__send_schema.assert_called_once() # test we can only launch start once - mocked_create_json_api_schema.reset_mock() mocked_schema_emitter__get_serialized_schema.reset_mock() mocked_forest_http_api__send_schema.reset_mock() @@ -296,7 +292,6 @@ def test_start( self.loop.run_until_complete(agent._start()) self.assertEqual(logger.output, ["DEBUG:forestadmin:Agent already started."]) - mocked_create_json_api_schema.assert_not_called() mocked_schema_emitter__get_serialized_schema.assert_not_called() mocked_forest_http_api__send_schema.assert_not_called() diff --git a/src/datasource_django/forestadmin/datasource_django/utils/query_factory.py b/src/datasource_django/forestadmin/datasource_django/utils/query_factory.py index 4bc12d090..fa14ecb05 100644 --- a/src/datasource_django/forestadmin/datasource_django/utils/query_factory.py +++ b/src/datasource_django/forestadmin/datasource_django/utils/query_factory.py @@ -184,7 +184,7 @@ def mk_aggregate( } qs = qs.aggregate(**aggregate_kwargs) - value = float(qs[aggregated_field]) + value = float(qs.get(aggregated_field, 0)) return [{"value": value, "group": {}}] else: