diff --git a/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/decorator_stack.py b/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/decorator_stack.py index 78af2edbb..5a1d297fd 100644 --- a/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/decorator_stack.py +++ b/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/decorator_stack.py @@ -7,6 +7,7 @@ from forestadmin.datasource_toolkit.decorators.datasource_decorator import DatasourceDecorator from forestadmin.datasource_toolkit.decorators.empty.collection import EmptyCollectionDecorator from forestadmin.datasource_toolkit.decorators.hook.collections import CollectionHookDecorator +from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator from forestadmin.datasource_toolkit.decorators.operators_emulate.collections import OperatorsEmulateCollectionDecorator from forestadmin.datasource_toolkit.decorators.operators_equivalence.collections import ( OperatorEquivalenceCollectionDecorator, @@ -40,6 +41,8 @@ def __init__(self, datasource: Datasource) -> None: last = self.early_op_emulate = DatasourceDecorator(last, OperatorsEmulateCollectionDecorator) last = self.early_op_equivalence = DatasourceDecorator(last, OperatorEquivalenceCollectionDecorator) last = self.relation = DatasourceDecorator(last, RelationCollectionDecorator) + # lazy join is just before relation, to avoid relations to do useless stuff + last = self.lazy_joins = DatasourceDecorator(last, LazyJoinCollectionDecorator) last = self.late_computed = DatasourceDecorator(last, ComputedCollectionDecorator) last = self.late_op_emulate = DatasourceDecorator(last, OperatorsEmulateCollectionDecorator) last = self.late_op_equivalence = DatasourceDecorator(last, OperatorEquivalenceCollectionDecorator) diff --git a/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/__init__.py b/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/collection.py b/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/collection.py new file mode 100644 index 000000000..e89783717 --- /dev/null +++ b/src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/collection.py @@ -0,0 +1,97 @@ +from typing import List, Union, cast + +from forestadmin.agent_toolkit.utils.context import User +from forestadmin.datasource_toolkit.decorators.collection_decorator import CollectionDecorator +from forestadmin.datasource_toolkit.interfaces.fields import ManyToOne, is_many_to_one +from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf +from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter +from forestadmin.datasource_toolkit.interfaces.query.filter.unpaginated import Filter +from forestadmin.datasource_toolkit.interfaces.query.projections import Projection +from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias + + +class LazyJoinCollectionDecorator(CollectionDecorator): + async def list(self, caller: User, filter_: PaginatedFilter, projection: Projection) -> List[RecordsDataAlias]: + simplified_projection = self._get_projection_without_useless_joins(projection) + + refined_filter = cast(PaginatedFilter, await self._refine_filter(caller, filter_)) + ret = await self.child_collection.list(caller, refined_filter, simplified_projection) + + return self._apply_joins_on_records(projection, simplified_projection, ret) + + async def _refine_filter( + self, caller: User, _filter: Union[Filter, PaginatedFilter, None] + ) -> Union[Filter, PaginatedFilter, None]: + if _filter is None or _filter.condition_tree is None: + return _filter + + _filter.condition_tree = _filter.condition_tree.replace( + lambda leaf: ( + ConditionTreeLeaf( + self._get_fk_field_for_projection(leaf.field), + leaf.operator, + leaf.value, + ) + if self._is_useless_join(leaf.field.split(":")[0], _filter.condition_tree.projection) + else leaf + ) + ) + + return _filter + + def _is_useless_join(self, relation: str, projection: Projection) -> bool: + relation_schema = self.schema["fields"][relation] + sub_projections = projection.relations[relation] + + return ( + is_many_to_one(relation_schema) + and len(sub_projections) == 1 + and sub_projections[0] == relation_schema["foreign_key_target"] + ) + + def _get_fk_field_for_projection(self, projection: str) -> str: + relation_name = projection.split(":")[0] + relation_schema = cast(ManyToOne, self.schema["fields"][relation_name]) + + return relation_schema["foreign_key"] + + def _get_projection_without_useless_joins(self, projection: Projection) -> Projection: + returned_projection = Projection(*projection) + for relation, relation_projections in projection.relations.items(): + if self._is_useless_join(relation, projection): + # remove foreign key target from projection + returned_projection.remove(f"{relation}:{relation_projections[0]}") + + # add foreign keys to projection + fk_field = self._get_fk_field_for_projection(relation) + if fk_field not in returned_projection: + returned_projection.append(fk_field) + + return returned_projection + + def _apply_joins_on_records( + self, initial_projection: Projection, requested_projection: Projection, records: List[RecordsDataAlias] + ) -> List[RecordsDataAlias]: + if requested_projection == initial_projection: + return records + + projections_to_add = Projection(*[p for p in initial_projection if p not in requested_projection]) + projections_to_rm = Projection(*[p for p in requested_projection if p not in initial_projection]) + + for record in records: + # add to records relation:id + for relation, relation_projections in projections_to_add.relations.items(): + relation_schema = self.schema["fields"][relation] + + if is_many_to_one(relation_schema): + record[relation] = { + relation_projections[0]: record[ + self._get_fk_field_for_projection(f"{relation}:{relation_projections[0]}") + ] + } + + # remove foreign keys + for projection in projections_to_rm: + del record[projection] + + return records diff --git a/src/datasource_toolkit/forestadmin/datasource_toolkit/interfaces/query/condition_tree/nodes/base.py b/src/datasource_toolkit/forestadmin/datasource_toolkit/interfaces/query/condition_tree/nodes/base.py index 9688aa67c..889df3060 100644 --- a/src/datasource_toolkit/forestadmin/datasource_toolkit/interfaces/query/condition_tree/nodes/base.py +++ b/src/datasource_toolkit/forestadmin/datasource_toolkit/interfaces/query/condition_tree/nodes/base.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import abc import sys -from typing import Any, Awaitable, Callable, Dict, List, TypeVar, Union +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, TypeVar, Union if sys.version_info >= (3, 9): import zoneinfo @@ -13,6 +15,9 @@ from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias from typing_extensions import TypedDict +if TYPE_CHECKING: + from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf + class ConditionTreeException(DatasourceToolkitException): pass @@ -75,7 +80,7 @@ class ConditionTreeComponent(TypedDict): HandlerResult = TypeVar("HandlerResult") -HandlerAlias = Callable[[ConditionTree], HandlerResult] +HandlerAlias = Callable[["ConditionTreeLeaf"], HandlerResult] ReplacerAlias = HandlerAlias[Union[ConditionTree, ConditionTreeComponent]] AsyncReplacerAlias = HandlerAlias[Awaitable[Union[ConditionTree, ConditionTreeComponent]]] CallbackAlias = HandlerAlias[None] diff --git a/src/datasource_toolkit/tests/decorators/lazy_join/__init__.py b/src/datasource_toolkit/tests/decorators/lazy_join/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/datasource_toolkit/tests/decorators/lazy_join/test_lazy_join_decorator.py b/src/datasource_toolkit/tests/decorators/lazy_join/test_lazy_join_decorator.py new file mode 100644 index 000000000..64f8c789e --- /dev/null +++ b/src/datasource_toolkit/tests/decorators/lazy_join/test_lazy_join_decorator.py @@ -0,0 +1,227 @@ +import asyncio +import sys +from unittest import TestCase +from unittest.mock import AsyncMock, patch + +if sys.version_info >= (3, 9): + import zoneinfo +else: + from backports import zoneinfo + +from forestadmin.agent_toolkit.utils.context import User +from forestadmin.datasource_toolkit.collections import Collection +from forestadmin.datasource_toolkit.datasources import Datasource +from forestadmin.datasource_toolkit.decorators.datasource_decorator import DatasourceDecorator +from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator +from forestadmin.datasource_toolkit.interfaces.fields import Column, FieldType, ManyToOne, OneToMany, PrimitiveType +from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf +from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter +from forestadmin.datasource_toolkit.interfaces.query.projections import Projection + + +class TestEmptyCollectionDecorator(TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.loop = asyncio.new_event_loop() + Collection.__abstractmethods__ = set() # to instantiate abstract class + cls.datasource: Datasource = Datasource() + cls.datasource.get_collection = lambda x: cls.datasource._collections[x] + cls.mocked_caller = User( + rendering_id=1, + user_id=1, + tags={}, + email="dummy@user.fr", + first_name="dummy", + last_name="user", + team="operational", + timezone=zoneinfo.ZoneInfo("Europe/Paris"), + request={"ip": "127.0.0.1"}, + ) + + cls.collection_book = Collection("Book", cls.datasource) + cls.collection_book.add_fields( + { + "id": Column(column_type=PrimitiveType.NUMBER, is_primary_key=True, type=FieldType.COLUMN), + "author_id": Column(column_type=PrimitiveType.NUMBER, type=FieldType.COLUMN), + "author": ManyToOne( + foreign_collection="Person", + foreign_key="author_id", + foreign_key_target="id", + type=FieldType.MANY_TO_ONE, + ), + "title": Column( + column_type=PrimitiveType.STRING, + type=FieldType.COLUMN, + ), + } + ) + cls.collection_person = Collection("Person", cls.datasource) + cls.collection_person.add_fields( + { + "id": Column(column_type=PrimitiveType.NUMBER, is_primary_key=True, type=FieldType.COLUMN), + "first_name": Column(column_type=PrimitiveType.STRING, type=FieldType.COLUMN), + "last_name": Column(column_type=PrimitiveType.STRING, type=FieldType.COLUMN), + "books": OneToMany(origin_key="author_id", origin_key_target="id", foreign_collection="Book"), + } + ) + + cls.datasource.add_collection(cls.collection_book) + cls.datasource.add_collection(cls.collection_person) + cls.datasource_decorator = DatasourceDecorator(cls.datasource, LazyJoinCollectionDecorator) + cls.decorated_book_collection = cls.datasource_decorator.get_collection("Book") + + def test_should_not_join_when_projection_ask_for_target_field_only(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}], + ) as mock_list: + result = self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter({}), + Projection("id", "author:id"), + ) + ) + mock_list.assert_awaited_once_with(self.mocked_caller, PaginatedFilter({}), Projection("id", "author_id")) + + # should contain author object, without author_id FK + self.assertEqual([{"id": 1, "author": {"id": 2}}, {"id": 2, "author": {"id": 5}}], result) + + def test_should_join_when_projection_ask_for_multiple_fields_in_foreign_collection(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[ + {"id": 1, "author": {"id": 2, "first_name": "Isaac"}}, + {"id": 2, "author": {"id": 5, "first_name": "J.K."}}, + ], + ) as mock_list: + result = self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter({}), + Projection("id", "author:id", "author:first_name"), + ) + ) + mock_list.assert_awaited_once_with( + self.mocked_caller, PaginatedFilter({}), Projection("id", "author:id", "author:first_name") + ) + + self.assertEqual( + [ + {"id": 1, "author": {"id": 2, "first_name": "Isaac"}}, + {"id": 2, "author": {"id": 5, "first_name": "J.K."}}, + ], + result, + ) + + def test_should_not_join_when_condition_tree_is_on_foreign_key_target(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}, {"id": 3, "author_id": 5}], + ) as mock_list: + self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:id", "in", [2, 5])}), + Projection("id", "author:id"), + ) + ) + mock_list.assert_awaited_once_with( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author_id", "in", [2, 5])}), + Projection("id", "author_id"), + ) + + def test_should_join_when_condition_tree_is_on_foreign_collection_fields(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}, {"id": 3, "author_id": 5}], + ) as mock_list: + self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter( + {"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])} + ), + Projection("id", "author:id"), + ) + ) + mock_list.assert_awaited_once_with( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}), + Projection("id", "author_id"), + ) + + def test_should_disable_join_on_condition_tree_but_not_in_projection(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[ + {"id": 1, "author": {"first_name": "Isaac"}}, + {"id": 2, "author": {"first_name": "J.K."}}, + {"id": 3, "author": {"first_name": "J.K."}}, + ], + ) as mock_list: + response = self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:id", "in", [2, 5])}), + Projection("id", "author:first_name"), + ) + ) + mock_list.assert_awaited_once_with( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author_id", "in", [2, 5])}), + Projection("id", "author:first_name"), + ) + self.assertEqual( + [ + {"id": 1, "author": {"first_name": "Isaac"}}, + {"id": 2, "author": {"first_name": "J.K."}}, + {"id": 3, "author": {"first_name": "J.K."}}, + ], + response, + ) + + def test_should_disable_join_on_projection_but_not_in_condition_tree(self): + with patch.object( + self.collection_book, + "list", + new_callable=AsyncMock, + return_value=[ + {"id": 1, "author_id": 2}, + {"id": 2, "author_id": 5}, + {"id": 3, "author_id": 5}, + ], + ) as mock_list: + response = self.loop.run_until_complete( + self.decorated_book_collection.list( + self.mocked_caller, + PaginatedFilter( + {"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])} + ), + Projection("id", "author:id"), + ) + ) + mock_list.assert_awaited_once_with( + self.mocked_caller, + PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}), + Projection("id", "author_id"), + ) + self.assertEqual( + [ + {"id": 1, "author": {"id": 2}}, + {"id": 2, "author": {"id": 5}}, + {"id": 3, "author": {"id": 5}}, + ], + response, + ) diff --git a/src/datasource_toolkit/tests/decorators/test_decorator_stack.py b/src/datasource_toolkit/tests/decorators/test_decorator_stack.py index 07fd3f2be..7e9a84341 100644 --- a/src/datasource_toolkit/tests/decorators/test_decorator_stack.py +++ b/src/datasource_toolkit/tests/decorators/test_decorator_stack.py @@ -9,10 +9,12 @@ from forestadmin.datasource_toolkit.decorators.decorator_stack import DecoratorStack from forestadmin.datasource_toolkit.decorators.empty.collection import EmptyCollectionDecorator from forestadmin.datasource_toolkit.decorators.hook.collections import CollectionHookDecorator +from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator from forestadmin.datasource_toolkit.decorators.operators_emulate.collections import OperatorsEmulateCollectionDecorator from forestadmin.datasource_toolkit.decorators.operators_equivalence.collections import ( OperatorEquivalenceCollectionDecorator, ) +from forestadmin.datasource_toolkit.decorators.override.collection import OverrideCollectionDecorator from forestadmin.datasource_toolkit.decorators.relation.collections import RelationCollectionDecorator from forestadmin.datasource_toolkit.decorators.rename_field.collections import RenameFieldCollectionDecorator from forestadmin.datasource_toolkit.decorators.schema.collection import SchemaCollectionDecorator @@ -68,11 +70,13 @@ def test_creation_instantiate_all_decorator_with_datasource_decorator(self): DecoratorStack(self.datasource) call_list = [ + call(self.datasource, OverrideCollectionDecorator), call(self.datasource, EmptyCollectionDecorator), call(self.datasource, ComputedCollectionDecorator), call(self.datasource, OperatorsEmulateCollectionDecorator), call(self.datasource, OperatorEquivalenceCollectionDecorator), call(self.datasource, RelationCollectionDecorator), + call(self.datasource, LazyJoinCollectionDecorator), call(self.datasource, ComputedCollectionDecorator), call(self.datasource, OperatorsEmulateCollectionDecorator), call(self.datasource, OperatorEquivalenceCollectionDecorator),