perf(table_join): avoid join if we only want the target field of the relation

src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/decorator_stack.py

@@ -7,6 +7,7 @@
 from forestadmin.datasource_toolkit.decorators.datasource_decorator import DatasourceDecorator
 from forestadmin.datasource_toolkit.decorators.empty.collection import EmptyCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.hook.collections import CollectionHookDecorator
+from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.operators_emulate.collections import OperatorsEmulateCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.operators_equivalence.collections import (
     OperatorEquivalenceCollectionDecorator,
@@ -40,6 +41,8 @@ def __init__(self, datasource: Datasource) -> None:
         last = self.early_op_emulate = DatasourceDecorator(last, OperatorsEmulateCollectionDecorator)
         last = self.early_op_equivalence = DatasourceDecorator(last, OperatorEquivalenceCollectionDecorator)
         last = self.relation = DatasourceDecorator(last, RelationCollectionDecorator)
+        # lazy join is just before relation, to avoid relations to do useless stuff
+        last = self.lazy_joins = DatasourceDecorator(last, LazyJoinCollectionDecorator)
         last = self.late_computed = DatasourceDecorator(last, ComputedCollectionDecorator)
         last = self.late_op_emulate = DatasourceDecorator(last, OperatorsEmulateCollectionDecorator)
         last = self.late_op_equivalence = DatasourceDecorator(last, OperatorEquivalenceCollectionDecorator)

src/datasource_toolkit/forestadmin/datasource_toolkit/decorators/lazy_join/collection.py

@@ -0,0 +1,97 @@
+from typing import List, Union, cast
+
+from forestadmin.agent_toolkit.utils.context import User
+from forestadmin.datasource_toolkit.decorators.collection_decorator import CollectionDecorator
+from forestadmin.datasource_toolkit.interfaces.fields import ManyToOne, is_many_to_one
+from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf
+from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter
+from forestadmin.datasource_toolkit.interfaces.query.filter.unpaginated import Filter
+from forestadmin.datasource_toolkit.interfaces.query.projections import Projection
+from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias
+
+
+class LazyJoinCollectionDecorator(CollectionDecorator):
+    async def list(self, caller: User, filter_: PaginatedFilter, projection: Projection) -> List[RecordsDataAlias]:
+        simplified_projection = self._get_projection_without_useless_joins(projection)
+
+        refined_filter = cast(PaginatedFilter, await self._refine_filter(caller, filter_))
+        ret = await self.child_collection.list(caller, refined_filter, simplified_projection)
+
+        return self._apply_joins_on_records(projection, simplified_projection, ret)
+
+    async def _refine_filter(
+        self, caller: User, _filter: Union[Filter, PaginatedFilter, None]
+    ) -> Union[Filter, PaginatedFilter, None]:
+        if _filter is None or _filter.condition_tree is None:
+            return _filter
+
+        _filter.condition_tree = _filter.condition_tree.replace(
+            lambda leaf: (
+                ConditionTreeLeaf(
+                    self._get_fk_field_for_projection(leaf.field),
+                    leaf.operator,
+                    leaf.value,
+                )
+                if self._is_useless_join(leaf.field.split(":")[0], _filter.condition_tree.projection)
+                else leaf
+            )
+        )
+
+        return _filter
+
+    def _is_useless_join(self, relation: str, projection: Projection) -> bool:
+        relation_schema = self.schema["fields"][relation]
+        sub_projections = projection.relations[relation]
+
+        return (
+            is_many_to_one(relation_schema)
+            and len(sub_projections) == 1
+            and sub_projections[0] == relation_schema["foreign_key_target"]
+        )
+
+    def _get_fk_field_for_projection(self, projection: str) -> str:
+        relation_name = projection.split(":")[0]
+        relation_schema = cast(ManyToOne, self.schema["fields"][relation_name])
+
+        return relation_schema["foreign_key"]
+
+    def _get_projection_without_useless_joins(self, projection: Projection) -> Projection:
+        returned_projection = Projection(*projection)
+        for relation, relation_projections in projection.relations.items():
+            if self._is_useless_join(relation, projection):
+                # remove foreign key target from projection
+                returned_projection.remove(f"{relation}:{relation_projections[0]}")
+
+                # add foreign keys to projection
+                fk_field = self._get_fk_field_for_projection(relation)
+                if fk_field not in returned_projection:
+                    returned_projection.append(fk_field)
+
+        return returned_projection
+
+    def _apply_joins_on_records(
+        self, initial_projection: Projection, requested_projection: Projection, records: List[RecordsDataAlias]
+    ) -> List[RecordsDataAlias]:
+        if requested_projection == initial_projection:
+            return records
+
+        projections_to_add = Projection(*[p for p in initial_projection if p not in requested_projection])
+        projections_to_rm = Projection(*[p for p in requested_projection if p not in initial_projection])
+
+        for record in records:
+            # add to records relation:id
+            for relation, relation_projections in projections_to_add.relations.items():
+                relation_schema = self.schema["fields"][relation]
+
+                if is_many_to_one(relation_schema):
+                    record[relation] = {
+                        relation_projections[0]: record[
+                            self._get_fk_field_for_projection(f"{relation}:{relation_projections[0]}")
+                        ]
+                    }
+
+            # remove foreign keys
+            for projection in projections_to_rm:
+                del record[projection]
+
+        return records

src/datasource_toolkit/forestadmin/datasource_toolkit/interfaces/query/condition_tree/nodes/base.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import abc
 import sys
-from typing import Any, Awaitable, Callable, Dict, List, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, TypeVar, Union
 
 if sys.version_info >= (3, 9):
     import zoneinfo
@@ -13,6 +15,9 @@
 from forestadmin.datasource_toolkit.interfaces.records import RecordsDataAlias
 from typing_extensions import TypedDict
 
+if TYPE_CHECKING:
+    from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf
+
 
 class ConditionTreeException(DatasourceToolkitException):
     pass
@@ -75,7 +80,7 @@ class ConditionTreeComponent(TypedDict):
 
 
 HandlerResult = TypeVar("HandlerResult")
-HandlerAlias = Callable[[ConditionTree], HandlerResult]
+HandlerAlias = Callable[["ConditionTreeLeaf"], HandlerResult]
 ReplacerAlias = HandlerAlias[Union[ConditionTree, ConditionTreeComponent]]
 AsyncReplacerAlias = HandlerAlias[Awaitable[Union[ConditionTree, ConditionTreeComponent]]]
 CallbackAlias = HandlerAlias[None]

src/datasource_toolkit/tests/decorators/lazy_join/test_lazy_join_decorator.py

@@ -0,0 +1,227 @@
+import asyncio
+import sys
+from unittest import TestCase
+from unittest.mock import AsyncMock, patch
+
+if sys.version_info >= (3, 9):
+    import zoneinfo
+else:
+    from backports import zoneinfo
+
+from forestadmin.agent_toolkit.utils.context import User
+from forestadmin.datasource_toolkit.collections import Collection
+from forestadmin.datasource_toolkit.datasources import Datasource
+from forestadmin.datasource_toolkit.decorators.datasource_decorator import DatasourceDecorator
+from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator
+from forestadmin.datasource_toolkit.interfaces.fields import Column, FieldType, ManyToOne, OneToMany, PrimitiveType
+from forestadmin.datasource_toolkit.interfaces.query.condition_tree.nodes.leaf import ConditionTreeLeaf
+from forestadmin.datasource_toolkit.interfaces.query.filter.paginated import PaginatedFilter
+from forestadmin.datasource_toolkit.interfaces.query.projections import Projection
+
+
+class TestEmptyCollectionDecorator(TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.loop = asyncio.new_event_loop()
+        Collection.__abstractmethods__ = set()  # to instantiate abstract class
+        cls.datasource: Datasource = Datasource()
+        cls.datasource.get_collection = lambda x: cls.datasource._collections[x]
+        cls.mocked_caller = User(
+            rendering_id=1,
+            user_id=1,
+            tags={},
+            email="dummy@user.fr",
+            first_name="dummy",
+            last_name="user",
+            team="operational",
+            timezone=zoneinfo.ZoneInfo("Europe/Paris"),
+            request={"ip": "127.0.0.1"},
+        )
+
+        cls.collection_book = Collection("Book", cls.datasource)
+        cls.collection_book.add_fields(
+            {
+                "id": Column(column_type=PrimitiveType.NUMBER, is_primary_key=True, type=FieldType.COLUMN),
+                "author_id": Column(column_type=PrimitiveType.NUMBER, type=FieldType.COLUMN),
+                "author": ManyToOne(
+                    foreign_collection="Person",
+                    foreign_key="author_id",
+                    foreign_key_target="id",
+                    type=FieldType.MANY_TO_ONE,
+                ),
+                "title": Column(
+                    column_type=PrimitiveType.STRING,
+                    type=FieldType.COLUMN,
+                ),
+            }
+        )
+        cls.collection_person = Collection("Person", cls.datasource)
+        cls.collection_person.add_fields(
+            {
+                "id": Column(column_type=PrimitiveType.NUMBER, is_primary_key=True, type=FieldType.COLUMN),
+                "first_name": Column(column_type=PrimitiveType.STRING, type=FieldType.COLUMN),
+                "last_name": Column(column_type=PrimitiveType.STRING, type=FieldType.COLUMN),
+                "books": OneToMany(origin_key="author_id", origin_key_target="id", foreign_collection="Book"),
+            }
+        )
+
+        cls.datasource.add_collection(cls.collection_book)
+        cls.datasource.add_collection(cls.collection_person)
+        cls.datasource_decorator = DatasourceDecorator(cls.datasource, LazyJoinCollectionDecorator)
+        cls.decorated_book_collection = cls.datasource_decorator.get_collection("Book")
+
+    def test_should_not_join_when_projection_ask_for_target_field_only(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}],
+        ) as mock_list:
+            result = self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter({}),
+                    Projection("id", "author:id"),
+                )
+            )
+            mock_list.assert_awaited_once_with(self.mocked_caller, PaginatedFilter({}), Projection("id", "author_id"))
+
+        # should contain author object, without author_id FK
+        self.assertEqual([{"id": 1, "author": {"id": 2}}, {"id": 2, "author": {"id": 5}}], result)
+
+    def test_should_join_when_projection_ask_for_multiple_fields_in_foreign_collection(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[
+                {"id": 1, "author": {"id": 2, "first_name": "Isaac"}},
+                {"id": 2, "author": {"id": 5, "first_name": "J.K."}},
+            ],
+        ) as mock_list:
+            result = self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter({}),
+                    Projection("id", "author:id", "author:first_name"),
+                )
+            )
+            mock_list.assert_awaited_once_with(
+                self.mocked_caller, PaginatedFilter({}), Projection("id", "author:id", "author:first_name")
+            )
+
+        self.assertEqual(
+            [
+                {"id": 1, "author": {"id": 2, "first_name": "Isaac"}},
+                {"id": 2, "author": {"id": 5, "first_name": "J.K."}},
+            ],
+            result,
+        )
+
+    def test_should_not_join_when_condition_tree_is_on_foreign_key_target(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}, {"id": 3, "author_id": 5}],
+        ) as mock_list:
+            self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:id", "in", [2, 5])}),
+                    Projection("id", "author:id"),
+                )
+            )
+            mock_list.assert_awaited_once_with(
+                self.mocked_caller,
+                PaginatedFilter({"condition_tree": ConditionTreeLeaf("author_id", "in", [2, 5])}),
+                Projection("id", "author_id"),
+            )
+
+    def test_should_join_when_condition_tree_is_on_foreign_collection_fields(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[{"id": 1, "author_id": 2}, {"id": 2, "author_id": 5}, {"id": 3, "author_id": 5}],
+        ) as mock_list:
+            self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter(
+                        {"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}
+                    ),
+                    Projection("id", "author:id"),
+                )
+            )
+            mock_list.assert_awaited_once_with(
+                self.mocked_caller,
+                PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}),
+                Projection("id", "author_id"),
+            )
+
+    def test_should_disable_join_on_condition_tree_but_not_in_projection(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[
+                {"id": 1, "author": {"first_name": "Isaac"}},
+                {"id": 2, "author": {"first_name": "J.K."}},
+                {"id": 3, "author": {"first_name": "J.K."}},
+            ],
+        ) as mock_list:
+            response = self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:id", "in", [2, 5])}),
+                    Projection("id", "author:first_name"),
+                )
+            )
+            mock_list.assert_awaited_once_with(
+                self.mocked_caller,
+                PaginatedFilter({"condition_tree": ConditionTreeLeaf("author_id", "in", [2, 5])}),
+                Projection("id", "author:first_name"),
+            )
+        self.assertEqual(
+            [
+                {"id": 1, "author": {"first_name": "Isaac"}},
+                {"id": 2, "author": {"first_name": "J.K."}},
+                {"id": 3, "author": {"first_name": "J.K."}},
+            ],
+            response,
+        )
+
+    def test_should_disable_join_on_projection_but_not_in_condition_tree(self):
+        with patch.object(
+            self.collection_book,
+            "list",
+            new_callable=AsyncMock,
+            return_value=[
+                {"id": 1, "author_id": 2},
+                {"id": 2, "author_id": 5},
+                {"id": 3, "author_id": 5},
+            ],
+        ) as mock_list:
+            response = self.loop.run_until_complete(
+                self.decorated_book_collection.list(
+                    self.mocked_caller,
+                    PaginatedFilter(
+                        {"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}
+                    ),
+                    Projection("id", "author:id"),
+                )
+            )
+            mock_list.assert_awaited_once_with(
+                self.mocked_caller,
+                PaginatedFilter({"condition_tree": ConditionTreeLeaf("author:first_name", "in", ["Isaac", "J.K."])}),
+                Projection("id", "author_id"),
+            )
+        self.assertEqual(
+            [
+                {"id": 1, "author": {"id": 2}},
+                {"id": 2, "author": {"id": 5}},
+                {"id": 3, "author": {"id": 5}},
+            ],
+            response,
+        )

src/datasource_toolkit/tests/decorators/test_decorator_stack.py

@@ -9,10 +9,12 @@
 from forestadmin.datasource_toolkit.decorators.decorator_stack import DecoratorStack
 from forestadmin.datasource_toolkit.decorators.empty.collection import EmptyCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.hook.collections import CollectionHookDecorator
+from forestadmin.datasource_toolkit.decorators.lazy_join.collection import LazyJoinCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.operators_emulate.collections import OperatorsEmulateCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.operators_equivalence.collections import (
     OperatorEquivalenceCollectionDecorator,
 )
+from forestadmin.datasource_toolkit.decorators.override.collection import OverrideCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.relation.collections import RelationCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.rename_field.collections import RenameFieldCollectionDecorator
 from forestadmin.datasource_toolkit.decorators.schema.collection import SchemaCollectionDecorator
@@ -68,11 +70,13 @@ def test_creation_instantiate_all_decorator_with_datasource_decorator(self):
             DecoratorStack(self.datasource)
 
             call_list = [
+                call(self.datasource, OverrideCollectionDecorator),
                 call(self.datasource, EmptyCollectionDecorator),
                 call(self.datasource, ComputedCollectionDecorator),
                 call(self.datasource, OperatorsEmulateCollectionDecorator),
                 call(self.datasource, OperatorEquivalenceCollectionDecorator),
                 call(self.datasource, RelationCollectionDecorator),
+                call(self.datasource, LazyJoinCollectionDecorator),
                 call(self.datasource, ComputedCollectionDecorator),
                 call(self.datasource, OperatorsEmulateCollectionDecorator),
                 call(self.datasource, OperatorEquivalenceCollectionDecorator),