diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7274fdbbdd..a5868c756a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 diff --git a/pants-plugins/pack_metadata/register.py b/pants-plugins/pack_metadata/register.py index 34a6aa85cc..36c11079d9 100644 --- a/pants-plugins/pack_metadata/register.py +++ b/pants-plugins/pack_metadata/register.py @@ -11,13 +11,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pack_metadata import tailor -from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule +from pack_metadata import tailor, target_types_rules +from pack_metadata.target_types import ( + PackMetadata, + PackMetadataInGitSubmodule, + PacksGlob, +) def rules(): - return tailor.rules() + return [ + *tailor.rules(), + *target_types_rules.rules(), + ] def target_types(): - return [PackMetadata, PackMetadataInGitSubmodule] + return [ + PackMetadata, + PackMetadataInGitSubmodule, + PacksGlob, + ] diff --git a/pants-plugins/pack_metadata/target_types.py b/pants-plugins/pack_metadata/target_types.py index cfadbaab01..0b2d41e2c2 100644 --- a/pants-plugins/pack_metadata/target_types.py +++ b/pants-plugins/pack_metadata/target_types.py @@ -17,6 +17,7 @@ from pants.core.target_types import ( ResourcesGeneratingSourcesField, ResourcesGeneratorTarget, + GenericTarget, ) @@ -77,3 +78,18 @@ class PackMetadataInGitSubmodule(PackMetadata): "has unmatched globs. It prints instructions on how to checkout git " "submodules." ) + + +class PacksGlobDependencies(Dependencies): + pass + + +class PacksGlob(GenericTarget): + alias = "packs_glob" + core_fields = (*COMMON_TARGET_FIELDS, PacksGlobDependencies) + help = ( + "Packs glob.\n\n" + "Avoid using this target. It gets automatic dependencies on all " + "subdirectories (packs) except those listed with ! in dependencies. " + "This is unfortunately needed by tests that use a glob to load pack fixtures." + ) diff --git a/pants-plugins/pack_metadata/target_types_rules.py b/pants-plugins/pack_metadata/target_types_rules.py new file mode 100644 index 0000000000..d67555849d --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_rules.py @@ -0,0 +1,83 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from dataclasses import dataclass + +from pants.engine.addresses import Address +from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths +from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule +from pants.engine.target import ( + DependenciesRequest, + ExplicitlyProvidedDependencies, + FieldSet, + InferDependenciesRequest, + InferredDependencies, +) +from pants.util.logging import LogLevel + +from pack_metadata.target_types import PacksGlobDependencies + + +@dataclass(frozen=True) +class PacksGlobInferenceFieldSet(FieldSet): + required_fields = (PacksGlobDependencies,) + + dependencies: PacksGlobDependencies + + +class InferPacksGlobDependencies(InferDependenciesRequest): + infer_from = PacksGlobInferenceFieldSet + + +@rule( + desc="Inferring packs glob dependencies", + level=LogLevel.DEBUG, +) +async def infer_packs_globs_dependencies( + request: InferPacksGlobDependencies, +) -> InferredDependencies: + address = request.field_set.address + + pack_build_paths, explicitly_provided_deps = await MultiGet( + Get( + Paths, + PathGlobs( + [os.path.join(address.spec_path, "*", "BUILD")], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"{address}'s packs glob", + ), + ), + Get( + ExplicitlyProvidedDependencies, + DependenciesRequest(request.field_set.dependencies), + ), + ) + + implicit_packs_deps = { + Address(os.path.dirname(path)) for path in pack_build_paths.files + } + + inferred_packs_deps = ( + implicit_packs_deps + - explicitly_provided_deps.ignores # FrozenOrderedSet[Address] + - explicitly_provided_deps.includes # FrozenOrderedSet[Address] + ) + return InferredDependencies(inferred_packs_deps) + + +def rules(): + return [ + *collect_rules(), + UnionRule(InferDependenciesRequest, InferPacksGlobDependencies), + ] diff --git a/pants-plugins/pack_metadata/target_types_rules_test.py b/pants-plugins/pack_metadata/target_types_rules_test.py new file mode 100644 index 0000000000..28780e0025 --- /dev/null +++ b/pants-plugins/pack_metadata/target_types_rules_test.py @@ -0,0 +1,125 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from textwrap import dedent + +from pants.backend.python.target_types import ( + PythonSourceTarget, + PythonSourcesGeneratorTarget, +) +from pants.backend.python.target_types_rules import rules as python_target_types_rules +from pants.engine.addresses import Address +from pants.engine.target import InferredDependencies +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .target_types_rules import ( + InferPacksGlobDependencies, + PacksGlobInferenceFieldSet, + rules as pack_metadata_target_types_rules, +) +from .target_types import PacksGlob + + +def test_infer_packs_globs_dependencies() -> None: + rule_runner = RuleRunner( + rules=[ + *python_target_types_rules(), + *pack_metadata_target_types_rules(), + QueryRule(InferredDependencies, (InferPacksGlobDependencies,)), + ], + target_types=[ + PythonSourceTarget, + PythonSourcesGeneratorTarget, + PacksGlob, + ], + ) + rule_runner.write_files( + { + "packs/BUILD": dedent( + """\ + python_sources( + name="git_submodule", + sources=["./git_submodule/*.py"], + ) + + packs_glob( + name="all_packs_glob", + dependencies=[ + "!./configs", # explicit ignore + "./a", # explicit include + ], + ) + """ + ), + "packs/a/BUILD": "python_sources()", + "packs/a/__init__.py": "", + "packs/a/fixture.py": "", + "packs/b/BUILD": dedent( + """\ + python_sources( + dependencies=["packs/configs/b.yaml"], + ) + """ + ), + "packs/b/__init__.py": "", + "packs/b/fixture.py": "", + "packs/c/BUILD": "python_sources()", + "packs/c/__init__.py": "", + "packs/c/fixture.py": "", + "packs/d/BUILD": "python_sources()", + "packs/d/__init__.py": "", + "packs/d/fixture.py": "", + # imitate a pack in a git submodule (should NOT have a BUILD file) + "packs/git_submodule/__init__.py": "", + "packs/git_submodule/fixture.py": "", + "packs/configs/BUILD": dedent( + """\ + resources( + sources=["*.yaml"], + ) + """ + ), + "packs/configs/b.yaml": dedent( + """\ + --- + # pack config for pack b + """ + ), + } + ) + + def run_dep_inference(address: Address) -> InferredDependencies: + args = [ + "--source-root-patterns=/packs", + ] + rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"}) + target = rule_runner.get_target(address) + return rule_runner.request( + InferredDependencies, + [InferPacksGlobDependencies(PacksGlobInferenceFieldSet.create(target))], + ) + + assert run_dep_inference( + Address("packs", target_name="all_packs_glob") + ) == InferredDependencies( + [ + # should not have packs/a (explicit dep does not need to be inferred) + # should not have packs/configs (explicitly ignored) + # should not have packs/git_submodule (no BUILD file = no targets to add) + Address("packs/b"), + Address("packs/c"), + Address("packs/d"), + ], + ) diff --git a/st2api/tests/unit/controllers/v1/test_pack_config_schema.py b/st2api/tests/unit/controllers/v1/test_pack_config_schema.py index a38c278f07..9ca48aea98 100644 --- a/st2api/tests/unit/controllers/v1/test_pack_config_schema.py +++ b/st2api/tests/unit/controllers/v1/test_pack_config_schema.py @@ -17,12 +17,12 @@ from st2tests.api import FunctionalTest -from st2tests.fixturesloader import get_fixtures_packs_base_path +# import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH __all__ = ["PackConfigSchemasControllerTestCase"] -PACKS_PATH = get_fixtures_packs_base_path() -CONFIG_SCHEMA_COUNT = len(glob.glob("%s/*/config.schema.yaml" % (PACKS_PATH))) +CONFIG_SCHEMA_COUNT = len(glob.glob(f"{PACKS_PATH}/*/config.schema.yaml")) assert CONFIG_SCHEMA_COUNT > 1 diff --git a/st2api/tests/unit/controllers/v1/test_pack_configs.py b/st2api/tests/unit/controllers/v1/test_pack_configs.py index 5a87719eaa..91fd712826 100644 --- a/st2api/tests/unit/controllers/v1/test_pack_configs.py +++ b/st2api/tests/unit/controllers/v1/test_pack_configs.py @@ -19,12 +19,13 @@ from st2tests.api import FunctionalTest from st2api.controllers.v1.pack_configs import PackConfigsController -from st2tests.fixturesloader import get_fixtures_packs_base_path + +# import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH __all__ = ["PackConfigsControllerTestCase"] -PACKS_PATH = get_fixtures_packs_base_path() -CONFIGS_COUNT = len(glob.glob("%s/configs/*.yaml" % (PACKS_PATH))) +CONFIGS_COUNT = len(glob.glob(f"{PACKS_PATH}/configs/*.yaml")) assert CONFIGS_COUNT > 1 diff --git a/st2common/tests/integration/test_register_content_script.py b/st2common/tests/integration/test_register_content_script.py index d416ae9af1..b0288a910e 100644 --- a/st2common/tests/integration/test_register_content_script.py +++ b/st2common/tests/integration/test_register_content_script.py @@ -22,9 +22,9 @@ from st2tests.base import IntegrationTestCase from st2common.util.shell import run_command from st2tests import config as test_config -from st2tests.fixturesloader import get_fixtures_packs_base_path # import this so that pants can infer dependencies for the glob below +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_PATH as DUMMY_PACK_1_PATH from st2tests.fixtures.packs.dummy_pack_4.fixture import PACK_PATH as DUMMY_PACK_4_PATH from st2tests.fixtures.packs.runners.fixture import FIXTURE_PATH as RUNNER_DIRS @@ -38,8 +38,7 @@ BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, "--config-file=conf/st2.tests.conf", "-v"] BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ["--register-actions"] -PACKS_PATH = get_fixtures_packs_base_path() -PACKS_COUNT = len(glob.glob("%s/*/pack.yaml" % (PACKS_PATH))) +PACKS_COUNT = len(glob.glob(f"{PACKS_PATH}/*/pack.yaml")) assert PACKS_COUNT >= 2 diff --git a/st2common/tests/unit/test_policies_registrar.py b/st2common/tests/unit/test_policies_registrar.py index 231c477446..3a435ab0f3 100644 --- a/st2common/tests/unit/test_policies_registrar.py +++ b/st2common/tests/unit/test_policies_registrar.py @@ -27,7 +27,7 @@ from st2common.persistence.policy import Policy from st2common.persistence.policy import PolicyType from st2tests.base import CleanDbTestCase -from st2tests.fixturesloader import get_fixtures_packs_base_path +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import ( PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, @@ -63,8 +63,7 @@ def test_register_all_policies(self): policies_dbs = Policy.get_all() self.assertEqual(len(policies_dbs), 0) - packs_base_path = get_fixtures_packs_base_path() - count = policies_registrar.register_policies(packs_base_paths=[packs_base_path]) + count = policies_registrar.register_policies(packs_base_paths=[PACKS_PATH]) # Verify PolicyDB objects have been created policies_dbs = Policy.get_all() diff --git a/st2common/tests/unit/test_triggers_registrar.py b/st2common/tests/unit/test_triggers_registrar.py index ef06ec5d8a..9ad3526b18 100644 --- a/st2common/tests/unit/test_triggers_registrar.py +++ b/st2common/tests/unit/test_triggers_registrar.py @@ -19,7 +19,7 @@ from st2common.persistence.trigger import Trigger from st2common.persistence.trigger import TriggerType from st2tests.base import CleanDbTestCase -from st2tests.fixturesloader import get_fixtures_packs_base_path +from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH from st2tests.fixtures.packs.dummy_pack_1.fixture import ( PACK_NAME as DUMMY_PACK_1, PACK_PATH as DUMMY_PACK_1_PATH, @@ -33,8 +33,7 @@ def test_register_all_triggers(self): trigger_type_dbs = TriggerType.get_all() self.assertEqual(len(trigger_type_dbs), 0) - packs_base_path = get_fixtures_packs_base_path() - count = triggers_registrar.register_triggers(packs_base_paths=[packs_base_path]) + count = triggers_registrar.register_triggers(packs_base_paths=[PACKS_PATH]) self.assertEqual(count, 2) # Verify TriggerTypeDB and corresponding TriggerDB objects have been created diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index c53022a3ca..bbcae502dd 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -34,3 +34,19 @@ python_sources( "test_content_version/**/*.py", ], ) + +packs_glob( + name="all_packs", + dependencies=[ + # core is a symlink instead of a dir + "./core", + # use :test_content_version instead because of the git submodule + ":test_content_version", + "!./test_content_version_fixture", + # these are not packs + "!./configs", + "!./executions", + "!./runners", + "!./all_packs_glob", # the fixture that pulls in this target + ], +) diff --git a/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD b/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD new file mode 100644 index 0000000000..13726c3e4b --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD @@ -0,0 +1,3 @@ +python_sources( + dependencies=["st2tests/st2tests/fixtures/packs:all_packs"], +) diff --git a/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py new file mode 100644 index 0000000000..d41e2dad06 --- /dev/null +++ b/st2tests/st2tests/fixtures/packs/all_packs_glob/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from st2tests.fixturesloader import get_fixtures_packs_base_path + +PACKS_PATH = get_fixtures_packs_base_path() diff --git a/st2tests/st2tests/fixtures/packs_1/BUILD b/st2tests/st2tests/fixtures/packs_1/BUILD new file mode 100644 index 0000000000..251e480e8a --- /dev/null +++ b/st2tests/st2tests/fixtures/packs_1/BUILD @@ -0,0 +1,3 @@ +packs_glob( + name="all_packs", +) diff --git a/st2tests/st2tests/fixtures/packs_invalid/BUILD b/st2tests/st2tests/fixtures/packs_invalid/BUILD new file mode 100644 index 0000000000..251e480e8a --- /dev/null +++ b/st2tests/st2tests/fixtures/packs_invalid/BUILD @@ -0,0 +1,3 @@ +packs_glob( + name="all_packs", +)