diff --git a/MANIFEST.in b/MANIFEST.in index 1ed0f12..5731df1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/* +include Makefile CHANGES.rst LICENSE AUTHORS juniper/artifacts/* requirements/* diff --git a/Makefile b/Makefile index 1798a39..3680bee 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ install-dev: pip install -q -e .[venv3] pip install -r requirements/dev.txt +deps: + pip-compile -U --allow-unsafe -o requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in + pip-compile -U --allow-unsafe -o requirements/requirements.txt requirements/requirements.in + test: clean-pyc python -m pytest --cov . @@ -36,14 +40,12 @@ gh-pages: release: rm -rf ./dist - python3 -m pip install --upgrade build python3 -m build twine check dist/* twine upload dist/* test-release: rm -rf ./dist - python3 -m pip install --upgrade build python3 -m build twine check dist/* python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* @@ -51,4 +53,4 @@ test-release: clean-pyc: find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + - find . -name '*~' -exec rm -f {} + \ No newline at end of file + find . -name '*~' -exec rm -f {} + diff --git a/README.rst b/README.rst index 930ad31..366345c 100644 --- a/README.rst +++ b/README.rst @@ -257,6 +257,7 @@ This list defines the entire scope of Juniper. Nothing more, nothing else. * Specify docker image to package lamdba functions using different python runtimes * Define pip command line arguments using a pip.conf file * Packaging of lambda layers +* Support for building arm64 layers and lambdas (Graviton2) via specifying the platform Contributing ************ diff --git a/juniper/__init__.py b/juniper/__init__.py index a6dfad5..65a64c8 100644 --- a/juniper/__init__.py +++ b/juniper/__init__.py @@ -14,4 +14,4 @@ limitations under the License. """ -__version__ = '0.5.5' +__version__ = '0.8.1' diff --git a/juniper/actions.py b/juniper/actions.py index 70b79f3..ea85449 100644 --- a/juniper/actions.py +++ b/juniper/actions.py @@ -18,6 +18,7 @@ import shutil import subprocess from jinja2 import Template +import functools from juniper.constants import DEFAULT_OUT_DIR, DEFAULT_DOCKER_IMAGE from juniper.io import (get_artifact, write_tmp_file, get_artifact_path) @@ -36,25 +37,28 @@ def build_artifacts(logger, manifest): """ compose_fn = build_compose(logger, manifest) - logger.debug(f'docker-compose -f {compose_fn} --project-directory . run sample-lambda bash') + logger.debug(f'docker compose -f {compose_fn} --project-directory . run sample-lambda bash') try: # Must copy the bin directory to the client's folder structure. This directory # will be promtly cleaned up after the artifacts are built. os.makedirs('./.juni/bin', exist_ok=True) - shutil.copy(get_artifact_path('package.sh'), './.juni/bin/') - shutil.copy(get_artifact_path('build_layer.sh'), './.juni/bin/') + with get_artifact_path('package.sh') as path: + shutil.copy(path, './.juni/bin/') + with get_artifact_path('build_layer.sh') as path: + shutil.copy(path, './.juni/bin/') # Use docker as a way to pip install dependencies, and copy the business logic # specified in the function definitions. - subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'down']) - subprocess.run(["docker-compose", "-f", compose_fn, '--project-directory', '.', 'up']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down', '--remove-orphans']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'up']) + subprocess.run(["docker", "compose", "-f", compose_fn, '--project-directory', '.', 'down']) finally: shutil.rmtree('./.juni', ignore_errors=True) def build_compose(logger, manifest): """ - Builds a docker-compose file with the lambda functions defined in the manifest. + Builds a docker compose file with the lambda functions defined in the manifest. The definition of the lambda functions includes the name of the function as well as the set of dependencies to include in the packaging. @@ -63,14 +67,14 @@ def build_compose(logger, manifest): """ compose = _get_compose_template(manifest) - # Returns the name of the temp file that has the docker-compose definition. + # Returns the name of the temp file that has the docker compose definition. return write_tmp_file(compose) def _get_compose_template(manifest): """ Build the service entry for each one of the functions in the given context. - Each docker-compose entry will depend on the same image and it's just a static + Each docker compose entry will depend on the same image and it's just a static definition that gets built from a template. The template is in the artifacts folder. """ @@ -81,6 +85,7 @@ def build_section(label): { 'name': name, 'image': _get_docker_image(manifest, sls_section), + 'platform': _get_platform(manifest, sls_section), 'volumes': _get_volumes(manifest, sls_section) } for name, sls_section in manifest.get(label, {}).items() @@ -132,21 +137,27 @@ def get_vol(include): return volumes -def _get_docker_image(manifest, sls_function): +def _get_attr(key, default, manifest, sls_function): """ - Get the docker image that will be used to package a given function. Precedence - is as follows: function level override, global image override, default. + Get an attribute from the manifest, looking under the function first, + then global:, and finally using a default. - :params manfiest: The juniper manifest file. + + :params key: The key to retrieve + :params default: The value to return if your key is not defined anywhere + :params manifest: The juniper manifest file. :params sls_function: The serverless function definition. """ + function_value = sls_function.get(key) + if function_value: + return function_value + + global_value = manifest.get('global', {}).get(key) + if global_value: + return global_value - function_image = sls_function.get('image') - if function_image: - return function_image + return default - global_image = manifest.get('global', {}).get('image') - if global_image: - return global_image - return DEFAULT_DOCKER_IMAGE +_get_docker_image = functools.partial(_get_attr, "image", DEFAULT_DOCKER_IMAGE) +_get_platform = functools.partial(_get_attr, "platform", None) diff --git a/juniper/artifacts/compose-template.yml b/juniper/artifacts/compose-template.yml index 13af3c9..33c2d7f 100644 --- a/juniper/artifacts/compose-template.yml +++ b/juniper/artifacts/compose-template.yml @@ -11,6 +11,9 @@ services: - {{volume}} {% endfor %} command: sh /var/task/bin/package.sh {{lambda_fn.name}} +{%- if lambda_fn.platform %} + platform: {{ lambda_fn.platform }} +{%- endif %} {% endfor %} {% for layer in layers %} {{layer.name}}-layer: @@ -22,4 +25,7 @@ services: - {{volume}} {% endfor %} command: sh /var/task/bin/build_layer.sh {{layer.name}} +{%- if layer.platform %} + platform: {{ layer.platform }} +{%- endif %} {% endfor %} diff --git a/juniper/io.py b/juniper/io.py index a67bfb0..e327fab 100644 --- a/juniper/io.py +++ b/juniper/io.py @@ -18,7 +18,9 @@ import os import yaml import tempfile -import pkg_resources +import importlib.resources +import contextlib +import pathlib def reader(file_name): @@ -39,12 +41,14 @@ def write_tmp_file(content): def get_artifact(template_name): fn_template_path = get_artifact_path(template_name) - with open(fn_template_path, 'r') as f: - return f.read() + with fn_template_path as path: + with path.open("r") as fd: + return fd.read() -def get_artifact_path(artifact_name): +def get_artifact_path(artifact_name) -> contextlib.AbstractContextManager[pathlib.Path]: """ Reads an artifact out of the rio-tools project. """ - return pkg_resources.resource_filename(__name__, os.path.join('artifacts', artifact_name)) + files = importlib.resources.files("juniper") + return importlib.resources.as_file(files.joinpath(os.path.join('artifacts', artifact_name))) diff --git a/requirements/dev.txt b/requirements/dev.txt index a1269b9..68bb177 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,15 +1,103 @@ -Click==7.0 -click-log==0.3.2 -ipdb==0.11 -coverage==4.5.1 -coverage-badge==0.2.0 -docker>=5 -docker-compose>=1.29 -pytest==3.5.1 -pytest-mock==1.10.0 -pytest-pythonpath==0.7.2 -pytest-cov==2.5.1 -flake8==3.5.0 -PyYAML==5.3 -Sphinx==1.7.6 -phix==0.6.1 \ No newline at end of file +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=requirements/dev.txt requirements/requirements-dev.in requirements/requirements.in +# +bleach==6.0.0 + # via readme-renderer +build==0.10.0 + # via + # -r requirements/requirements-dev.in + # pip-tools +certifi==2023.7.22 + # via requests +charset-normalizer==3.2.0 + # via requests +click==8.1.7 + # via + # -r requirements/requirements.in + # click-log + # pip-tools +click-log==0.4.0 + # via -r requirements/requirements.in +docutils==0.20.1 + # via readme-renderer +idna==3.4 + # via requests +importlib-metadata==6.8.0 + # via + # keyring + # twine +iniconfig==2.0.0 + # via pytest +jaraco-classes==3.3.0 + # via keyring +jinja2==3.1.2 + # via -r requirements/requirements.in +keyring==24.2.0 + # via twine +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.3 + # via jinja2 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.1.0 + # via jaraco-classes +packaging==23.1 + # via + # build + # pytest +pip-tools==7.3.0 + # via -r requirements/requirements-dev.in +pkginfo==1.9.6 + # via twine +pluggy==1.2.0 + # via pytest +pygments==2.16.1 + # via + # readme-renderer + # rich +pyproject-hooks==1.0.0 + # via build +pytest==7.4.0 + # via + # -r requirements/requirements-dev.in + # pytest-mock +pytest-mock==3.11.1 + # via -r requirements/requirements-dev.in +pyyaml==6.0.1 + # via -r requirements/requirements.in +readme-renderer==41.0 + # via twine +requests==2.31.0 + # via + # requests-toolbelt + # twine +requests-toolbelt==1.0.0 + # via twine +rfc3986==2.0.0 + # via twine +rich==13.5.2 + # via twine +six==1.16.0 + # via bleach +twine==4.0.2 + # via -r requirements/requirements-dev.in +urllib3==2.0.4 + # via + # requests + # twine +webencodings==0.5.1 + # via bleach +wheel==0.41.1 + # via pip-tools +zipp==3.16.2 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +pip==23.2.1 + # via pip-tools +setuptools==68.1.2 + # via pip-tools diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in new file mode 100644 index 0000000..252d994 --- /dev/null +++ b/requirements/requirements-dev.in @@ -0,0 +1,5 @@ +pytest==7.4.0 +pip-tools +pytest-mock +twine +build diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 0000000..c17ee0f --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,4 @@ +click>=8.0 +click-log +PyYAML>=6.0 +Jinja2>=3.0 diff --git a/requirements/requirements.txt b/requirements/requirements.txt new file mode 100644 index 0000000..932f0ee --- /dev/null +++ b/requirements/requirements.txt @@ -0,0 +1,18 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --output-file=requirements/requirements.txt requirements/requirements.in +# +click==8.1.7 + # via + # -r requirements/requirements.in + # click-log +click-log==0.4.0 + # via -r requirements/requirements.in +jinja2==3.1.2 + # via -r requirements/requirements.in +markupsafe==2.1.3 + # via jinja2 +pyyaml==6.0.1 + # via -r requirements/requirements.in diff --git a/setup.py b/setup.py index f5e5464..5b1cc1b 100644 --- a/setup.py +++ b/setup.py @@ -12,44 +12,38 @@ with io.open('juniper/__init__.py', 'rt', encoding='utf8') as f: version = re.search(r'__version__ = \'(.*?)\'', f.read()).group(1) +with open('requirements/requirements.in', 'r') as fd: + install_requires=fd.readlines() + setup( - name="juniper", + name="juniper-aidentified", version=version, - author='EAB Tech', - author_email='eabtech@eab.com', - description="Tool to streamline the build of python lambda functions.", + author='Aidentified LLC', + author_email='dgilman@aidentified.com', + description="Tool to streamline the build of python lambda functions. (fork of juniper)", long_description=readme, + long_description_content_type="text/x-rst", project_urls=OrderedDict(( - ('Documentation', 'https://eabglobal.github.io/juniper/'), - ('Code', 'https://github.com/eabglobal/juniper'), - ('Issue tracker', 'https://github.com/eabglobal/juniper/issues'), + ('Code', 'https://github.com/dgilmanAIDENTIFIED/juniper'), )), license='Apache Software License', - packages=find_packages(), + packages=find_packages(exclude=("tests",)), include_package_data=True, entry_points={ 'console_scripts': ['juni=juniper.cli:main'], }, - python_requires='>=3.6', + python_requires='>=3.9', test_suite="tests", - install_requires=[ - 'click>=5.1', - 'click-log', - 'PyYAML >= 4.3, <= 5.3', - 'Jinja2>=2.10', - 'docker[ssh] >= 5', - 'docker-compose >= 1.29' - ], - setup_requires=["pytest-runner"], - tests_require=["pytest"], + install_requires=install_requires, classifiers=[ 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Software Development :: Build Tools', ]) diff --git a/tests/expectations/custom-docker-platforms.yml b/tests/expectations/custom-docker-platforms.yml new file mode 100644 index 0000000..8d8e9e2 --- /dev/null +++ b/tests/expectations/custom-docker-platforms.yml @@ -0,0 +1,47 @@ +version: '3.6' + +services: + + default-platform-lambda: + image: python:3.8 + environment: + - AWS_DEFAULT_REGION=us-east-1 + platform: linux/amd64 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./src/edge:/var/task/common/edge + command: sh /var/task/bin/package.sh default-platform + + override-platform-lambda: + image: python:3.6-alpine + platform: linux/arm64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./src/worker/sequential_worker:/var/task/common/sequential_worker + command: sh /var/task/bin/package.sh override-platform + + default-platform-layer-layer: + image: python:3.6-alpine + platform: linux/amd64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./requirements/default.txt:/var/task/common/requirements.txt + command: sh /var/task/bin/build_layer.sh default-platform-layer + + override-platform-layer-layer: + image: python:3.6-alpine + platform: linux/arm64 + environment: + - AWS_DEFAULT_REGION=us-east-1 + volumes: + - ./dist:/var/task/dist + - ./.juni/bin:/var/task/bin + - ./requirements/override.txt:/var/task/common/requirements.txt + command: sh /var/task/bin/build_layer.sh override-platform-layer diff --git a/tests/manifests/custom-docker-platforms.yml b/tests/manifests/custom-docker-platforms.yml new file mode 100644 index 0000000..16ab124 --- /dev/null +++ b/tests/manifests/custom-docker-platforms.yml @@ -0,0 +1,24 @@ +global: + image: python:3.6-alpine + include: + - ./src/libs/ + - ./src/common/ + platform: linux/amd64 + +functions: + default-platform: + image: python:3.8 + include: + - ./src/edge/ + + override-platform: + include: + - ./src/worker/sequential_worker + platform: linux/arm64 + +layers: + default-platform-layer: + requirements: ./requirements/default.txt + override-platform-layer: + requirements: ./requirements/override.txt + platform: linux/arm64 diff --git a/tests/test_actions.py b/tests/test_actions.py index 2d7c5ae..29e6537 100644 --- a/tests/test_actions.py +++ b/tests/test_actions.py @@ -17,8 +17,8 @@ import yaml from juniper import actions -from unittest.mock import MagicMock, call -from juniper.io import reader, get_artifact_path +from unittest.mock import MagicMock +from juniper.io import reader logger = MagicMock() @@ -26,7 +26,7 @@ def test_build_compose_writes_compose_definition_to_tmp_file(mocker): """ - The docker-compose file created, is written to a tmp file. Make sure that + The docker compose file created, is written to a tmp file. Make sure that the file is writen and validate that the contents of the file match the expected result. """ @@ -45,9 +45,9 @@ def test_build_compose_writes_compose_definition_to_tmp_file(mocker): def test_build_artifacts_invokes_docker_commands(mocker): """ - Validate that the docker-compose commands are executed with the valid paramters. - Since the docker-compose file was dynamically generated, we must pass the full - path of that file to docker-compose command. Also, set the context of the execution + Validate that the docker compose commands are executed with the valid paramters. + Since the docker compose file was dynamically generated, we must pass the full + path of that file to docker compose command. Also, set the context of the execution to the current path. """ @@ -55,14 +55,15 @@ def test_build_artifacts_invokes_docker_commands(mocker): mock_builder = mocker.patch('juniper.actions.build_compose', return_value=tmp_filename) # Mocking the dependencies of this action. These three high level packages are - # needed to invoke docker-compose in the right context! + # needed to invoke docker compose in the right context! mocker.patch('juniper.actions.os') mocker.patch('juniper.actions.shutil') mock_subprocess_run = mocker.patch('juniper.actions.subprocess.run') compose_cmd_calls = [ - mocker.call(["docker-compose", "-f", tmp_filename, '--project-directory', '.', 'down']), - mocker.call(["docker-compose", "-f", tmp_filename, '--project-directory', '.', 'up']) + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'down', "--remove-orphans"]), + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'up']), + mocker.call(["docker", "compose", "-f", tmp_filename, '--project-directory', '.', 'down']), ] processor_ctx = reader('./tests/manifests/processor-test.yml') @@ -74,7 +75,7 @@ def test_build_artifacts_invokes_docker_commands(mocker): def test_build_artifacts_copies_scriopts(mocker): """ - Since the docker-compose command will be executed from within the context + Since the docker compose command will be executed from within the context of where the lambda functions live. We need to make sure that the `package.sh` lives in the right context. @@ -86,7 +87,7 @@ def test_build_artifacts_copies_scriopts(mocker): mock_builder = mocker.patch('juniper.actions.build_compose', return_value=tmp_filename) # Mocking the dependencies of this action. These three high level packages are - # needed to invoke docker-compose in the right context! + # needed to invoke docker compose in the right context! mock_os = mocker.patch('juniper.actions.os') mock_shutil = mocker.patch('juniper.actions.shutil') mocker.patch('juniper.actions.subprocess.run') @@ -97,10 +98,14 @@ def test_build_artifacts_copies_scriopts(mocker): # Validate that this three step process is correctly executed. mock_os.makedirs.assert_called_with('./.juni/bin', exist_ok=True) - mock_shutil.copy.assert_has_calls([ - call(get_artifact_path('package.sh'), './.juni/bin/'), - call(get_artifact_path('build_layer.sh'), './.juni/bin/'), - ]) + assert len(mock_shutil.copy.call_args_list) == 2 + assert len(mock_shutil.copy.call_args_list[0][0]) == 2 + + assert mock_shutil.copy.call_args_list[0][0][0].name == "package.sh" + assert mock_shutil.copy.call_args_list[0][0][1] == "./.juni/bin/" + + assert mock_shutil.copy.call_args_list[1][0][0].name == "build_layer.sh" + assert mock_shutil.copy.call_args_list[1][0][1] == "./.juni/bin/" mock_shutil.rmtree.assert_called_with('./.juni', ignore_errors=True) mock_builder.assert_called_once() @@ -213,7 +218,7 @@ def test_build_compose_section_supports_layers(): def test_build_compose_supports_layers(mocker): """ Validate that given a manifest that has both functions and layers builds the - correct docker-compose template. + correct docker compose template. """ tmp_filename = '/var/folders/xw/yk2rrhks1w72y0zr_7t7b851qlt8b3/T/tmp52bd77s3' diff --git a/tests/test_build_with_overrides.py b/tests/test_build_with_overrides.py index 5a4063d..39a5ffa 100644 --- a/tests/test_build_with_overrides.py +++ b/tests/test_build_with_overrides.py @@ -89,6 +89,14 @@ def test_volumes_includes_pipconf(): assert './path/pip.conf:/etc/pip.conf' in volumes +def test_platform_default_override(): + docker_ctx = reader('./tests/manifests/custom-docker-platforms.yml') + result = actions._get_compose_template(docker_ctx) + + expected = read_file('./tests/expectations/custom-docker-platforms.yml') + assert yaml.safe_load(result) == yaml.safe_load(expected) + + def read_file(file_name): with open(file_name, 'r') as f: return f.read()