diff --git a/docs/recipe-format.md b/docs/recipe-format.md index daeb0de..032534e 100644 --- a/docs/recipe-format.md +++ b/docs/recipe-format.md @@ -209,6 +209,29 @@ For example, `build:autotools` and `libvncserver>=0.9.13` are valid dependency s **Dependencies declared in the `makedepends` field are only satisfied during the build process, not at install time** — see the [`installdepends`](#installdepends-field) below for declaring install-time dependencies. +#### `preparedepends` field + + + + + + + + + +
Required?No, defaults to () +
TypeArray of dependency specifications (strings)
+ +The list of Debian, Toltec or Entware packages that are needed to prepare this package. +Dependency specifications have the following format: `[host:|build:]package-name`. +For example, `build:autotools` and `libvncserver>=0.9.13` are valid dependency specifications. + +*Build-type dependencies* (prefixed with `build:`) are packages from Debian to install in the container’s root system before the recipe’s build script is executed. + +*Host-type dependencies* (prefixed with `host:`) are packages from Toltec or Entware to install in the container’s `$SYSROOT` before the recipe’s build script is executed. The packages are offline-installed (i.e., none of their [install scripts](#install-section) are executed). + +**Dependencies declared in the `preparedepends` field are only satisfied during the prepare process, not at install time** — see the [`installdepends`](#installdepends-field) below for declaring install-time dependencies. + #### `pkgnames` field diff --git a/pyproject.toml b/pyproject.toml index e2cb85d..ff49e98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "toltecmk" -version = "0.4.0" +version = "0.5.0" authors = [ { name="Mattéo Delabre", email="git.matteo@delab.re" }, { name="Eeems", email="eeems@eeems.email" }, diff --git a/tests/fixtures/rmkit/package b/tests/fixtures/rmkit/package index 50f297b..bfe5998 100644 --- a/tests/fixtures/rmkit/package +++ b/tests/fixtures/rmkit/package @@ -24,8 +24,10 @@ sha256sums=( SKIP SKIP ) +preparedepends=(build:rsync) prepare() { + rsync --help > /dev/null patch -p1 -d"$srcdir" < "$srcdir"/patch-remux-duplicate-xochitl.diff patch -p1 -d"$srcdir" < "$srcdir"/patch-remux-start-xochitl.diff rm "$srcdir"/*.diff diff --git a/tests/recipe_parsers/test_parser.py b/tests/recipe_parsers/test_parser.py index 205eb14..d56c6a6 100644 --- a/tests/recipe_parsers/test_parser.py +++ b/tests/recipe_parsers/test_parser.py @@ -80,6 +80,7 @@ def test_basic_recipe(self) -> None: }, ) self.assertEqual(recipe.makedepends, set()) + self.assertEqual(recipe.preparedepends, set()) self.assertEqual(recipe.maintainer, "None ") self.assertEqual(recipe.image, "base:v2.1") self.assertEqual(recipe.arch, "rmall") @@ -94,6 +95,7 @@ def test_basic_recipe(self) -> None: declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -136,6 +138,7 @@ def test_basic_recipe(self) -> None: declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -383,6 +386,7 @@ def test_split_packages(self): self.assertEqual(recipe.path, rec_path) self.assertEqual(recipe.makedepends, set()) + self.assertEqual(recipe.preparedepends, set()) self.assertEqual(recipe.maintainer, "None ") self.assertEqual(recipe.image, "base:v2.1") self.assertEqual(recipe.arch, "rmall") @@ -396,6 +400,7 @@ def test_split_packages(self): declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -417,6 +422,7 @@ def test_split_packages(self): declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -457,6 +463,7 @@ def test_split_packages(self): declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -505,6 +512,7 @@ def test_split_packages(self): declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -554,6 +562,7 @@ def test_split_packages(self): declare -a sha256sums=([0]=SKIP) declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=base:v2.1 declare -- arch=rmall @@ -664,6 +673,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='Mattéo Delabre ' declare -- image=qt:v1.1 declare -- arch=rm1 @@ -718,6 +728,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='Mattéo Delabre ' declare -- image=qt:v1.1 declare -- arch=rm1 @@ -753,6 +764,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='Mattéo Delabre ' declare -- image=qt:v1.1 declare -- arch=rm1 @@ -810,6 +822,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='Mattéo Delabre ' declare -- image=qt:v1.1 declare -- arch=rm1 @@ -845,6 +858,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='Mattéo Delabre ' declare -- image=qt:v1.1 declare -- arch=rm1 @@ -883,6 +897,7 @@ def test_split_archs(self): ) self.assertEqual(rm2.sources, set()) self.assertEqual(rm2.makedepends, set()) + self.assertEqual(rm2.preparedepends, set()) self.assertEqual(rm2.maintainer, "None ") self.assertEqual(rm2.image, "qt:v1.3") self.assertEqual(rm2.arch, "rm2") @@ -897,6 +912,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=qt:v1.3 declare -- arch=rm2 @@ -954,6 +970,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=qt:v1.3 declare -- arch=rm2 @@ -989,6 +1006,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=qt:v1.3 declare -- arch=rm2 @@ -1049,6 +1067,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=qt:v1.3 declare -- arch=rm2 @@ -1084,6 +1103,7 @@ def test_split_archs(self): declare -a sha256sums=() declare -a noextract=() declare -a makedepends=() +declare -a preparedepends=() declare -- maintainer='None ' declare -- image=qt:v1.3 declare -- arch=rm2 diff --git a/tests/test_recipe.py b/tests/test_recipe.py index ff6d67a..fe2b79b 100644 --- a/tests/test_recipe.py +++ b/tests/test_recipe.py @@ -20,6 +20,7 @@ def test_derived_fields(self) -> None: timestamp=datetime.now(), sources=set(), makedepends=set(), + preparedepends=set(), maintainer="Test ", image="", arch="armv7-3.2", diff --git a/toltec/builder.py b/toltec/builder.py index b49f6ab..5248d34 100644 --- a/toltec/builder.py +++ b/toltec/builder.py @@ -15,7 +15,7 @@ from . import hooks from . import bash, util, ipk from .recipe import RecipeBundle, Recipe, Package -from .version import DependencyKind +from .version import Dependency, DependencyKind logger = logging.getLogger(__name__) @@ -261,69 +261,28 @@ def _fetch_sources( local_path, ) - def _prepare(self, recipe: Recipe, src_dir: str) -> None: - """Prepare source files before building.""" - if not recipe.prepare: - logger.debug("Skipping prepare (nothing to do)") - return - - logger.info("Preparing source files") - mount_src = "/src" - uid = os.getuid() - gid = os.getgid() - logs = bash.run_script_in_container( - self.docker, - image=self.IMAGE_PREFIX + "toolchain:v4.0", - mounts=[ - docker.types.Mount( - type="bind", - source=os.path.abspath(src_dir), - target=mount_src, - ), - ], - variables={ - "srcdir": mount_src, - }, - script="\n".join( - [ - f'cd "{mount_src}"', - recipe.prepare, - f"chown -R {uid}:{gid} {mount_src}", - ] - ), - ) - bash.pipe_logs(logger, logs, "prepare()") - - # pylint: disable=too-many-locals - def _build(self, recipe: Recipe, src_dir: str) -> None: - """Build artifacts for a recipe.""" - if not recipe.build: - logger.debug("Skipping build (nothing to do)") - return - - logger.info("Building artifacts") - - # Set fixed atime and mtime for all the source files - epoch = int(recipe.timestamp.timestamp()) - - for filename in util.list_tree(src_dir): - os.utime(filename, (epoch, epoch)) - - mount_src = "/src" - repo_src = "/repo" - uid = os.getuid() - gid = os.getgid() + def _run_script( # pylint: disable=R0913, R0917, R0914 + self, + script: list[str], + arch: str = "rmall", + image: str = "toolchain:v4.0", + dependencies: set[Dependency] | None = None, + mounts: list[docker.types.Mount] | None = None, + variables: bash.Variables | None = None, + ) -> bash.LogGenerator: + """Run a recipe script in a container""" pre_script: list[str] = [] # Install required dependencies build_deps: list[str] = [] host_deps: list[str] = [] - for dep in recipe.makedepends: - if dep.kind == DependencyKind.BUILD: - build_deps.append(dep.package) - elif dep.kind == DependencyKind.HOST: - host_deps.append(dep.package) + if dependencies is not None: + for dep in dependencies: + if dep.kind == DependencyKind.BUILD: + build_deps.append(dep.package) + elif dep.kind == DependencyKind.HOST: + host_deps.append(dep.package) if build_deps: pre_script.extend( @@ -338,25 +297,16 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: ) ) + is_aarch64 = arch.startswith("rmpp") if host_deps: opkg_conf_path = ( "$SYSROOT_AARCH64/etc/opkg/opkg.conf" - if recipe.arch.startswith("rmpp") + if is_aarch64 else "$SYSROOT/etc/opkg/opkg.conf" ) - opkg_exec = ( - "opkg-aarch64" if recipe.arch.startswith("rmpp") else "opkg" - ) - opkg_arch = ( - "aarch64-3.10" - if recipe.arch.startswith("rmpp") - else "armv7-3.2" - ) - opkg_src = ( - "aarch64-k3.10" - if recipe.arch.startswith("rmpp") - else "armv7sf-k3.2" - ) + opkg_exec = "opkg-aarch64" if is_aarch64 else "opkg" + opkg_arch = "aarch64-3.10" if is_aarch64 else "armv7-3.2" + opkg_src = "aarch64-k3.10" if is_aarch64 else "armv7sf-k3.2" pre_script.extend( ( @@ -370,11 +320,11 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: ) ) - if recipe.arch != "rmall": + if arch != "rmall": pre_script.extend( ( - f'echo -n "arch {recipe.arch} 250', - f"src/gz toltec-{recipe.arch} file:///repo/{recipe.arch}", + f'echo -n "arch {arch} 250', + f"src/gz toltec-{arch} file:///repo/{arch}", f'" >> "{opkg_conf_path}"', ) ) @@ -388,12 +338,88 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: ) ) - if recipe.arch.startswith("rmpp"): - pre_script.append(("source /opt/x-tools/switch-aarch64.sh")) + if is_aarch64: + pre_script.extend( + ( + "if [ -f /opt/x-tools/switch-aarch64.sh ]; then", + " source /opt/x-tools/switch-aarch64.sh", + "fi", + ) + ) - logs = bash.run_script_in_container( + return bash.run_script_in_container( self.docker, - image=self.IMAGE_PREFIX + recipe.image, + image=self.IMAGE_PREFIX + image, + mounts=mounts or [], + variables=variables or {}, + script="\n".join( + [ + *pre_script, + *script, + ] + ), + ) + + def _prepare(self, recipe: Recipe, src_dir: str) -> None: + """Prepare source files before building.""" + if not recipe.prepare: + logger.debug("Skipping prepare (nothing to do)") + return + + logger.info("Preparing source files") + mount_src = "/src" + uid = os.getuid() + gid = os.getgid() + logs = self._run_script( + [ + f'cd "{mount_src}"', + recipe.prepare, + f"chown -R {uid}:{gid} {mount_src}", + ], + arch=recipe.arch, + dependencies=recipe.preparedepends, + mounts=[ + docker.types.Mount( + type="bind", + source=os.path.abspath(src_dir), + target=mount_src, + ), + ], + variables={ + "srcdir": mount_src, + }, + ) + bash.pipe_logs(logger, logs, "prepare()") + + # pylint: disable=too-many-locals + def _build(self, recipe: Recipe, src_dir: str) -> None: + """Build artifacts for a recipe.""" + if not recipe.build: + logger.debug("Skipping build (nothing to do)") + return + + logger.info("Building artifacts") + + # Set fixed atime and mtime for all the source files + epoch = int(recipe.timestamp.timestamp()) + + for filename in util.list_tree(src_dir): + os.utime(filename, (epoch, epoch)) + + mount_src = "/src" + repo_src = "/repo" + uid = os.getuid() + gid = os.getgid() + + logs = self._run_script( + [ + f'cd "{mount_src}"', + recipe.build, + f"chown -R {uid}:{gid} {mount_src} {repo_src}", + ], + image=recipe.image, + arch=recipe.arch, + dependencies=recipe.makedepends, mounts=[ docker.types.Mount( type="bind", @@ -409,14 +435,6 @@ def _build(self, recipe: Recipe, src_dir: str) -> None: variables={ "srcdir": mount_src, }, - script="\n".join( - ( - *pre_script, - f'cd "{mount_src}"', - recipe.build, - f"chown -R {uid}:{gid} {mount_src} {repo_src}", - ) - ), ) bash.pipe_logs(logger, logs, "build()") diff --git a/toltec/recipe.py b/toltec/recipe.py index 361feae..04c7aad 100644 --- a/toltec/recipe.py +++ b/toltec/recipe.py @@ -69,6 +69,9 @@ class Recipe: # pylint: disable=too-many-instance-attributes # Set of packages that are needed to build this recipe makedepends: set[Dependency] + # Set of packages that are needed to prepare this recipe + preparedepends: set[Dependency] + # Full name and email address of this recipe’s maintainer maintainer: str diff --git a/toltec/recipe_parsers/bash.py b/toltec/recipe_parsers/bash.py index 0547f0e..f8cde37 100644 --- a/toltec/recipe_parsers/bash.py +++ b/toltec/recipe_parsers/bash.py @@ -181,6 +181,14 @@ def _parse_recipe( # pylint: disable=too-many-locals, disable=too-many-statemen Dependency.parse(dep or "") for dep in makedepends_raw } + preparedepends_raw = _pop_field_indexed( + path, variables, "preparedepends", [] + ) + raw_vars["preparedepends"] = preparedepends_raw + attrs["preparedepends"] = { + Dependency.parse(dep or "") for dep in preparedepends_raw + } + attrs["maintainer"] = raw_vars["maintainer"] = _pop_field_string( path, variables, "maintainer" )