diff --git a/portable-python.yml b/portable-python.yml index a3e59fd..c48c4fa 100644 --- a/portable-python.yml +++ b/portable-python.yml @@ -29,7 +29,18 @@ cpython-additional-packages: # Uncomment to override cpython or a dependency source URL # Note: string "$version" will be replaced with version string (e.g. 1.2.3) # cpython-url: https://my-cpython-mirror/cpython-$version.tar.gz +# cpython-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} # zlib-url: https://my-zlib-mirror/zlib-$version.tar.gz +# zlib-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} +# +# The .tar.gz in projects public releases has additional files not present the tarball of the git tag +# uuid-url: https://my-github-enterprise/api/v3/repos/opensource/libuuid/releases/assets/48151 +# uuid-src-suffix: .tar.gz +# uuid-http-headers: +# - Authorization: Bearer ${GITHUB_TOKEN} +# Accept: application/octet-stream # Uncomment to override the ./configure arguments for a dependency # Note: this will replace the default arguments, not extend them diff --git a/pyproject.toml b/pyproject.toml index 6864983..7513e0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ ignore = [ order-by-type = false [tool.ruff.lint.mccabe] -max-complexity = 14 +max-complexity = 18 [tool.ruff.lint.pydocstyle] convention = "numpy" diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 411aac9..3de7038 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -492,10 +492,23 @@ def is_usable_module(self, name): def cfg_version(self, default): return PPG.config.get_value("%s-version" % self.m_name) or default + def cfg_http_headers(self): + if config_http_headers := PPG.config.get_value("%s-http-headers" % self.m_name): + expanded_http_headers = {} + for header_dict in config_http_headers: + for key, value in header_dict.items(): + expanded_http_headers[os.path.expandvars(key)] = os.path.expandvars(value) + + return expanded_http_headers + def cfg_url(self, version): if config_url := PPG.config.get_value("%s-url" % self.m_name): url_template = Template(config_url) - return url_template.substitute(version=version) + url_subbed = url_template.substitute(version=version) + return os.path.expandvars(url_subbed) + + def cfg_src_suffix(self): + return PPG.config.get_value("%s-src-suffix" % self.m_name) def cfg_configure(self, deps_lib_dir, deps_lib64_dir): if configure := PPG.config.get_value("%s-configure" % self.m_name): @@ -510,6 +523,16 @@ def url(self): """Url of source tarball, if any""" return "" + @property + def headers(self): + """Headers for connecting to source url, if any""" + return self.cfg_http_headers() + + @property + def src_suffix(self): + """Suffix of src archive for when URL doesn't end in the file extension""" + return self.cfg_src_suffix() + @property def version(self): """Version to use""" @@ -632,8 +655,16 @@ def compile(self): self._finalize() return + # Some URL's may not end in file extension, such as with redirects. + # Github releases asset endpoint is this way .../releases/assets/48151 + # Split on '#' for urls that include a checksum, such as #sha256=... fragment basename = runez.basename(self.url, extension_marker="#") + if not basename.endswith((".zip", ".tar.gz")): + suffix = self.src_suffix or ".tar.gz" + suffix = ".%s" % (suffix.strip(".")) # Ensure it starts with a dot (in case config forgot leading dot) + basename = f"{self.m_name}-{self.version}{suffix}" + path = self.setup.folders.sources / basename if not path.exists(): proxies = {} @@ -643,7 +674,8 @@ def compile(self): https_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy") if https_proxy: proxies["https"] = https_proxy - RestClient().download(self.url, path, proxies=proxies) + + RestClient().download(self.url, path, proxies=proxies, headers=self.headers) runez.decompress(path, self.m_src_build, simplify=True) diff --git a/src/portable_python/external/__init__.py b/src/portable_python/external/__init__.py index 10f3fd9..34a443a 100644 --- a/src/portable_python/external/__init__.py +++ b/src/portable_python/external/__init__.py @@ -6,7 +6,7 @@ class GettextTiny(ModuleBuilder): @property def url(self): - return f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz" + return self.cfg_url(self.version) or f"https://github.com/sabotage-linux/gettext-tiny/archive/refs/tags/v{self.version}.tar.gz" @property def version(self): diff --git a/tests/sample-config1.yml b/tests/sample-config1.yml index b33f231..5199071 100644 --- a/tests/sample-config1.yml +++ b/tests/sample-config1.yml @@ -40,3 +40,11 @@ cpython-pep668-externally-managed: cpython-configure: - --enable-shared + +# Exercise custom url +bzip2-url: https://my-enterprise/.../assets/bzip2/123#sha256=123...def +bzip2-version: 1.2.3 +bzip2-src-suffix: tar.gz # Forgot leading dot on purpose +bzip2-http-headers: + - Authorization: Bearer foo + Accept: application/octet-stream diff --git a/tests/test_setup.py b/tests/test_setup.py index 7640ad1..42fcb5c 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -9,8 +9,15 @@ def test_config(cli): with pytest.raises(runez.system.AbortException): PPG.config.parsed_yaml("a: b\ninvalid line", "testing") + # Exercise custom url in config + cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mbzip2") + assert cli.succeeded + assert "Would download https://my-enterprise/.../assets/bzip2/123#sha256=123...def\n" in cli.logged + assert "Would untar build/sources/bzip2-1.2.3.tar.gz -> build/components/bzip2\n" in cli.logged + cli.run("-ntmacos-arm64", "-c", cli.tests_path("sample-config1.yml"), "build", "3.9.7", "-mnone") assert cli.succeeded + assert " -mpip install --no-cache-dir --upgrade my-additional-package" in cli.logged assert "env MACOSX_DEPLOYMENT_TARGET=12" in cli.logged # Comes from more specific macos-arm64.yml assert " -> dist/cpython-3.9.7-macos-arm64.tar.xz" in cli.logged # Comes from macos.yml (not defined in macos-arm64.yml)