diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..d56a644d 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -136,6 +136,16 @@ default=False ) @option("--dryrun", "-n", is_flag=True, default=False) +@option( + "--sign_result_loc", + "-l", + default="/tmp/sign", + help=""" + The local save path for oras to pull the radas signature result. + Sign request will use this path to download the signature result, + Upload will use the file on this path to generate the corresponding .asc files + """, +) @command() def upload( repo: str, @@ -150,7 +160,8 @@ def upload( sign_key: str = "redhatdevel", debug=False, quiet=False, - dryrun=False + dryrun=False, + sign_result_loc="/tmp/sign" ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -221,7 +232,8 @@ def upload( key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, - config=config + config=config, + sign_result_loc=sign_result_loc ) if not succeeded: sys.exit(1) diff --git a/charon/config.py b/charon/config.py index 35efe6eb..2995ffdf 100644 --- a/charon/config.py +++ b/charon/config.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import os from typing import Dict, List, Optional @@ -34,13 +35,20 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__quay_radas_registry_config: Optional[str] = data.get( + "quay_radas_registry_config", None + ) + self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) + self.__radas_sign_timeout_retry_interval: int = data.get( + "radas_sign_timeout_retry_interval", 60 + ) def validate(self) -> bool: if not self.__umb_host: logger.error("Missing host name setting for UMB!") return False if not self.__result_queue: - logger.error("Missing the queue setting to receive siging result in UMB!") + logger.error("Missing the queue setting to receive signing result in UMB!") return False if not self.__request_queue: logger.error("Missing the queue setting to send signing request in UMB!") @@ -57,10 +65,17 @@ def validate(self) -> bool: if self.__root_ca and not os.access(self.__root_ca, os.R_OK): logger.error("The root ca file is not valid!") return False + if self.__quay_radas_registry_config and not os.access( + self.__quay_radas_registry_config, os.R_OK + ): + self.__quay_radas_registry_config = None + logger.warning( + "The quay registry config for oras is not valid, will ignore the registry config!" + ) return True def umb_target(self) -> str: - return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + return f"amqps://{self.__umb_host}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue @@ -77,7 +92,7 @@ def client_key(self) -> str: def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): - with open(pass_file, 'r') as f: + with open(pass_file, "r") as f: return f.read() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") @@ -86,6 +101,15 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def quay_radas_registry_config(self) -> Optional[str]: + return self.__quay_radas_registry_config + + def radas_sign_timeout_retry_count(self) -> int: + return self.__radas_sign_timeout_retry_count + + def radas_sign_timeout_retry_interval(self) -> int: + return self.__radas_sign_timeout_retry_interval + class CharonConfig(object): """CharonConfig is used to store all configurations for charon @@ -102,9 +126,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config__: RadasConfig = RadasConfig(radas_config) + self.__radas_config__ = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -133,7 +158,10 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable - def get_radas_config(self) -> RadasConfig: + def is_radas_enabled(self) -> bool: + return bool(self.__radas_config__ and self.__radas_config__.validate()) + + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ @@ -141,14 +169,12 @@ def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) - data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') + data = read_yaml_from_file_path(config_file_path, "schemas/charon.json") return CharonConfig(data) def get_template(template_file: str) -> str: - template = os.path.join( - os.getenv("HOME", ''), ".charon/template", template_file - ) + template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file) if os.path.isfile(template): with open(template, encoding="utf-8") as file_: return file_.read() diff --git a/charon/constants.py b/charon/constants.py index 6751aecd..35ea560a 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,3 +175,5 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..4ca1be0d 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,6 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature +import charon.pkgs.radas_signature_handler as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix @@ -274,7 +275,8 @@ def handle_maven_uploading( key=None, dry_run=False, manifest_bucket_name=None, - config=None + config=None, + sign_result_loc="/tmp/sign" ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -408,11 +410,38 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if contain_signature is set to True - if gen_sign: - conf = get_config(config) - if not conf: - sys.exit(1) + # 10. Generate signature file if radas sign is enabled, + # or do detached sign if contain_signature is set to True + conf = get_config(config) + if not conf: + sys.exit(1) + + if conf.is_radas_enabled(): + logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( + top_level=top_level, sign_result_loc=sign_result_loc + ) + if not _generated_signs: + logger.error( + "No sign result files were downloaded, " + "please make sure the sign process is already done and without timeout") + return (tmp_root, False) + + failed_metas.extend(_failed_metas) + generated_signs.extend(_generated_signs) + logger.info("Radas signature files generation done.\n") + + logger.info("Start upload radas signature files to s3 bucket %s\n", bucket_name) + _failed_metas = s3_client.upload_signatures( + meta_file_paths=generated_signs, + target=(bucket_name, prefix), + product=None, + root=top_level + ) + failed_metas.extend(_failed_metas) + logger.info("Radas signature files uploading done.\n") + + elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) command = conf.get_detach_signature_command() artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))] diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py new file mode 100644 index 00000000..b5446def --- /dev/null +++ b/charon/pkgs/oras_client.py @@ -0,0 +1,72 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oras.client +import logging +from charon.config import get_config +from typing import List +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +class OrasClient: + """ + Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. + """ + + def __init__(self): + self.conf = get_config() + self.client = oras.client.OrasClient() + + def login_if_needed(self, registry: str) -> None: + """ + If quay_radas_registry_config is provided, call login to authenticate. + """ + if not registry.startswith("http://") and not registry.startswith("https://"): + registry = "https://" + registry + registry = urlparse(registry).netloc + + rconf = self.conf.get_radas_config() if self.conf else None + if rconf and rconf.quay_radas_registry_config(): + logger.info("Logging in to registry: %s", registry) + res = self.client.login( + hostname=registry, + config_path=rconf.quay_radas_registry_config(), + ) + logger.info(res) + else: + logger.info("Registry config is not provided, skip login.") + + def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: + """ + Call oras‑py’s pull method to pull the remote file to local. + Args: + result_reference_url (str): + Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”). + """ + files = [] + try: + self.login_if_needed(registry=result_reference_url) + files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) + logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) + except Exception as e: + logger.error( + "Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e + ) + return files diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py new file mode 100644 index 00000000..c04f0bbf --- /dev/null +++ b/charon/pkgs/radas_signature_handler.py @@ -0,0 +1,181 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import json +import os +import asyncio +import sys +from typing import List, Any, Tuple, Callable, Dict +from charon.config import get_config +from charon.pkgs.oras_client import OrasClient +from proton import Event +from proton.handlers import MessagingHandler + +logger = logging.getLogger(__name__) + + +class UmbListener(MessagingHandler): + """ + UmbListener class (AMQP version), register this when setup UmbClient + Attributes: + sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, + this value transfers from the cmd flag, should register UmbListener when the client starts + """ + + def __init__(self, sign_result_loc: str) -> None: + super().__init__() + self.sign_result_loc = sign_result_loc + + def on_start(self, event: Event) -> None: + """ + On start callback + """ + conf = get_config() + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + # explicit check to pass the type checker + if rconf is None: + sys.exit(1) + conn = event.container.connect(rconf.umb_target()) + event.container.create_receiver(conn, rconf.result_queue()) + logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) + + def on_message(self, event: Event) -> None: + """ + On message callback + """ + self._process_message(event.message.body) + + def on_connection_error(self, event: Event) -> None: + """ + On connection error callback + """ + logger.error("Received an error event:\n%s", event) + + def on_disconnected(self, event: Event) -> None: + """ + On disconnected callback + """ + logger.error("Disconnected from AMQP broker.") + + def _process_message(self, msg: Any) -> None: + """ + Process a message received from UMB + Args: + msg: The message body received + """ + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") + + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return + + logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) + + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) + + +def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: + """ + Generate .asc files based on RADAS sign result json file + """ + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] + + if not files: + return [], [] + + if len(files) > 1: + logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + return [], [] + + # should only have the single sign result json file from the radas registry + json_file_path = files[0] + try: + with open(json_file_path, "r") as f: + data = json.load(f) + except Exception as e: + logger.error("Failed to read or parse the JSON file: %s", e) + raise + + async def generate_single_sign_file( + file_path: str, + signature: str, + failed_paths: List[str], + generated_signs: List[str], + sem: asyncio.BoundedSemaphore, + ): + async with sem: + if not file_path or not signature: + logger.error("Invalid JSON entry") + return + # remove the root path maven-repository + filename = file_path.split("/", 1)[1] + + artifact_path = os.path.join(top_level, filename) + asc_filename = f"{filename}.asc" + signature_path = os.path.join(top_level, asc_filename) + + if not os.path.isfile(artifact_path): + logger.warning("Artifact missing, skip signature file generation") + return + + try: + with open(signature_path, "w") as asc_file: + asc_file.write(signature) + generated_signs.append(signature_path) + logger.info("Generated .asc file: %s", signature_path) + except Exception as e: + failed_paths.append(signature_path) + logger.error("Failed to write .asc file for %s: %s", artifact_path, e) + + result = data.get("result", []) + return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + + +def __do_path_cut_and( + path_handler: Callable, data: List[Dict[str, str]] +) -> Tuple[List[str], List[str]]: + + failed_paths: List[str] = [] + generated_signs: List[str] = [] + tasks = [] + sem = asyncio.BoundedSemaphore(10) + for item in data: + file_path = item.get("file") + signature = item.get("signature") + tasks.append( + asyncio.ensure_future( + path_handler(file_path, signature, failed_paths, generated_signs, sem) + ) + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.gather(*tasks)) + return (failed_paths, generated_signs) diff --git a/requirements.txt b/requirements.txt index 7919fc27..75bb4b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 +oras>=0.2.31 +python-qpid-proton>=0.39.0 \ No newline at end of file