diff --git a/charon.spec b/charon.spec index 147d9885..fb9f2073 100644 --- a/charon.spec +++ b/charon.spec @@ -1,7 +1,7 @@ %global owner Commonjava %global modulename charon -%global charon_version 1.3.3 +%global charon_version 1.4.0 %global sdist_tar_name %{modulename}-%{charon_version} %global python3_pkgversion 3 @@ -64,6 +64,23 @@ export LANG=en_US.UTF-8 LANGUAGE=en_US.en LC_ALL=en_US.UTF-8 %changelog +* Fri Jun 27 2025 Gang Li +- 1.4.0 release +- Add RADAS signature support + +* Mon Jun 23 2025 Gang Li +- 1.3.4 release +- Fix the sorting problem of index page items + +* Mon Dec 16 2024 Gang Li +- 1.3.3 release +- Fix npm del error when deleting a package which has overlapped name with others +- Some code refinement + +* Thu Jul 11 2024 Gang Li +- 1.3.2 release +- Some updates in the Containerfile. + * Tue May 7 2024 Gang Li - 1.3.1 release - Add checksum refresh command: refresh checksum files for maven artifacts diff --git a/charon/cmd/__init__.py b/charon/cmd/__init__.py index 16a0129d..985d7f79 100644 --- a/charon/cmd/__init__.py +++ b/charon/cmd/__init__.py @@ -13,16 +13,19 @@ See the License for the specific language governing permissions and limitations under the License. """ -from click import group +from click import group, version_option, pass_context from charon.cmd.cmd_upload import upload from charon.cmd.cmd_delete import delete from charon.cmd.cmd_index import index from charon.cmd.cmd_checksum import init_checksum, checksum from charon.cmd.cmd_cache import init_cf, cf +from charon.cmd.cmd_sign import sign @group() -def cli(): +@version_option() +@pass_context +def cli(ctx): """Charon is a tool to synchronize several types of artifacts repository data to Red Hat Ronda service (maven.repository.redhat.com). @@ -41,3 +44,6 @@ def cli(): # init checksum command init_checksum() cli.add_command(checksum) + +# radas sign cmd +cli.add_command(sign) diff --git a/charon/cmd/cmd_sign.py b/charon/cmd/cmd_sign.py new file mode 100644 index 00000000..49b2bf22 --- /dev/null +++ b/charon/cmd/cmd_sign.py @@ -0,0 +1,137 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +from typing import List + +from charon.config import get_config +from charon.pkgs.radas_sign import sign_in_radas +from charon.cmd.internal import _decide_mode +from charon.constants import DEFAULT_RADAS_SIGN_IGNORES + +from click import command, option, argument + +import traceback +import logging +import sys +import datetime + +logger = logging.getLogger(__name__) + + +@argument( + "repo_url", + type=str +) +@option( + "--requester", + "-r", + help=""" + The requester who sends the signing request. + """, + required=True +) +@option( + "--result_path", + "-p", + help=""" + The path which will save the sign result file. + """, + required=True +) +@option( + "--ignore_patterns", + "-i", + multiple=True, + help=""" + The regex patterns list to filter out the files which should + not be allowed to upload to S3. Can accept more than one pattern. + """ +) +@option( + "--config", + "-c", + help=""" + The charon configuration yaml file path. Default is + $HOME/.charon/charon.yaml + """ +) +@option( + "--sign_key", + "-k", + help=""" + rpm-sign key to be used, will replace {{ key }} in default configuration for signature. + Does noting if detach_signature_command does not contain {{ key }} field. + """, + required=True +) +@option( + "--debug", + "-D", + help="Debug mode, will print all debug logs for problem tracking.", + is_flag=True, + default=False +) +@option( + "--quiet", + "-q", + help="Quiet mode, will shrink most of the logs except warning and errors.", + is_flag=True, + default=False +) +@command() +def sign( + repo_url: str, + requester: str, + result_path: str, + sign_key: str, + ignore_patterns: List[str] = None, + config: str = None, + debug=False, + quiet=False +): + """Do signing against files in the repo zip in repo_url through + radas service. The repo_url points to the maven zip repository + in quay.io, which will be sent as the source of the signing. + """ + logger.debug("%s", ignore_patterns) + try: + current = datetime.datetime.now().strftime("%Y%m%d%I%M") + _decide_mode("radas_sign", current, is_quiet=quiet, is_debug=debug) + conf = get_config(config) + if not conf: + logger.error("The charon configuration is not valid!") + sys.exit(1) + radas_conf = conf.get_radas_config() + if not radas_conf or not radas_conf.validate(): + logger.error("The configuration for radas is not valid!") + sys.exit(1) + # All ignore files in global config should also be ignored in signing. + ig_patterns = conf.get_ignore_patterns() + ig_patterns.extend(DEFAULT_RADAS_SIGN_IGNORES) + if ignore_patterns: + ig_patterns.extend(ignore_patterns) + ig_patterns = list(set(ig_patterns)) + args = { + "repo_url": repo_url, + "requester": requester, + "sign_key": sign_key, + "result_path": result_path, + "ignore_patterns": ig_patterns, + "radas_config": radas_conf + } + sign_in_radas(**args) # type: ignore + except Exception: + print(traceback.format_exc()) + sys.exit(2) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..3a0e6990 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -136,6 +136,14 @@ default=False ) @option("--dryrun", "-n", is_flag=True, default=False) +@option( + "--sign_result_file", + "-l", + help=""" + The path of the file which contains radas signature result. + Upload will use the file to generate the corresponding .asc files + """, +) @command() def upload( repo: str, @@ -150,7 +158,8 @@ def upload( sign_key: str = "redhatdevel", debug=False, quiet=False, - dryrun=False + dryrun=False, + sign_result_file=None, ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -221,7 +230,8 @@ def upload( key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, - config=config + config=config, + sign_result_file=sign_result_file ) if not succeeded: sys.exit(1) diff --git a/charon/config.py b/charon/config.py index 86f826ea..65f23020 100644 --- a/charon/config.py +++ b/charon/config.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import os from typing import Dict, List, Optional @@ -24,6 +25,104 @@ logger = logging.getLogger(__name__) +class RadasConfig(object): + def __init__(self, data: Dict): + self.__umb_host: str = data.get("umb_host", None) + self.__umb_host_port: str = data.get("umb_host_port", "5671") + self.__result_queue: str = data.get("result_queue", None) + self.__request_chan: str = data.get("request_channel", None) + self.__client_ca: str = data.get("client_ca", None) + self.__client_key: str = data.get("client_key", None) + self.__client_key_pass_file: str = data.get("client_key_pass_file", None) + self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__quay_radas_registry_config: Optional[str] = data.get( + "quay_radas_registry_config", None + ) + self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) + self.__radas_sign_timeout_retry_interval: int = data.get( + "radas_sign_timeout_retry_interval", 60 + ) + self.__radas_receiver_timeout: int = int(data.get("radas_receiver_timeout", 1800)) + + def validate(self) -> bool: + if not self.__umb_host: + logger.error("Missing host name setting for UMB!") + return False + if not self.__result_queue: + logger.error("Missing the queue setting to receive signing result in UMB!") + return False + if not self.__request_chan: + logger.error("Missing the queue setting to send signing request in UMB!") + return False + if self.__client_ca and not os.access(self.__client_ca, os.R_OK): + logger.error("The client CA file is not valid!") + return False + if self.__client_key and not os.access(self.__client_key, os.R_OK): + logger.error("The client key file is not valid!") + return False + if self.__client_key_pass_file and not os.access(self.__client_key_pass_file, os.R_OK): + logger.error("The client key password file is not valid!") + return False + if self.__root_ca and not os.access(self.__root_ca, os.R_OK): + logger.error("The root ca file is not valid!") + return False + if self.__quay_radas_registry_config and not os.access( + self.__quay_radas_registry_config, os.R_OK + ): + self.__quay_radas_registry_config = None + logger.warning( + "The quay registry config for oras is not valid, will ignore the registry config!" + ) + return True + + def umb_target(self) -> str: + if self.ssl_enabled(): + return f"amqps://{self.__umb_host.strip()}:{self.__umb_host_port}" + else: + return f"amqp://{self.__umb_host.strip()}:{self.__umb_host_port}" + + def result_queue(self) -> str: + return self.__result_queue.strip() + + def request_channel(self) -> str: + return self.__request_chan.strip() + + def client_ca(self) -> str: + return self.__client_ca.strip() + + def client_key(self) -> str: + return self.__client_key.strip() + + def client_key_password(self) -> str: + pass_file = self.__client_key_pass_file + if os.access(pass_file, os.R_OK): + with open(pass_file, "r") as f: + return f.read().strip() + elif pass_file: + logger.warning("The key password file is not accessible. Will ignore the password.") + return "" + + def root_ca(self) -> str: + return self.__root_ca.strip() + + def ssl_enabled(self) -> bool: + return bool(self.__client_ca and self.__client_key and self.__root_ca) + + def quay_radas_registry_config(self) -> Optional[str]: + if self.__quay_radas_registry_config: + return self.__quay_radas_registry_config.strip() + return None + + def radas_sign_timeout_retry_count(self) -> int: + return self.__radas_sign_timeout_retry_count + + def radas_sign_timeout_retry_interval(self) -> int: + return self.__radas_sign_timeout_retry_interval + + def receiver_timeout(self) -> int: + return self.__radas_receiver_timeout + + class CharonConfig(object): """CharonConfig is used to store all configurations for charon tools. @@ -39,6 +138,13 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + radas_config: Dict = data.get("radas", None) + self.__radas_config: Optional[RadasConfig] = None + if radas_config: + self.__radas_config = RadasConfig(radas_config) + self.__radas_enabled = bool(self.__radas_config and self.__radas_config.validate()) + else: + self.__radas_enabled = False def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -67,19 +173,23 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_enabled(self) -> bool: + return self.__radas_enabled + + def get_radas_config(self) -> Optional[RadasConfig]: + return self.__radas_config + def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) - data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') + data = read_yaml_from_file_path(config_file_path, "schemas/charon.json") return CharonConfig(data) def get_template(template_file: str) -> str: - template = os.path.join( - os.getenv("HOME", ''), ".charon/template", template_file - ) + template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file) if os.path.isfile(template): with open(template, encoding="utf-8") as file_: return file_.read() diff --git a/charon/constants.py b/charon/constants.py index 6751aecd..e8056dc0 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,3 +175,10 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 + +DEFAULT_RADAS_SIGN_IGNORES = [ + r".*\.md5$", r".*\.sha1$", r".*\.sha128$", r".*\.sha256$", + r".*\.sha512$", r".*\.asc$" +] diff --git a/charon/pkgs/indexing.py b/charon/pkgs/indexing.py index 4d50e036..4710cdab 100644 --- a/charon/pkgs/indexing.py +++ b/charon/pkgs/indexing.py @@ -23,7 +23,7 @@ from jinja2 import Template import os import logging -from typing import List, Set, Dict +from typing import List, Dict from charon.utils.strings import remove_prefix @@ -48,7 +48,7 @@ def __get_index_template(package_type: str) -> str: class IndexedHTML(object): # object for holding index html file data - def __init__(self, title: str, header: str, items: Set[str]): + def __init__(self, title: str, header: str, items: List[str]): self.title = title self.header = header self.items = items @@ -174,8 +174,8 @@ def __to_html_content(package_type: str, contents: List[str], folder: str) -> st items = temp_items else: items.extend(contents) - items_set = set(__sort_index_items(items)) - index = IndexedHTML(title=folder, header=folder, items=items_set) + items_result = list(filter(lambda c: c.strip(), __sort_index_items(set(items)))) + index = IndexedHTML(title=folder, header=folder, items=items_result) return index.generate_index_file_content(package_type) @@ -303,8 +303,8 @@ def re_index( real_contents.append(c) else: real_contents = contents - logger.debug(real_contents) index_content = __to_html_content(package_type, real_contents, path) + logger.debug("The re-indexed page content: %s", index_content) if not dry_run: index_path = os.path.join(path, "index.html") if path == "/": diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..5ccee694 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,6 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature +import charon.pkgs.radas_sign as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix @@ -274,7 +275,8 @@ def handle_maven_uploading( key=None, dry_run=False, manifest_bucket_name=None, - config=None + config=None, + sign_result_file=None ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -408,11 +410,38 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if contain_signature is set to True - if gen_sign: - conf = get_config(config) - if not conf: - sys.exit(1) + # 10. Generate signature file if radas sign is enabled, + # or do detached sign if contain_signature is set to True + conf = get_config(config) + if not conf: + sys.exit(1) + + if conf.is_radas_enabled() and sign_result_file and os.path.isfile(sign_result_file): + logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( + top_level=top_level, sign_result_file=sign_result_file + ) + if not _generated_signs: + logger.error( + "No sign result files were generated, " + "please make sure the sign process is already done and without timeout") + return (tmp_root, False) + + failed_metas.extend(_failed_metas) + generated_signs.extend(_generated_signs) + logger.info("Radas signature files generation done.\n") + + logger.info("Start upload radas signature files to s3 bucket %s\n", bucket_name) + _failed_metas = s3_client.upload_signatures( + meta_file_paths=generated_signs, + target=(bucket_name, prefix), + product=None, + root=top_level + ) + failed_metas.extend(_failed_metas) + logger.info("Radas signature files uploading done.\n") + + elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) command = conf.get_detach_signature_command() artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))] diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py new file mode 100644 index 00000000..b5446def --- /dev/null +++ b/charon/pkgs/oras_client.py @@ -0,0 +1,72 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import oras.client +import logging +from charon.config import get_config +from typing import List +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + + +class OrasClient: + """ + Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. + """ + + def __init__(self): + self.conf = get_config() + self.client = oras.client.OrasClient() + + def login_if_needed(self, registry: str) -> None: + """ + If quay_radas_registry_config is provided, call login to authenticate. + """ + if not registry.startswith("http://") and not registry.startswith("https://"): + registry = "https://" + registry + registry = urlparse(registry).netloc + + rconf = self.conf.get_radas_config() if self.conf else None + if rconf and rconf.quay_radas_registry_config(): + logger.info("Logging in to registry: %s", registry) + res = self.client.login( + hostname=registry, + config_path=rconf.quay_radas_registry_config(), + ) + logger.info(res) + else: + logger.info("Registry config is not provided, skip login.") + + def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: + """ + Call oras‑py’s pull method to pull the remote file to local. + Args: + result_reference_url (str): + Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”). + """ + files = [] + try: + self.login_if_needed(registry=result_reference_url) + files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) + logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) + except Exception as e: + logger.error( + "Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e + ) + return files diff --git a/charon/pkgs/radas_sign.py b/charon/pkgs/radas_sign.py new file mode 100644 index 00000000..fcdf6e49 --- /dev/null +++ b/charon/pkgs/radas_sign.py @@ -0,0 +1,414 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import json +import os +import sys +import asyncio +import uuid +import time +from typing import List, Any, Tuple, Callable, Dict, Optional +from charon.config import RadasConfig +from charon.pkgs.oras_client import OrasClient +from charon.utils import files +from proton import SSLDomain, Message, Event, Sender, Connection +from proton.handlers import MessagingHandler +from proton.reactor import Container + +logger = logging.getLogger(__name__) + + +class RadasReceiver(MessagingHandler): + """ + This receiver will listen to UMB message queue to receive signing message for + signing result. + Attributes: + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”) for oras pull result, this value transfers + from the cmd flag,should register UmbListener when the client starts + request_id (str): + Identifier of the request for the signing result + rconf (RadasConfig): + the configurations for the radas messaging system. + sign_result_status (str): + Result of the signing(success/failed) + sign_result_errors (list): + Any errors encountered if signing fails, this will be empty list if successful + """ + + def __init__(self, sign_result_loc: str, request_id: str, rconf: RadasConfig) -> None: + super().__init__() + self.sign_result_loc = sign_result_loc + self.request_id = request_id + self.sign_result_status: Optional[str] = None + self.sign_result_errors: List[str] = [] + self.rconf = rconf + self._conn: Optional[Connection] = None + self._message_handled = False + self._start_time = 0.0 + self._timeout_check_delay = 30.0 + self._ssl: Optional[SSLDomain] = None + if rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasReceiver") + + def on_start(self, event: Event) -> None: + umb_target = self.rconf.umb_target() + container = event.container + self._conn = container.connect( + url=umb_target, + ssl_domain=self._ssl, + heartbeat=500 + ) + receiver = container.create_receiver( + context=self._conn, source=self.rconf.result_queue(), + ) + self.log.info("Listening on %s, queue: %s", + umb_target, + receiver.source.address) + self._start_time = time.time() + container.schedule(self._timeout_check_delay, self) + + def on_timer_task(self, event: Event) -> None: + current = time.time() + timeout = self.rconf.receiver_timeout() + idle_time = current - self._start_time + self.log.debug("Checking timeout: passed %s seconds, timeout time %s seconds", + idle_time, timeout) + if idle_time > self.rconf.receiver_timeout(): + self.log.error("The receiver did not receive messages for more than %s seconds," + " and needs to stop receiving and quit.", timeout) + self._close(event) + else: + event.container.schedule(self._timeout_check_delay, self) + + def on_message(self, event: Event) -> None: + self.log.debug("Got message: %s", event.message.body) + self._process_message(event.message.body) + if self._message_handled: + self.log.debug("The signing result is handled.") + self._close(event) + + def on_error(self, event: Event) -> None: + self.log.error("Received an error event:\n%s", event.message.body) + + def on_disconnected(self, event: Event) -> None: + self.log.info("Disconnected from AMQP broker: %s", + event.connection.connected_address) + + def _close(self, event: Event) -> None: + if event: + if event.connection: + event.connection.close() + if event.container: + event.container.stop() + + def _process_message(self, msg: Any) -> None: + """ + Process a message received from UMB + Args: + msg: The message body received + """ + msg_dict = json.loads(msg) + radas_response = msg_dict.get("msg") + if not radas_response: + self.log.info( + "Message %s is not valid, ignoring", + msg_dict + ) + return + + msg_request_id = radas_response.get("request_id") + if msg_request_id != self.request_id: + self.log.info( + "Message request_id %s does not match the request_id %s from sender, ignoring", + msg_request_id, + self.request_id, + ) + return + + self._message_handled = True + self.log.info( + "Start to process the sign event message, request_id %s is matched", msg_request_id + ) + self.sign_result_status = radas_response.get("signing_status") + self.sign_result_errors = radas_response.get("errors", []) + if self.sign_result_status == "success": + result_reference_url = radas_response.get("result_reference") + if not result_reference_url: + self.log.warning("Not found result_reference in message，ignore.") + return + + self.log.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) + + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + if files and len(files) > 0: + self.log.info("Number of files pulled: %d, path: %s", len(files), files[0]) + else: + self.log.error("The signing result received with failed status. Errors: %s", + self.sign_result_errors) + + +class RadasSender(MessagingHandler): + """ + This simple sender will send given string massage to UMB message queue to request signing. + Attributes: + payload (str): payload json string for radas to read, + this value construct from the cmd flag + rconf (RadasConfig): the configurations for the radas messaging + system. + status (str): tell if status for message sending, only "success" + means the message is sent successfully. + """ + def __init__(self, payload: Any, rconf: RadasConfig): + super(RadasSender, self).__init__() + self.payload = payload + self.rconf = rconf + self.status: Optional[str] = None + self._message_sent = False # Flag to track if message was sent + self._retried = 0 + self._pending: Optional[Message] = None + self._message: Optional[Message] = None + self._container: Optional[Container] = None + self._sender: Optional[Sender] = None + self._ssl: Optional[SSLDomain] = None + if self.rconf.ssl_enabled(): + self._ssl = SSLDomain(SSLDomain.MODE_CLIENT) + self._ssl.set_trusted_ca_db(self.rconf.root_ca()) + self._ssl.set_peer_authentication(SSLDomain.VERIFY_PEER) + self._ssl.set_credentials( + self.rconf.client_ca(), + self.rconf.client_key(), + self.rconf.client_key_password() + ) + self.log = logging.getLogger("charon.pkgs.radas_sign.RadasSender") + + def on_start(self, event): + self._container = event.container + self.log.debug("Start creating connection for sender to %s", self.rconf.umb_target()) + conn = self._container.connect( + url=self.rconf.umb_target(), + ssl_domain=self._ssl, + heartbeat=500 + ) + if conn: + self.log.debug("Start creating sender") + self._sender = self._container.create_sender(conn, self.rconf.request_channel()) + self.log.debug("Sender created. Remote address: %s", self._sender.target.address) + + def on_connection_opened(self, event): + conn = event.connection + self.log.debug("Connection to %s is created.", conn.hostname) + + def on_sendable(self, event): + if not self._message_sent: + msg = Message(body=self.payload, durable=True) + self.log.debug("Sending message: %s to %s", msg.body, event.sender.target.address) + self._send_msg(msg) + self._message = msg + self._message_sent = True + + def on_error(self, event): + self.log.error("Error happened during message sending, reason %s", + event.description) + self.status = "failed" + + def on_rejected(self, event): + self._pending = self._message + self._handle_failed_delivery("Rejected") + + def on_released(self, event): + self._pending = self._message + self._handle_failed_delivery("Released") + + def on_accepted(self, event): + self.log.info("Message accepted by receiver: %s", event.delivery.link.target.address) + self.status = "success" + self.close() # Close connection after confirmation + + def on_timer_task(self, event): + message_to_retry = self._message + self._send_msg(message_to_retry) + self._pending = None + + def close(self): + self.log.info("Message has been sent successfully, close connection") + if self._sender: + self._sender.close() + if self._container: + self._container.stop() + + def _send_msg(self, msg: Message): + if self._sender and self._sender.credit > 0: + self._sender.send(msg) + self.log.debug("Message %s sent", msg.body) + else: + self.log.warning("Sender not ready or no credit available") + + def _handle_failed_delivery(self, reason: str): + if self._pending: + msg = self._pending + self.log.warning("Message %s failed for reason: %s", msg.body, reason) + max_retries = self.rconf.radas_sign_timeout_retry_count() + if self._retried < max_retries: + # Schedule retry + self._retried = self._retried + 1 + self.log.info("Scheduling retry %s/%s for message %s", + self._retried, max_retries, msg.body) + # Schedule retry after delay + if self._container: + self._container.schedule(self.rconf.radas_sign_timeout_retry_interval(), self) + else: + # Max retries exceeded + self.log.error("Message %s failed after %s retries", msg.body, max_retries) + self.status = "failed" + self._pending = None + else: + self.log.info("Message has been sent successfully, close connection") + self.close() + + +def generate_radas_sign(top_level: str, sign_result_file: str) -> Tuple[List[str], List[str]]: + """ + Generate .asc files based on RADAS sign result json file + """ + if not sign_result_file or not os.path.isfile(sign_result_file): + logger.error("Sign result file does not exist: %s", sign_result_file) + return [], [] + + # should only have the single sign result json file from the radas registry + try: + with open(sign_result_file, "r") as f: + data = json.load(f) + except Exception as e: + logger.error("Failed to read or parse the JSON file: %s", e) + raise + + async def generate_single_sign_file( + file_path: str, + signature: str, + failed_paths: List[str], + generated_signs: List[str], + sem: asyncio.BoundedSemaphore, + ): + async with sem: + if not file_path or not signature: + logger.error("Invalid JSON entry") + return + # remove the root path maven-repository + filename = file_path.split("/", 1)[1] + + artifact_path = os.path.join(top_level, filename) + asc_filename = f"{filename}.asc" + signature_path = os.path.join(top_level, asc_filename) + + if not os.path.isfile(artifact_path): + logger.warning( + "Artifact %s missing, skip signature file generation.", + artifact_path) + return + + try: + files.overwrite_file(signature_path, signature) + generated_signs.append(signature_path) + logger.debug("Generated .asc file: %s", signature_path) + except Exception as e: + failed_paths.append(signature_path) + logger.error("Failed to write .asc file for %s: %s", artifact_path, e) + + result = data.get("results", []) + (_failed_metas, _generated_signs) = __do_path_cut_and(generate_single_sign_file, result) + logger.info( + "Signature generation done. There are %s signature files generated.", + len(_generated_signs)) + return (_failed_metas, _generated_signs) + + +def __do_path_cut_and( + path_handler: Callable, data: List[Dict[str, str]] +) -> Tuple[List[str], List[str]]: + + failed_paths: List[str] = [] + generated_signs: List[str] = [] + tasks = [] + sem = asyncio.BoundedSemaphore(10) + for item in data: + file_path = item.get("file") + signature = item.get("signature") + tasks.append( + asyncio.ensure_future( + path_handler(file_path, signature, failed_paths, generated_signs, sem) + ) + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.gather(*tasks)) + return (failed_paths, generated_signs) + + +def sign_in_radas(repo_url: str, + requester: str, + sign_key: str, + result_path: str, + ignore_patterns: List[str], + radas_config: RadasConfig): + """ + This function will be responsible to do the overall controlling of the whole process, + like trigger the send and register the receiver, and control the wait and timeout there. + """ + logger.debug("params. repo_url: %s, requester: %s, sign_key: %s, result_path: %s", + repo_url, requester, sign_key, result_path) + request_id = str(uuid.uuid4()) + exclude = ignore_patterns if ignore_patterns else [] + payload = { + "request_id": request_id, + "requested_by": requester, + "type": "mrrc", + "file_reference": repo_url, + "sig_keyname": sign_key, + "exclude": exclude + } + + sender = RadasSender(json.dumps(payload), radas_config) + container = Container(sender) + container.run() + + if not sender.status == "success": + logger.error("Something wrong happened in message sending, see logs") + sys.exit(1) + + # request_id = "some-request-id-1" # for test purpose + receiver = RadasReceiver(result_path, request_id, radas_config) + Container(receiver).run() + + status = receiver.sign_result_status + if status != "success": + logger.error("The signing result is processed with errors: %s", + receiver.sign_result_errors) + sys.exit(1) diff --git a/charon/schemas/charon.json b/charon/schemas/charon.json index f6a931d1..3ffde818 100644 --- a/charon/schemas/charon.json +++ b/charon/schemas/charon.json @@ -30,6 +30,44 @@ "type": "string", "description": "signature command to be used for signature" }, + "radas": { + "type": "object", + "descrition": "", + "properties": { + "umb_host": { + "type": "string", + "description": "The host of UMB" + }, + "umb_host_port": { + "type": "string", + "description": "The port of UMB host" + }, + "result_queue": { + "type": "string", + "description": "The queue in UMB to receive radas signing result" + }, + "request_queue": { + "type": "string", + "description": "The queue in UMB to send signing request to RADAS" + }, + "client_ca": { + "type": "string", + "description": "the client ca file path" + }, + "client_key": { + "type": "string", + "description": "the client key file path" + }, + "client_key_pass_file":{ + "type": "string", + "description": "the file contains password of the client key" + }, + "root_ca": { + "type": "string", + "description": "the root ca file path" + } + } + }, "targets": { "type": "object", "patternProperties": { diff --git a/image/Containerfile b/image/Containerfile index 267cb9f0..a84a8da9 100644 --- a/image/Containerfile +++ b/image/Containerfile @@ -19,16 +19,16 @@ # 4. Start using uploader # charon upload/delete from /home/charon/upload/... ### -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 as builder +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 as builder ARG GIT_BRANCH=release -RUN microdnf install -y git-core python3.12 python3.12-pip && microdnf clean all +RUN microdnf install -y git-core python3.12-devel python3.12-pip gcc openssl-devel && microdnf clean all RUN git clone -b ${GIT_BRANCH} --depth 1 https://github.com/Commonjava/charon.git RUN pip3 install --no-cache-dir --upgrade pip RUN pip3 wheel ./charon -FROM registry.access.redhat.com/ubi8-minimal:8.10-1052 +FROM registry.access.redhat.com/ubi8-minimal:8.10-1295 ARG USER=charon ARG UID=10000 @@ -38,7 +38,7 @@ WORKDIR ${HOME_DIR} USER root -RUN microdnf install -y python3.12 python3.12-pip shadow-utils && microdnf clean all +RUN microdnf install -y python3.12-devel python3.12-pip shadow-utils gcc openssl-devel && microdnf clean all RUN useradd -d ${HOME_DIR} -u ${UID} -g 0 -m -s /bin/bash ${USER} \ && chown ${USER}:0 ${HOME_DIR} \ && chmod -R g+rwx ${HOME_DIR} \ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..b667868d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,110 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "setuptools-scm"] + +[project] +name = "charon" +version = "1.4.0" +authors = [ + {name = "RedHat EXD SPMM"}, +] +readme = "README.md" +keywords = ["charon", "mrrc", "maven", "npm", "build", "java"] +license-files = ["LICENSE"] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 1 - Planning", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Topic :: Software Development :: Build Tools", + "Topic :: Utilities", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "Jinja2>=3.1.3", + "boto3>=1.18.35", + "botocore>=1.21.35", + "click>=8.1.3", + "requests>=2.25.0", + "PyYAML>=5.4.1", + "defusedxml>=0.7.1", + "subresource-integrity>=0.2", + "jsonschema>=4.9.1", + "urllib3>=1.25.10", + "semantic-version>=2.10.0", + "oras<=0.2.31", + "python-qpid-proton>=0.39.0" +] + +[project.optional-dependencies] +dev = [ + "pylint", + "flake8", + "pep8", + "mypy", + "tox", +] +test = [ + "flexmock>=0.10.6", + "responses>=0.9.0", + "pytest<=7.1.3", + "pytest-cov", + "pytest-html", + "requests-mock", + "moto>=5.0.16,<6", + "python-gnupg>=0.5.0,<1" +] + +[project.scripts] +charon = "charon.cmd:cli" + +[tool.setuptools] +packages = ["charon"] + +[tool.setuptools_scm] +fallback_version = "1.3.4+dev.fallback" + +[tool.setuptools.package-data] +charon = ["schemas/*.json"] + +[tool.mypy] +python_version = "3.9" + +[tool.coverage.report] +skip_covered = true +show_missing = true +fail_under = 90 +exclude_lines = [ + "def __repr__", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", + "return NotImplemented", +] + +[tool.pytest.ini_options] +log_cli_level = "DEBUG" +log_format = "%(asctime)s %(levelname)s %(message)s" +log_date_format = "%Y-%m-%d %H:%M:%S" +testpaths = [ + "tests", +] + +[tool.flake8] +show_source = true +ignore = [ + "D100", # missing docstring in public module + "D104", # missing docstring in public package + "D105", # missing docstring in magic method + "W503", # line break before binary operator + "E203", # whitespace before ':' + "E501", # line too long + "E731", # do not assign a lambda expression +] +per-file-ignores = [ + "tests/*:D101,D102,D103", # missing docstring in public class, method, function +] diff --git a/requirements-dev.txt b/requirements-dev.txt index f0ed1644..bc38b20f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ -r tests/requirements.txt pyflakes pep8 +tox diff --git a/requirements.txt b/requirements.txt index 7919fc27..d5b5ec75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 +oras<=0.2.31 +python-qpid-proton>=0.39.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 692b53eb..3935d97b 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ """ from setuptools import setup, find_packages -version = "1.3.3" +version = "1.4.0" long_description = """ This charon is a tool to synchronize several types of @@ -57,6 +57,8 @@ "subresource-integrity>=0.2", "jsonschema>=4.9.1", "urllib3>=1.25.10", - "semantic-version>=2.10.0" + "semantic-version>=2.10.0", + "oras<=0.2.31", + "python-qpid-proton>=0.39.0" ], ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 09f63266..408de626 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -3,7 +3,6 @@ responses>=0.9.0 pytest<=7.1.3 pytest-cov pytest-html -flake8 requests-mock moto>=5.0.16,<6 python-gnupg>=0.5.0,<1 diff --git a/tests/test_config_radas.py b/tests/test_config_radas.py new file mode 100644 index 00000000..a6c7d5a4 --- /dev/null +++ b/tests/test_config_radas.py @@ -0,0 +1,226 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import unittest +import os +import charon.config as config +import shutil +import tempfile +from tests.base import BaseTest +from charon.utils.files import overwrite_file + + +class RadasConfigTest(unittest.TestCase): + def setUp(self) -> None: + self.__base = BaseTest() + self.__prepare_ca() + + def tearDown(self) -> None: + self.__base.tearDown() + self.__clear_ca() + + def test_full_radas_config(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + print(radas_settings) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertTrue(rconf.validate()) + + def test_missing_umb_host(self): + radas_settings = """ +radas: + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_result_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_missing_request_queue(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_ca_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_key(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_path) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_client_password_file(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, self.__client_key_pass_file) + os.remove(self.__client_key_pass_file) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def test_unaccessible_root_ca(self): + radas_settings = """ +radas: + umb_host: test.umb.api.com + result_queue: queue.result.test + request_channel: topic://topic.request.test + client_ca: {} + client_key: {} + client_key_pass_file: {} + root_ca: {} + +targets: + ga: + - bucket: charon-test + """.format(self.__client_ca_path, self.__client_key_path, + self.__client_key_pass_file, self.__root_ca) + os.remove(self.__root_ca) + self.__change_config_content(radas_settings) + conf = config.get_config() + self.assertIsNotNone(conf) + rconf = conf.get_radas_config() + self.assertIsNotNone(rconf) + self.assertFalse(rconf.validate()) + + def __change_config_content(self, content: str): + self.__base.change_home() + config_base = self.__base.get_config_base() + os.mkdir(config_base) + self.__base.prepare_config(config_base, content) + + def __prepare_ca(self): + self.__tempdir = tempfile.mkdtemp() + self.__client_ca_path = os.path.join(self.__tempdir, "client_ca.crt") + self.__client_key_path = os.path.join(self.__tempdir, "client_key.crt") + self.__client_key_pass_file = os.path.join(self.__tempdir, "client_key_password.txt") + self.__root_ca = os.path.join(self.__tempdir, "root_ca.crt") + overwrite_file(self.__client_ca_path, "client ca") + overwrite_file(self.__client_key_path, "client key") + overwrite_file(self.__client_key_pass_file, "it's password") + overwrite_file(self.__root_ca, "root ca") + + def __clear_ca(self): + shutil.rmtree(self.__tempdir) diff --git a/tests/test_radas_sign_generation.py b/tests/test_radas_sign_generation.py new file mode 100644 index 00000000..ccc448a2 --- /dev/null +++ b/tests/test_radas_sign_generation.py @@ -0,0 +1,149 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging +import unittest +import tempfile +import os +import json +import shutil +import builtins +from unittest import mock +from charon.utils.files import overwrite_file +from charon.pkgs.radas_sign import generate_radas_sign + +logger = logging.getLogger(__name__) + + +class RadasSignHandlerTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.__prepare_sign_result_file() + + def tearDown(self) -> None: + super().tearDown() + self.__clear_sign_result_file() + + def test_multi_sign_files_generation(self): + self.__prepare_artifacts() + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(len(generated), 2) + self.assertIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + with open(expected_asc1) as f: + content1 = f.read() + with open(expected_asc2) as f: + content2 = f.read() + self.assertIn("signature1@hash", content1) + self.assertIn("signature2@hash", content2) + + def test_sign_files_generation_with_missing_artifacts(self): + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def test_sign_files_generation_with_failure(self): + self.__prepare_artifacts() + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + + # simulate expected_asc1 can not open to write properly + real_open = builtins.open + with mock.patch("builtins.open") as mock_open: + def side_effect(path, *args, **kwargs): + # this is for pylint check + mode = "r" + if len(args) > 0: + mode = args[0] + elif "mode" in kwargs: + mode = kwargs["mode"] + if path == expected_asc1 and "w" in mode: + raise IOError("mock write error") + return real_open(path, *args, **kwargs) + mock_open.side_effect = side_effect + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) + + self.assertEqual(len(failed), 1) + self.assertNotIn(expected_asc1, generated) + self.assertIn(expected_asc2, generated) + + def test_sign_files_generation_with_missing_result(self): + self.__prepare_artifacts() + # simulate missing pull result by removing the sign result file loc + shutil.rmtree(self.__sign_result_loc) + + failed, generated = generate_radas_sign(self.__repo_dir, self.__sign_result_file) + self.assertEqual(failed, []) + expected_asc1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar.asc") + expected_asc2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar.asc") + self.assertEqual(generated, []) + self.assertFalse(os.path.exists(expected_asc1)) + self.assertFalse(os.path.exists(expected_asc2)) + + def __prepare_sign_result_file(self): + self.__sign_result_loc = tempfile.mkdtemp() + self.__sign_result_file = os.path.join(self.__sign_result_loc, "result.json") + self.__repo_dir = os.path.join(tempfile.mkdtemp(), "maven-repository") + data = { + "request-id": "request-id", + "file-reference": "quay.io/org/maven-zip@hash", + "results": [ + { + "file": "maven-repository/foo/bar/1.0/foo-bar-1.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature1@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + { + "file": "maven-repository/foo/bar/2.0/foo-bar-2.0.jar", + "signature": ( + "-----BEGIN PGP SIGNATURE-----" + "signature2@hash" + "-----END PGP SIGNATURE-----" + ), + "checksum": "sha256:sha256-content", + }, + ], + } + json_str = json.dumps(data, indent=2) + overwrite_file(self.__sign_result_file, json_str) + + def __prepare_artifacts(self): + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/1.0"), exist_ok=True) + os.makedirs(os.path.join(self.__repo_dir, "foo/bar/2.0"), exist_ok=True) + artifact1 = os.path.join(self.__repo_dir, "foo/bar/1.0/foo-bar-1.0.jar") + artifact2 = os.path.join(self.__repo_dir, "foo/bar/2.0/foo-bar-2.0.jar") + with open(artifact1, "w") as f: + f.write("dummy1") + with open(artifact2, "w") as f: + f.write("dummy2") + + def __clear_sign_result_file(self): + if os.path.exists(self.__sign_result_loc): + shutil.rmtree(self.__sign_result_loc) + if os.path.exists(self.__repo_dir): + shutil.rmtree(self.__repo_dir) diff --git a/tests/test_radas_sign_receiver.py b/tests/test_radas_sign_receiver.py new file mode 100644 index 00000000..1090c61b --- /dev/null +++ b/tests/test_radas_sign_receiver.py @@ -0,0 +1,151 @@ +from unittest import mock +import unittest +import tempfile +import time +import json +from charon.pkgs.radas_sign import RadasReceiver + + +class RadasSignReceiverTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def reset_receiver(self, r_receiver: RadasReceiver) -> None: + r_receiver._message_handled = False + r_receiver.sign_result_errors = [] + r_receiver.sign_result_status = None + + def test_radas_receiver(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.receiver_timeout.return_value = 60 + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.OrasClient") as oras_client, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + test_result_path = tempfile.mkdtemp() + test_request_id = "test-request-id" + r_receiver = RadasReceiver(test_result_path, test_request_id, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_receiver.sign_result_loc, test_result_path) + self.assertEqual(r_receiver.request_id, test_request_id) + + # prepare mock + mock_receiver = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_receiver.return_value = mock_receiver + event.container = mock_container + event.message = mock.MagicMock() + event.connection = mock.MagicMock() + + # test on_start + r_receiver.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_receiver.call_count, 1) + self.assertTrue(r_receiver._start_time > 0.0) + self.assertTrue(r_receiver._start_time < time.time()) + self.assertEqual(mock_container.schedule.call_count, 1) + + # test on_message: unmatched case + test_ummatch_result = { + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id-no-match", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + } + event.message.body = json.dumps(test_ummatch_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 0) + self.assertEqual(mock_container.stop.call_count, 0) + self.assertFalse(r_receiver._message_handled) + self.assertIsNone(r_receiver.sign_result_status) + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with failed status + self.reset_receiver(r_receiver) + test_failed_result = { + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "failed", + "errors": ["error1", "error2"] + } + } + event.message.body = json.dumps(test_failed_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 1) + self.assertEqual(mock_container.stop.call_count, 1) + self.assertTrue(r_receiver._message_handled) + self.assertEqual(r_receiver.sign_result_status, "failed") + self.assertEqual(r_receiver.sign_result_errors, ["error1", "error2"]) + self.assertEqual(oras_client.call_count, 0) + + # test on_message: matched case with success status + self.reset_receiver(r_receiver) + test_success_result = { + "i": "1", + "msg_id": "test-id", + "timestamp": time.time(), + "topic": "test-topic", + "username": "test-user", + "msg": { + "request_id": "test-request-id", + "file_reference": "quay.io/example/test-repo", + "result_reference": "quay.io/example-sign/sign-repo", + "sig_keyname": "testkey", + "signing_status": "success", + "errors": [] + } + } + event.message.body = json.dumps(test_success_result) + r_receiver.on_message(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertTrue(r_receiver._message_handled) + self.assertEqual(r_receiver.sign_result_status, "success") + self.assertEqual(r_receiver.sign_result_errors, []) + self.assertEqual(oras_client.call_count, 1) + oras_client_call = oras_client.return_value + self.assertEqual(oras_client_call.pull.call_count, 1) + + # test on_timer_task: not timeout + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 2) + self.assertEqual(mock_container.stop.call_count, 2) + self.assertEqual(mock_container.schedule.call_count, 2) + + # test on_timer_task: timeout + mock_radas_config.receiver_timeout.return_value = 0 + r_receiver.on_timer_task(event) + self.assertEqual(event.connection.close.call_count, 3) + self.assertEqual(mock_container.stop.call_count, 3) + self.assertEqual(mock_container.schedule.call_count, 2) diff --git a/tests/test_radas_sign_sender.py b/tests/test_radas_sign_sender.py new file mode 100644 index 00000000..c1c1fee3 --- /dev/null +++ b/tests/test_radas_sign_sender.py @@ -0,0 +1,86 @@ +import json +from unittest import mock +import unittest +from charon.pkgs.radas_sign import RadasSender + + +class RadasSignSenderTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + + def tearDown(self) -> None: + super().tearDown() + + def test_radas_sender(self): + # Mock configuration + mock_radas_config = mock.MagicMock() + mock_radas_config.validate.return_value = True + mock_radas_config.client_ca.return_value = "test-client-ca" + mock_radas_config.client_key.return_value = "test-client-key" + mock_radas_config.client_key_password.return_value = "test-client-key-pass" + mock_radas_config.root_ca.return_value = "test-root-ca" + mock_radas_config.radas_sign_timeout_retry_count.return_value = 5 + + test_payload = { + "request_id": "mock-id", + "requested_by": "test-user", + "type": "mrrc", + "file_reference": "quay.io/test/repo", + "sig_keyname": "test-key", + "exclude": [] + } + + # Mock Container run to avoid real AMQP connection + with mock.patch( + "charon.pkgs.radas_sign.Container") as mock_container, \ + mock.patch("charon.pkgs.radas_sign.SSLDomain") as ssl_domain, \ + mock.patch("charon.pkgs.radas_sign.Event") as event: + + json_payload = json.dumps(test_payload) + r_sender = RadasSender(json_payload, mock_radas_config) + self.assertEqual(ssl_domain.call_count, 1) + self.assertEqual(r_sender.payload, json_payload) + self.assertIs(r_sender.rconf, mock_radas_config) + self.assertIsNone(r_sender._message) + self.assertIsNone(r_sender._pending) + + # test on_start + mock_sender = mock.MagicMock() + mock_conn = mock.MagicMock() + mock_container.connect.return_value = mock_conn + mock_container.create_sender.return_value = mock_sender + event.container = mock_container + r_sender.on_start(event) + self.assertEqual(mock_container.connect.call_count, 1) + self.assertEqual(mock_container.create_sender.call_count, 1) + + # test on_sendable + mock_sender.credit = 1 + r_sender.on_sendable(event) + self.assertIsNotNone(r_sender._message) + self.assertEqual(mock_sender.send.call_count, 1) + + # test on_accepted + r_sender.on_accepted(event) + self.assertEqual(r_sender.status, "success") + self.assertEqual(r_sender._retried, 0) + self.assertEqual(r_sender._sender.close.call_count, 1) + self.assertEqual(r_sender._container.stop.call_count, 1) + + # test on_rejected + r_sender.on_rejected(event) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 1) + self.assertEqual(r_sender._container.schedule.call_count, 1) + + # test on_released + r_sender.on_released(event) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) + self.assertEqual(r_sender._container.schedule.call_count, 2) + + # test on_released + r_sender.on_timer_task(event) + self.assertIsNone(r_sender._pending) + self.assertEqual(r_sender._retried, 2) + self.assertEqual(mock_sender.send.call_count, 2)