From e322b4eb6344c7fdc7c3f9a7561ca7e39546ca61 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 11:48:00 +0800 Subject: [PATCH 01/11] Feat: Support to accept the response of signing result from RADAS --- charon/constants.py | 3 + charon/pkgs/maven.py | 35 ++++- charon/pkgs/oras_client.py | 64 ++++++++ charon/pkgs/radas_signature_handler.py | 210 +++++++++++++++++++++++++ requirements.txt | 2 + 5 files changed, 309 insertions(+), 5 deletions(-) create mode 100644 charon/pkgs/oras_client.py create mode 100644 charon/pkgs/radas_signature_handler.py diff --git a/charon/constants.py b/charon/constants.py index 6751aecd..4c8320db 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,3 +175,6 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" +DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" +DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 9f50f35b..900cc4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -16,6 +16,7 @@ from charon.utils.files import HashType import charon.pkgs.indexing as indexing import charon.pkgs.signature as signature +import charon.pkgs.radas_signature_handler as radas_signature from charon.utils.files import overwrite_file, digest, write_manifest from charon.utils.archive import extract_zip_all from charon.utils.strings import remove_prefix @@ -408,11 +409,35 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if contain_signature is set to True - if gen_sign: - conf = get_config(config) - if not conf: - sys.exit(1) + # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + conf = get_config(config) + if not conf: + sys.exit(1) + + if conf.get_radas_sign_enabled(): + logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + if not _generated_signs: + logger.error( + "No sign result files were downloaded, " + "please make sure the sign process is already done and without timeout") + return (tmp_root, False) + + failed_metas.extend(_failed_metas) + generated_signs.extend(_generated_signs) + logger.info("Singature generation against radas done.\n") + + logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + _failed_metas = s3_client.upload_signatures( + meta_file_paths=generated_signs, + target=(bucket_name, prefix), + product=None, + root=top_level + ) + failed_metas.extend(_failed_metas) + logger.info("Signature files uploading against radas done.\n") + + elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) command = conf.get_detach_signature_command() artifacts = [s for s in valid_mvn_paths if not s.endswith(tuple(suffix_list))] diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py new file mode 100644 index 00000000..fca62006 --- /dev/null +++ b/charon/pkgs/oras_client.py @@ -0,0 +1,64 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import oras.client +import logging +from charon.config import get_config +from typing import List + +logger = logging.getLogger(__name__) + +class OrasClient: + """ + Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. + """ + + def __init__(self): + self.conf = get_config() + self.client = oras.client.OrasClient() + + def login_if_needed(self) -> None: + """ + If quay_radas_auth_enabled is true, call login to authenticate. + """ + + if self.conf and self.conf.is_quay_radas_auth_enabled(): + logger.info("Logging in to registry.") + res = self.client.login( + hostname=self.conf.get_quay_radas_registry(), + username=self.conf.get_quay_radas_username(), + password=self.conf.get_quay_radas_password(), + ) + logger.info(res) + else: + logger.info("Registry auth not enabled, skip login.") + + def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: + """ + Call oras‑py’s pull method to pull the remote file to local. + Args: + result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + """ + files = [] + try: + self.login_if_needed() + # the filename should be possibly named by the digest hash value based on the oras source code + files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) + logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) + except Exception as e: + logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) + finally: + return files \ No newline at end of file diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py new file mode 100644 index 00000000..5bacdab7 --- /dev/null +++ b/charon/pkgs/radas_signature_handler.py @@ -0,0 +1,210 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import proton +import proton.handlers +import threading +import logging +import json +import os +import asyncio +from typing import List, Any, Tuple, Callable, Dict +from charon.config import get_config +from charon.constants import DEFAULT_SIGN_RESULT_LOC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.pkgs.oras_client import OrasClient + +logger = logging.getLogger(__name__) + +class SignHandler: + """ + Handle the sign result status management + """ + _is_processing: bool = True + _downloaded_files: List[str] = [] + + @classmethod + def is_processing(cls) -> bool: + return cls._is_processing + + @classmethod + def get_downloaded_files(cls) -> List[str]: + return cls._downloaded_files.copy() + + @classmethod + def set_processing(cls, value: bool) -> None: + cls._is_processing = value + + @classmethod + def set_downloaded_files(cls, files: List[str]) -> None: + cls._downloaded_files = files + +class UmbListener(proton.handlers.MessagingHandler): + """ + UmbListener class (AMQP version), register this when setup UmbClient + """ + + def __init__(self) -> None: + super().__init__() + + def on_start(self, event: proton.Event) -> None: + """ + On start callback + """ + conf = get_config() + if not conf: + sys.exit(1) + event.container.create_receiver(conf.get_amqp_queue()) + + def on_message(self, event: proton.Event) -> None: + """ + On message callback + """ + # handle response from radas in a thread + thread = threading.Thread( + target=self._process_message, + args=[event.message.body] + ) + thread.start() + + def on_error(self, event: proton.Event) -> None: + """ + On error callback + """ + logger.error("Received an error event:\n%s", event) + + def on_disconnected(self, event: proton.Event) -> None: + """ + On disconnected callback + """ + logger.error("Disconnected from AMQP broker.") + + def _process_message(msg: Any) -> None: + """ + Process a message received from UMB + Args: + msg: The message body received + """ + try: + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") + + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return + + conf = get_config() + if not conf: + sign_result_loc = DEFAULT_SIGN_RESULT_LOC + sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() + logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + + sign_result_parent_dir = os.path.dirname(sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) + + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, + sign_result_loc=sign_result_loc + ) + SignHandler.set_downloaded_files(files) + finally: + SignHandler.set_processing(False) + +def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: + """ + Generate .asc files based on RADAS sign result json file + """ + conf = get_config() + timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + wait_count = 0 + while SignHandler.is_processing(): + wait_count += 1 + if wait_count > timeout_count: + logger.warning("Timeout when waiting for sign response.") + break + time.sleep(wait_interval_sec) + + files = SignHandler.get_downloaded_files() + if not files: + return [], [] + + # should only have the single sign result json file from the radas registry + json_file_path = files[0] + try: + with open(json_file_path, 'r') as f: + data = json.load(f) + except Exception as e: + logger.error(f"Failed to read or parse the JSON file: {e}") + raise + + async def generate_single_sign_file( + file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], + sem: asyncio.BoundedSemaphore + ): + async with sem: + if not file_path or not signature: + logger.error(f"Invalid JSON entry") + return + # remove the root path maven-repository + filename = file_path.split("/", 1)[1] + signature = item.get("signature") + + artifact_path = os.path.join(top_level, filename) + asc_filename = f"{filename}.asc" + signature_path = os.path.join(top_level, asc_filename) + + if not os.path.isfile(artifact_path): + logger.warning("Artifact missing, skip signature file generation") + return + + try: + with open(signature_path, 'w') as asc_file: + asc_file.write(signature) + generated_signs.append(signature_path) + logger.info(f"Generated .asc file: {signature_path}") + except Exception as e: + failed_paths.append(signature_path) + logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + + result = data.get("result", []) + return __do_path_cut_and( + path_handler=generate_single_sign_file, + data=result + ) + +def __do_path_cut_and( + path_handler: Callable, + data: List[Dict[str, str]] +) -> Tuple[List[str], List[str]]: + + failed_paths: List[str] = [] + generated_signs: List[str] = [] + tasks = [] + sem = asyncio.BoundedSemaphore(10) + for item in data: + file_path = item.get("file") + signature = item.get("signature") + tasks.append( + asyncio.ensure_future( + path_handler(file_path, signature, failed_paths, generated_signs, sem) + ) + ) + + loop = asyncio.get_event_loop() + loop.run_until_complete(asyncio.gather(*tasks)) + return (failed_paths, generated_signs) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7919fc27..75bb4b60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,5 @@ subresource-integrity>=0.2 jsonschema>=4.9.1 urllib3>=1.25.10 semantic-version>=2.10.0 +oras>=0.2.31 +python-qpid-proton>=0.39.0 \ No newline at end of file From 07e43f37ed067d2853865bb0c7eb491b895f0a70 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 13 May 2025 13:31:44 +0800 Subject: [PATCH 02/11] Format Code and fix typo --- charon/config.py | 6 +-- charon/constants.py | 2 +- charon/pkgs/maven.py | 5 ++- charon/pkgs/oras_client.py | 16 ++++--- charon/pkgs/radas_signature_handler.py | 58 ++++++++++++++------------ 5 files changed, 48 insertions(+), 39 deletions(-) diff --git a/charon/config.py b/charon/config.py index 35efe6eb..c9cbdc59 100644 --- a/charon/config.py +++ b/charon/config.py @@ -141,14 +141,12 @@ def get_config(cfgPath=None) -> CharonConfig: config_file_path = cfgPath if not config_file_path or not os.path.isfile(config_file_path): config_file_path = os.path.join(os.getenv("HOME", ""), ".charon", CONFIG_FILE) - data = read_yaml_from_file_path(config_file_path, 'schemas/charon.json') + data = read_yaml_from_file_path(config_file_path, "schemas/charon.json") return CharonConfig(data) def get_template(template_file: str) -> str: - template = os.path.join( - os.getenv("HOME", ''), ".charon/template", template_file - ) + template = os.path.join(os.getenv("HOME", ""), ".charon/template", template_file) if os.path.isfile(template): with open(template, encoding="utf-8") as file_: return file_.read() diff --git a/charon/constants.py b/charon/constants.py index 4c8320db..c8b8d125 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -177,4 +177,4 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 \ No newline at end of file +DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 900cc4cb..8c14d822 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -409,12 +409,13 @@ def handle_maven_uploading( if cf_enable: cf_invalidate_paths.extend(archetype_files) - # 10. Generate signature file if radas sign is enabled, or do detached sign if contain_signature is set to True + # 10. Generate signature file if radas sign is enabled, + # or do detached sign if contain_signature is set to True conf = get_config(config) if not conf: sys.exit(1) - if conf.get_radas_sign_enabled(): + if conf.is_radas_sign_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) if not _generated_signs: diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index fca62006..95310215 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import oras.client import logging from charon.config import get_config @@ -20,6 +21,7 @@ logger = logging.getLogger(__name__) + class OrasClient: """ Wrapper for oras‑py’s OrasClient, deciding whether to login based on config. @@ -49,16 +51,18 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ Call oras‑py’s pull method to pull the remote file to local. Args: - result_reference_url (str): Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). - sign_result_loc (str): Local save path (e.g. “/tmp/sign”). + result_reference_url (str): + Reference of the remote file (e.g. “quay.io/repository/signing/radas@hash”). + sign_result_loc (str): + Local save path (e.g. “/tmp/sign”). """ files = [] try: self.login_if_needed() - # the filename should be possibly named by the digest hash value based on the oras source code files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: - logger.error("Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e) - finally: - return files \ No newline at end of file + logger.error( + "Failed to pull file from %s to %s: %s", result_reference_url, sign_result_loc, e + ) + return files diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 5bacdab7..9120bc9b 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import proton import proton.handlers import threading @@ -20,6 +21,8 @@ import json import os import asyncio +import sys +import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC @@ -29,10 +32,12 @@ logger = logging.getLogger(__name__) + class SignHandler: """ Handle the sign result status management """ + _is_processing: bool = True _downloaded_files: List[str] = [] @@ -52,6 +57,7 @@ def set_processing(cls, value: bool) -> None: def set_downloaded_files(cls, files: List[str]) -> None: cls._downloaded_files = files + class UmbListener(proton.handlers.MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient @@ -74,10 +80,7 @@ def on_message(self, event: proton.Event) -> None: On message callback """ # handle response from radas in a thread - thread = threading.Thread( - target=self._process_message, - args=[event.message.body] - ) + thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() def on_error(self, event: proton.Event) -> None: @@ -103,8 +106,8 @@ def _process_message(msg: Any) -> None: result_reference_url = msg_dict.get("result_reference") if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return + logger.warning("Not found result_reference in message，ignore.") + return conf = get_config() if not conf: @@ -117,20 +120,24 @@ def _process_message(msg: Any) -> None: oras_client = OrasClient() files = oras_client.pull( - result_reference_url=result_reference_url, - sign_result_loc=sign_result_loc + result_reference_url=result_reference_url, sign_result_loc=sign_result_loc ) SignHandler.set_downloaded_files(files) finally: SignHandler.set_processing(False) + def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT - wait_interval_sec = conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_count = ( + conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + ) + wait_interval_sec = ( + conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 @@ -146,23 +153,25 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: # should only have the single sign result json file from the radas registry json_file_path = files[0] try: - with open(json_file_path, 'r') as f: + with open(json_file_path, "r") as f: data = json.load(f) except Exception as e: - logger.error(f"Failed to read or parse the JSON file: {e}") + logger.error("Failed to read or parse the JSON file: %s", e) raise async def generate_single_sign_file( - file_path: str, signature: str, failed_paths: List[str], generated_signs: List[str], - sem: asyncio.BoundedSemaphore + file_path: str, + signature: str, + failed_paths: List[str], + generated_signs: List[str], + sem: asyncio.BoundedSemaphore, ): async with sem: if not file_path or not signature: - logger.error(f"Invalid JSON entry") + logger.error("Invalid JSON entry") return # remove the root path maven-repository filename = file_path.split("/", 1)[1] - signature = item.get("signature") artifact_path = os.path.join(top_level, filename) asc_filename = f"{filename}.asc" @@ -173,23 +182,20 @@ async def generate_single_sign_file( return try: - with open(signature_path, 'w') as asc_file: + with open(signature_path, "w") as asc_file: asc_file.write(signature) generated_signs.append(signature_path) - logger.info(f"Generated .asc file: {signature_path}") + logger.info("Generated .asc file: %s", signature_path) except Exception as e: failed_paths.append(signature_path) - logger.error(f"Failed to write .asc file for {artifact_path}: {e}") + logger.error("Failed to write .asc file for %s: %s", artifact_path, e) result = data.get("result", []) - return __do_path_cut_and( - path_handler=generate_single_sign_file, - data=result - ) + return __do_path_cut_and(path_handler=generate_single_sign_file, data=result) + def __do_path_cut_and( - path_handler: Callable, - data: List[Dict[str, str]] + path_handler: Callable, data: List[Dict[str, str]] ) -> Tuple[List[str], List[str]]: failed_paths: List[str] = [] @@ -207,4 +213,4 @@ def __do_path_cut_and( loop = asyncio.get_event_loop() loop.run_until_complete(asyncio.gather(*tasks)) - return (failed_paths, generated_signs) \ No newline at end of file + return (failed_paths, generated_signs) From 42dbb01f8711362807985c0b9495bb7126ea0dc9 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 15:12:26 +0800 Subject: [PATCH 03/11] Rename radas_sign_timeout_retry_count and radas_sign_timeout_retry_interval --- charon/config.py | 15 +++++++++++++-- charon/constants.py | 4 ++-- charon/pkgs/radas_signature_handler.py | 19 +++++++++++-------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/charon/config.py b/charon/config.py index c9cbdc59..b6eb07cd 100644 --- a/charon/config.py +++ b/charon/config.py @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. """ + import logging import os from typing import Dict, List, Optional @@ -34,6 +35,10 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) + self.__radas_sign_timeout_retry_interval: int = data.get( + "radas_sign_timeout_retry_interval", 60 + ) def validate(self) -> bool: if not self.__umb_host: @@ -60,7 +65,7 @@ def validate(self) -> bool: return True def umb_target(self) -> str: - return f'amqps://{self.__umb_host}:{self.__umb_host_port}' + return f"amqps://{self.__umb_host}:{self.__umb_host_port}" def result_queue(self) -> str: return self.__result_queue @@ -77,7 +82,7 @@ def client_key(self) -> str: def client_key_password(self) -> str: pass_file = self.__client_key_pass_file if os.access(pass_file, os.R_OK): - with open(pass_file, 'r') as f: + with open(pass_file, "r") as f: return f.read() elif pass_file: logger.warning("The key password file is not accessible. Will ignore the password.") @@ -86,6 +91,12 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def radas_sign_timeout_retry_count(self) -> int: + return self.__radas_sign_timeout_retry_count + + def radas_sign_timeout_retry_interval(self) -> int: + return self.__radas_sign_timeout_retry_interval + class CharonConfig(object): """CharonConfig is used to store all configurations for charon diff --git a/charon/constants.py b/charon/constants.py index c8b8d125..25970c36 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -176,5 +176,5 @@ DEFAULT_REGISTRY = "localhost" DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" -DEFAULT_RADAS_SIGN_TIMEOUT_COUNT = 10 -DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC = 60 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 +DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9120bc9b..eb755086 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -26,8 +26,8 @@ from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config from charon.constants import DEFAULT_SIGN_RESULT_LOC -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT +from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient logger = logging.getLogger(__name__) @@ -132,19 +132,22 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: Generate .asc files based on RADAS sign result json file """ conf = get_config() - timeout_count = ( - conf.get_radas_sign_timeout_count() if conf else DEFAULT_RADAS_SIGN_TIMEOUT_COUNT + rconf = conf.get_radas_config() if conf else None + timeout_retry_count = ( + rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT ) - wait_interval_sec = ( - conf.get_radas_sign_wait_interval_sec() if conf else DEFAULT_RADAS_SIGN_WAIT_INTERVAL_SEC + timeout_retry_interval = ( + rconf.radas_sign_timeout_retry_interval() + if conf + else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 while SignHandler.is_processing(): wait_count += 1 - if wait_count > timeout_count: + if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break - time.sleep(wait_interval_sec) + time.sleep(timeout_retry_interval) files = SignHandler.get_downloaded_files() if not files: From 89be82f359f724903f5d759530bf5399f20e8882 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 19 May 2025 21:00:36 +0800 Subject: [PATCH 04/11] Use oras registry_config and registry url parse to finalize login --- charon/config.py | 11 +++++++++++ charon/pkgs/oras_client.py | 22 +++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/charon/config.py b/charon/config.py index b6eb07cd..fe3c1174 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,6 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") + self.__quay_radas_registry_config: str = data.get( + "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -62,6 +65,11 @@ def validate(self) -> bool: if self.__root_ca and not os.access(self.__root_ca, os.R_OK): logger.error("The root ca file is not valid!") return False + if self.__quay_radas_registry_config and not os.access( + self.__quay_radas_registry_config, os.R_OK + ): + logger.error("The quay registry config for oras is not valid!") + return False return True def umb_target(self) -> str: @@ -91,6 +99,9 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca + def quay_radas_registry_config(self) -> str: + return self.__quay_radas_registry_config + def radas_sign_timeout_retry_count(self) -> int: return self.__radas_sign_timeout_retry_count diff --git a/charon/pkgs/oras_client.py b/charon/pkgs/oras_client.py index 95310215..b5446def 100644 --- a/charon/pkgs/oras_client.py +++ b/charon/pkgs/oras_client.py @@ -18,6 +18,7 @@ import logging from charon.config import get_config from typing import List +from urllib.parse import urlparse logger = logging.getLogger(__name__) @@ -31,21 +32,24 @@ def __init__(self): self.conf = get_config() self.client = oras.client.OrasClient() - def login_if_needed(self) -> None: + def login_if_needed(self, registry: str) -> None: """ - If quay_radas_auth_enabled is true, call login to authenticate. + If quay_radas_registry_config is provided, call login to authenticate. """ + if not registry.startswith("http://") and not registry.startswith("https://"): + registry = "https://" + registry + registry = urlparse(registry).netloc - if self.conf and self.conf.is_quay_radas_auth_enabled(): - logger.info("Logging in to registry.") + rconf = self.conf.get_radas_config() if self.conf else None + if rconf and rconf.quay_radas_registry_config(): + logger.info("Logging in to registry: %s", registry) res = self.client.login( - hostname=self.conf.get_quay_radas_registry(), - username=self.conf.get_quay_radas_username(), - password=self.conf.get_quay_radas_password(), + hostname=registry, + config_path=rconf.quay_radas_registry_config(), ) logger.info(res) else: - logger.info("Registry auth not enabled, skip login.") + logger.info("Registry config is not provided, skip login.") def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ @@ -58,7 +62,7 @@ def pull(self, result_reference_url: str, sign_result_loc: str) -> List[str]: """ files = [] try: - self.login_if_needed() + self.login_if_needed(registry=result_reference_url) files = self.client.pull(target=result_reference_url, outdir=sign_result_loc) logger.info("Pull file from %s to %s", result_reference_url, sign_result_loc) except Exception as e: From d4eee680354d62fab7caca8f283426cbf345bd5e Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 09:32:24 +0800 Subject: [PATCH 05/11] Change sign_result_loc to cmd flag both for sign request and maven upload --- charon/cmd/cmd_upload.py | 16 +++- charon/config.py | 7 +- charon/constants.py | 1 - charon/pkgs/maven.py | 15 ++-- charon/pkgs/radas_signature_handler.py | 112 +++++++++++-------------- 5 files changed, 78 insertions(+), 73 deletions(-) diff --git a/charon/cmd/cmd_upload.py b/charon/cmd/cmd_upload.py index a867df01..d56a644d 100644 --- a/charon/cmd/cmd_upload.py +++ b/charon/cmd/cmd_upload.py @@ -136,6 +136,16 @@ default=False ) @option("--dryrun", "-n", is_flag=True, default=False) +@option( + "--sign_result_loc", + "-l", + default="/tmp/sign", + help=""" + The local save path for oras to pull the radas signature result. + Sign request will use this path to download the signature result, + Upload will use the file on this path to generate the corresponding .asc files + """, +) @command() def upload( repo: str, @@ -150,7 +160,8 @@ def upload( sign_key: str = "redhatdevel", debug=False, quiet=False, - dryrun=False + dryrun=False, + sign_result_loc="/tmp/sign" ): """Upload all files from a released product REPO to Ronda Service. The REPO points to a product released tarball which @@ -221,7 +232,8 @@ def upload( key=sign_key, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name, - config=config + config=config, + sign_result_loc=sign_result_loc ) if not succeeded: sys.exit(1) diff --git a/charon/config.py b/charon/config.py index fe3c1174..ce418b33 100644 --- a/charon/config.py +++ b/charon/config.py @@ -48,7 +48,7 @@ def validate(self) -> bool: logger.error("Missing host name setting for UMB!") return False if not self.__result_queue: - logger.error("Missing the queue setting to receive siging result in UMB!") + logger.error("Missing the queue setting to receive signing result in UMB!") return False if not self.__request_queue: logger.error("Missing the queue setting to send signing request in UMB!") @@ -124,8 +124,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) + self.__radas_config_enable: bool = data.get("radas_config_enable", False) radas_config: Dict = data.get("radas", None) if radas_config: + self.__radas_config_enable = True self.__radas_config__: RadasConfig = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: @@ -155,6 +157,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_config_enable(self) -> bool: + return self.__radas_config_enable + def get_radas_config(self) -> RadasConfig: return self.__radas_config__ diff --git a/charon/constants.py b/charon/constants.py index 25970c36..35ea560a 100644 --- a/charon/constants.py +++ b/charon/constants.py @@ -175,6 +175,5 @@ DEFAULT_ERRORS_LOG = "errors.log" DEFAULT_REGISTRY = "localhost" -DEFAULT_SIGN_RESULT_LOC = "/tmp/sign" DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT = 10 DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL = 60 diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 8c14d822..dde1a4cb 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -275,7 +275,8 @@ def handle_maven_uploading( key=None, dry_run=False, manifest_bucket_name=None, - config=None + config=None, + sign_result_loc="/tmp/sign" ) -> Tuple[str, bool]: """ Handle the maven product release tarball uploading process. * repo is the location of the tarball in filesystem @@ -415,9 +416,11 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_sign_enabled(): + if conf.is_radas_config_enable(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) - (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign(top_level) + (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( + top_level=top_level, sign_result_loc=sign_result_loc + ) if not _generated_signs: logger.error( "No sign result files were downloaded, " @@ -426,9 +429,9 @@ def handle_maven_uploading( failed_metas.extend(_failed_metas) generated_signs.extend(_generated_signs) - logger.info("Singature generation against radas done.\n") + logger.info("Radas signature files generation done.\n") - logger.info("Start upload radas singature files to s3 bucket %s\n", bucket_name) + logger.info("Start upload radas signature files to s3 bucket %s\n", bucket_name) _failed_metas = s3_client.upload_signatures( meta_file_paths=generated_signs, target=(bucket_name, prefix), @@ -436,7 +439,7 @@ def handle_maven_uploading( root=top_level ) failed_metas.extend(_failed_metas) - logger.info("Signature files uploading against radas done.\n") + logger.info("Radas signature files uploading done.\n") elif gen_sign: suffix_list = __get_suffix(PACKAGE_TYPE_MAVEN, conf) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index eb755086..9ef00d8f 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,8 +14,6 @@ limitations under the License. """ -import proton -import proton.handlers import threading import logging import json @@ -25,57 +23,40 @@ import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_SIGN_RESULT_LOC from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient +from proton import Event +from proton.handlers import MessagingHandler logger = logging.getLogger(__name__) -class SignHandler: - """ - Handle the sign result status management - """ - - _is_processing: bool = True - _downloaded_files: List[str] = [] - - @classmethod - def is_processing(cls) -> bool: - return cls._is_processing - - @classmethod - def get_downloaded_files(cls) -> List[str]: - return cls._downloaded_files.copy() - - @classmethod - def set_processing(cls, value: bool) -> None: - cls._is_processing = value - - @classmethod - def set_downloaded_files(cls, files: List[str]) -> None: - cls._downloaded_files = files - - -class UmbListener(proton.handlers.MessagingHandler): +class UmbListener(MessagingHandler): """ UmbListener class (AMQP version), register this when setup UmbClient + Attributes: + sign_result_loc (str): Local save path (e.g. “/tmp/sign”) for oras pull result, + this value transfers from the cmd flag, should register UmbListener when the client starts """ - def __init__(self) -> None: + def __init__(self, sign_result_loc: str) -> None: super().__init__() + self.sign_result_loc = sign_result_loc - def on_start(self, event: proton.Event) -> None: + def on_start(self, event: Event) -> None: """ On start callback """ conf = get_config() - if not conf: + rconf = conf.get_radas_config() if conf else None + if not rconf: sys.exit(1) - event.container.create_receiver(conf.get_amqp_queue()) + conn = event.container.connect(rconf.umb_target()) + event.container.create_receiver(conn, rconf.result_queue()) + logger.info("Listening on %s, queue: %s", rconf.umb_target(), rconf.result_queue()) - def on_message(self, event: proton.Event) -> None: + def on_message(self, event: Event) -> None: """ On message callback """ @@ -83,51 +64,43 @@ def on_message(self, event: proton.Event) -> None: thread = threading.Thread(target=self._process_message, args=[event.message.body]) thread.start() - def on_error(self, event: proton.Event) -> None: + def on_connection_error(self, event: Event) -> None: """ - On error callback + On connection error callback """ logger.error("Received an error event:\n%s", event) - def on_disconnected(self, event: proton.Event) -> None: + def on_disconnected(self, event: Event) -> None: """ On disconnected callback """ logger.error("Disconnected from AMQP broker.") - def _process_message(msg: Any) -> None: + def _process_message(self, msg: Any) -> None: """ Process a message received from UMB Args: msg: The message body received """ - try: - msg_dict = json.loads(msg) - result_reference_url = msg_dict.get("result_reference") + msg_dict = json.loads(msg) + result_reference_url = msg_dict.get("result_reference") - if not result_reference_url: - logger.warning("Not found result_reference in message，ignore.") - return - - conf = get_config() - if not conf: - sign_result_loc = DEFAULT_SIGN_RESULT_LOC - sign_result_loc = os.getenv("SIGN_RESULT_LOC") or conf.get_sign_result_loc() - logger.info("Using SIGN RESULT LOC: %s", sign_result_loc) + if not result_reference_url: + logger.warning("Not found result_reference in message，ignore.") + return - sign_result_parent_dir = os.path.dirname(sign_result_loc) - os.makedirs(sign_result_parent_dir, exist_ok=True) + logger.info("Using SIGN RESULT LOC: %s", self.sign_result_loc) + sign_result_parent_dir = os.path.dirname(self.sign_result_loc) + os.makedirs(sign_result_parent_dir, exist_ok=True) - oras_client = OrasClient() - files = oras_client.pull( - result_reference_url=result_reference_url, sign_result_loc=sign_result_loc - ) - SignHandler.set_downloaded_files(files) - finally: - SignHandler.set_processing(False) + oras_client = OrasClient() + files = oras_client.pull( + result_reference_url=result_reference_url, sign_result_loc=self.sign_result_loc + ) + logger.info("Number of files pulled: %d, path: %s", len(files), files[0]) -def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: +def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str], List[str]]: """ Generate .asc files based on RADAS sign result json file """ @@ -138,21 +111,34 @@ def generate_radas_sign(top_level: str) -> Tuple[List[str], List[str]]: ) timeout_retry_interval = ( rconf.radas_sign_timeout_retry_interval() - if conf + if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL ) wait_count = 0 - while SignHandler.is_processing(): + + # Wait until files appear in the sign_result_loc directory + while True: + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] + if files: # If files exist, break the loop + break + wait_count += 1 if wait_count > timeout_retry_count: logger.warning("Timeout when waiting for sign response.") break time.sleep(timeout_retry_interval) - files = SignHandler.get_downloaded_files() if not files: return [], [] + if len(files) > 1: + logger.error("Multiple files found in %s. Expected only one file.", sign_result_loc) + return [], [] + # should only have the single sign result json file from the radas registry json_file_path = files[0] try: From d2e86c969fa5bd23e244e9d7e8999258e6d152a7 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 10:22:18 +0800 Subject: [PATCH 06/11] Update quay_radas_registry_config to default None since it's not necessary for public --- charon/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/charon/config.py b/charon/config.py index ce418b33..ca238e63 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,9 +35,7 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get( - "quay_radas_registry_config", os.path.join(os.getenv("HOME", ""), ".oras/config.json") - ) + self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 From 75740e5a952156109734fe75a6f15477f71fa174 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 11:12:06 +0800 Subject: [PATCH 07/11] Use radas_config validate instead of is_radas_config_enable option --- charon/config.py | 10 +++------- charon/pkgs/maven.py | 3 ++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/charon/config.py b/charon/config.py index ca238e63..a53bdaa6 100644 --- a/charon/config.py +++ b/charon/config.py @@ -122,11 +122,10 @@ def __init__(self, data: Dict): self.__ignore_signature_suffix: Dict = data.get("ignore_signature_suffix", None) self.__signature_command: str = data.get("detach_signature_command", None) self.__aws_cf_enable: bool = data.get("aws_cf_enable", False) - self.__radas_config_enable: bool = data.get("radas_config_enable", False) + self.__radas_config__: Optional[RadasConfig] = None radas_config: Dict = data.get("radas", None) if radas_config: - self.__radas_config_enable = True - self.__radas_config__: RadasConfig = RadasConfig(radas_config) + self.__radas_config__ = RadasConfig(radas_config) def get_ignore_patterns(self) -> List[str]: return self.__ignore_patterns @@ -155,10 +154,7 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable - def is_radas_config_enable(self) -> bool: - return self.__radas_config_enable - - def get_radas_config(self) -> RadasConfig: + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index dde1a4cb..3971fd06 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,7 +416,8 @@ def handle_maven_uploading( if not conf: sys.exit(1) - if conf.is_radas_config_enable(): + rconf = conf.get_radas_config() + if rconf and rconf.validate(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc From 947f23ce5434e0131c32e3825e8f566eca2f9753 Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 14:12:41 +0800 Subject: [PATCH 08/11] Ignore the registry config if the provided config path is not valid to read --- charon/config.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index a53bdaa6..5e72f270 100644 --- a/charon/config.py +++ b/charon/config.py @@ -35,7 +35,9 @@ def __init__(self, data: Dict): self.__client_key: str = data.get("client_key", None) self.__client_key_pass_file: str = data.get("client_key_pass_file", None) self.__root_ca: str = data.get("root_ca", "/etc/pki/tls/certs/ca-bundle.crt") - self.__quay_radas_registry_config: str = data.get("quay_radas_registry_config", None) + self.__quay_radas_registry_config: Optional[str] = data.get( + "quay_radas_registry_config", None + ) self.__radas_sign_timeout_retry_count: int = data.get("radas_sign_timeout_retry_count", 10) self.__radas_sign_timeout_retry_interval: int = data.get( "radas_sign_timeout_retry_interval", 60 @@ -66,8 +68,10 @@ def validate(self) -> bool: if self.__quay_radas_registry_config and not os.access( self.__quay_radas_registry_config, os.R_OK ): - logger.error("The quay registry config for oras is not valid!") - return False + self.__quay_radas_registry_config = None + logger.warning( + "The quay registry config for oras is not valid, will ignore the registry config!" + ) return True def umb_target(self) -> str: @@ -97,7 +101,7 @@ def client_key_password(self) -> str: def root_ca(self) -> str: return self.__root_ca - def quay_radas_registry_config(self) -> str: + def quay_radas_registry_config(self) -> Optional[str]: return self.__quay_radas_registry_config def radas_sign_timeout_retry_count(self) -> int: From da803ddddc2d62b6d443fb2f7effbb9f19165c5e Mon Sep 17 00:00:00 2001 From: yma Date: Tue, 20 May 2025 15:43:39 +0800 Subject: [PATCH 09/11] Add is_radas_enabled unified method to check radas enablement --- charon/config.py | 3 +++ charon/pkgs/maven.py | 3 +-- charon/pkgs/radas_signature_handler.py | 8 ++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/charon/config.py b/charon/config.py index 5e72f270..2995ffdf 100644 --- a/charon/config.py +++ b/charon/config.py @@ -158,6 +158,9 @@ def get_detach_signature_command(self) -> str: def is_aws_cf_enable(self) -> bool: return self.__aws_cf_enable + def is_radas_enabled(self) -> bool: + return bool(self.__radas_config__ and self.__radas_config__.validate()) + def get_radas_config(self) -> Optional[RadasConfig]: return self.__radas_config__ diff --git a/charon/pkgs/maven.py b/charon/pkgs/maven.py index 3971fd06..4ca1be0d 100644 --- a/charon/pkgs/maven.py +++ b/charon/pkgs/maven.py @@ -416,8 +416,7 @@ def handle_maven_uploading( if not conf: sys.exit(1) - rconf = conf.get_radas_config() - if rconf and rconf.validate(): + if conf.is_radas_enabled(): logger.info("Start generating radas signature files for s3 bucket %s\n", bucket_name) (_failed_metas, _generated_signs) = radas_signature.generate_radas_sign( top_level=top_level, sign_result_loc=sign_result_loc diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 9ef00d8f..27a14548 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -49,8 +49,12 @@ def on_start(self, event: Event) -> None: On start callback """ conf = get_config() - rconf = conf.get_radas_config() if conf else None - if not rconf: + if not (conf and conf.is_radas_enabled()): + sys.exit(1) + + rconf = conf.get_radas_config() + # explicit check to pass the type checker + if rconf is None: sys.exit(1) conn = event.container.connect(rconf.umb_target()) event.container.create_receiver(conn, rconf.result_queue()) From f784b1627df28806ba9762fd91d6bc9bd5718cea Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 15:20:27 +0800 Subject: [PATCH 10/11] Change on_message process method without using threads --- charon/pkgs/radas_signature_handler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index 27a14548..a62a0d96 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -14,7 +14,6 @@ limitations under the License. """ -import threading import logging import json import os @@ -64,9 +63,7 @@ def on_message(self, event: Event) -> None: """ On message callback """ - # handle response from radas in a thread - thread = threading.Thread(target=self._process_message, args=[event.message.body]) - thread.start() + self._process_message(event.message.body) def on_connection_error(self, event: Event) -> None: """ From 83a92f99ea1b529df329c835e738c47dd960afda Mon Sep 17 00:00:00 2001 From: yma Date: Wed, 21 May 2025 17:29:44 +0800 Subject: [PATCH 11/11] Remove timeout retry handling for sign result fetch from maven upload --- charon/pkgs/radas_signature_handler.py | 35 ++++---------------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/charon/pkgs/radas_signature_handler.py b/charon/pkgs/radas_signature_handler.py index a62a0d96..c04f0bbf 100644 --- a/charon/pkgs/radas_signature_handler.py +++ b/charon/pkgs/radas_signature_handler.py @@ -19,11 +19,8 @@ import os import asyncio import sys -import time from typing import List, Any, Tuple, Callable, Dict from charon.config import get_config -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT -from charon.constants import DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL from charon.pkgs.oras_client import OrasClient from proton import Event from proton.handlers import MessagingHandler @@ -105,33 +102,11 @@ def generate_radas_sign(top_level: str, sign_result_loc: str) -> Tuple[List[str] """ Generate .asc files based on RADAS sign result json file """ - conf = get_config() - rconf = conf.get_radas_config() if conf else None - timeout_retry_count = ( - rconf.radas_sign_timeout_retry_count() if rconf else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_COUNT - ) - timeout_retry_interval = ( - rconf.radas_sign_timeout_retry_interval() - if rconf - else DEFAULT_RADAS_SIGN_TIMEOUT_RETRY_INTERVAL - ) - wait_count = 0 - - # Wait until files appear in the sign_result_loc directory - while True: - files = [ - os.path.join(sign_result_loc, f) - for f in os.listdir(sign_result_loc) - if os.path.isfile(os.path.join(sign_result_loc, f)) - ] - if files: # If files exist, break the loop - break - - wait_count += 1 - if wait_count > timeout_retry_count: - logger.warning("Timeout when waiting for sign response.") - break - time.sleep(timeout_retry_interval) + files = [ + os.path.join(sign_result_loc, f) + for f in os.listdir(sign_result_loc) + if os.path.isfile(os.path.join(sign_result_loc, f)) + ] if not files: return [], []