From e50a267c0dc4624a4bf0141b5f6e6000f85e2122 Mon Sep 17 00:00:00 2001 From: Guncuke <759398504@qq.com> Date: Sat, 29 Nov 2025 16:15:35 +0800 Subject: [PATCH 1/2] update --- src/lighteval/logging/evaluation_tracker.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 976b21c86..17ea99fee 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -305,7 +305,7 @@ def save_results(self, date_id: str, results_dict: dict): org = org_model_parts[0] if len(org_model_parts) >= 2 else "" model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0] output_dir = self.output_dir - output_dir_results = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) + output_dir_results = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) / "results" else: output_dir_results = Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/") self.fs.mkdirs(output_dir_results, exist_ok=True) @@ -315,7 +315,16 @@ def save_results(self, date_id: str, results_dict: dict): f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) def _get_details_sub_folder(self, date_id: str): - output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/") + if self.results_path_template is not None: + org_model_parts = self.general_config_logger.model_name.split("/") + org = org_model_parts[0] if len(org_model_parts) >= 2 else "" + model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0] + output_dir = self.output_dir + output_dir_details = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) / "details" + return output_dir_details + else: + output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/") + if date_id in ["first", "last"]: # Get all folders in output_dir_details if not self.fs.exists(output_dir_details): @@ -334,7 +343,8 @@ def _get_details_sub_folder(self, date_id: str): def load_details_datasets(self, date_id: str, task_names: list[str]) -> dict[str, Dataset]: output_dir_details_sub_folder = self._get_details_sub_folder(date_id) logger.info(f"Loading details from {output_dir_details_sub_folder}") - date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest + if self.results_path_template is None: + date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest details_datasets = {} for file in self.fs.glob(str(output_dir_details_sub_folder / f"details_*_{date_id}.parquet")): task_name = Path(file).stem.replace("details_", "").replace(f"_{date_id}", "") @@ -747,4 +757,4 @@ def push_to_tensorboard( # noqa: C901 logger.info( f"Pushed to tensorboard at https://huggingface.co/{self.tensorboard_repo}/{output_dir_tb}/tensorboard" f" at global_step {global_step}" - ) + ) \ No newline at end of file From b8fba554d734ed44c839811615cfc04b1cacb4a4 Mon Sep 17 00:00:00 2001 From: Guncuke <759398504@qq.com> Date: Sat, 29 Nov 2025 16:23:37 +0800 Subject: [PATCH 2/2] update --- src/lighteval/logging/evaluation_tracker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py index 17ea99fee..dbddb2a58 100644 --- a/src/lighteval/logging/evaluation_tracker.py +++ b/src/lighteval/logging/evaluation_tracker.py @@ -321,7 +321,6 @@ def _get_details_sub_folder(self, date_id: str): model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0] output_dir = self.output_dir output_dir_details = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) / "details" - return output_dir_details else: output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/") @@ -343,8 +342,7 @@ def _get_details_sub_folder(self, date_id: str): def load_details_datasets(self, date_id: str, task_names: list[str]) -> dict[str, Dataset]: output_dir_details_sub_folder = self._get_details_sub_folder(date_id) logger.info(f"Loading details from {output_dir_details_sub_folder}") - if self.results_path_template is None: - date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest + date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest details_datasets = {} for file in self.fs.glob(str(output_dir_details_sub_folder / f"details_*_{date_id}.parquet")): task_name = Path(file).stem.replace("details_", "").replace(f"_{date_id}", "")