From 8ac3a56fe860d253a21f125c66b8667a532e9821 Mon Sep 17 00:00:00 2001 From: Dan McPherson Date: Thu, 26 Sep 2024 18:59:36 -0400 Subject: [PATCH] Remove max_workers and serving_gpus from constructor This change is not backward compatible with callers using max_workers or serving_gpus in the constructor. A minor version bump should be made for the next release. Any callers to max_workers= or serving_gpus= in the constructor should switch to use gen_answers and judge_answers for MTBenchEvaluator and MTBenchBranchEvaluator. Signed-off-by: Dan McPherson --- src/instructlab/eval/mt_bench.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/instructlab/eval/mt_bench.py b/src/instructlab/eval/mt_bench.py index da0d60fe..8607bb3c 100644 --- a/src/instructlab/eval/mt_bench.py +++ b/src/instructlab/eval/mt_bench.py @@ -33,8 +33,6 @@ class AbstractMTBenchEvaluator(Evaluator): model_name Name of the model to evaluate judge_model_name Name of the judge model output_dir The directory to use for evaluation output - max_workers Max parallel workers to run the evaluation with (int or "auto") - serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto. merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges) """ @@ -43,16 +41,12 @@ def __init__( model_name: str, judge_model_name: str, output_dir: str = "eval_output", - max_workers: int | str = "auto", - serving_gpus: int | None = None, merge_system_user_message: bool = False, ) -> None: self.model_name = model_name self.judge_model_name = judge_model_name self.output_dir = output_dir - self.serving_gpus = serving_gpus self.merge_system_user_message = merge_system_user_message - self.max_workers = self._calc_max_workers(max_workers, serving_gpus) def _calc_max_workers( self, max_workers: int | str | None, serving_gpus: int | None @@ -105,8 +99,6 @@ class MTBenchEvaluator(AbstractMTBenchEvaluator): model_name Name of the model to evaluate judge_model_name Name of the judge model output_dir The directory to use for evaluation output - max_workers Max parallel workers to run the evaluation with (int or "auto") - serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto. merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges) """ @@ -180,8 +172,6 @@ class MTBenchBranchEvaluator(AbstractMTBenchEvaluator): taxonomy_git_repo_path Taxonomy git repo path branch Branch of taxonomy repo to eval QNAs against model output_dir The directory to use for evaluation output - max_workers Max parallel workers to run the evaluation with (int or "auto") - serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto. merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges) """ @@ -194,16 +184,12 @@ def __init__( taxonomy_git_repo_path: str, branch: str, output_dir: str = "eval_output", - max_workers: int | str = "auto", - serving_gpus: int | None = None, merge_system_user_message: bool = False, ) -> None: super().__init__( model_name, judge_model_name, output_dir, - max_workers, - serving_gpus, merge_system_user_message, ) self.taxonomy_git_repo_path = taxonomy_git_repo_path