Skip to content

Remove max_workers and serving_gpus from constructor #140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mergify merged 1 commit into from
Sep 27, 2024
Merged

Remove max_workers and serving_gpus from constructor #140

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions src/instructlab/eval/mt_bench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ class AbstractMTBenchEvaluator(Evaluator):
model_name Name of the model to evaluate
judge_model_name Name of the judge model
output_dir The directory to use for evaluation output
max_workers Max parallel workers to run the evaluation with (int or "auto")
serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto.
merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges)
"""

Expand All @@ -43,16 +41,12 @@ def __init__(
model_name: str,
judge_model_name: str,
output_dir: str = "eval_output",
max_workers: int | str = "auto",
serving_gpus: int | None = None,
merge_system_user_message: bool = False,
) -> None:
self.model_name = model_name
self.judge_model_name = judge_model_name
self.output_dir = output_dir
self.serving_gpus = serving_gpus
self.merge_system_user_message = merge_system_user_message
self.max_workers = self._calc_max_workers(max_workers, serving_gpus)

def _calc_max_workers(
self, max_workers: int | str | None, serving_gpus: int | None
Expand Down Expand Up @@ -105,8 +99,6 @@ class MTBenchEvaluator(AbstractMTBenchEvaluator):
model_name Name of the model to evaluate
judge_model_name Name of the judge model
output_dir The directory to use for evaluation output
max_workers Max parallel workers to run the evaluation with (int or "auto")
serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto.
merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges)
"""

Expand Down Expand Up @@ -180,8 +172,6 @@ class MTBenchBranchEvaluator(AbstractMTBenchEvaluator):
taxonomy_git_repo_path Taxonomy git repo path
branch Branch of taxonomy repo to eval QNAs against model
output_dir The directory to use for evaluation output
max_workers Max parallel workers to run the evaluation with (int or "auto")
serving_gpus Number of gpus allocated for serving. Used to tune with max_workers=auto.
merge_system_user_message Boolean indicating whether to merge system and user messages (required for Mistral based judges)
"""

Expand All @@ -194,16 +184,12 @@ def __init__(
taxonomy_git_repo_path: str,
branch: str,
output_dir: str = "eval_output",
max_workers: int | str = "auto",
serving_gpus: int | None = None,
merge_system_user_message: bool = False,
) -> None:
super().__init__(
model_name,
judge_model_name,
output_dir,
max_workers,
serving_gpus,
merge_system_user_message,
)
self.taxonomy_git_repo_path = taxonomy_git_repo_path
Expand Down