From 28e4c7ad7305d07696d1a3a4e538fc483254cf78 Mon Sep 17 00:00:00 2001 From: Dan McPherson Date: Mon, 14 Oct 2024 13:00:13 -0400 Subject: [PATCH] Give nice error for empty taxonomy Previously, if the taxonomy didn't contain any skills, an exception would have occurred. This fix gives a nicer message for the user. Signed-off-by: Dan McPherson --- src/instructlab/eval/exceptions.py | 12 ++++++++++++ src/instructlab/eval/mt_bench_branch_generator.py | 9 ++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/instructlab/eval/exceptions.py b/src/instructlab/eval/exceptions.py index fb0c4a82..28b686df 100644 --- a/src/instructlab/eval/exceptions.py +++ b/src/instructlab/eval/exceptions.py @@ -134,3 +134,15 @@ class ModelServingAPIError(EvalError): def __init__(self) -> None: super().__init__() self.message = "Failed to receive a reply from model serving API." + + +class EmptyTaxonomyError(EvalError): + """ + Error raised when taxonomy doesn't contain any skill QNAs + Attributes + message error message to be printed on raise + """ + + def __init__(self) -> None: + super().__init__() + self.message = "Provided taxonomy doesn't contain any skill qna.yaml files" diff --git a/src/instructlab/eval/mt_bench_branch_generator.py b/src/instructlab/eval/mt_bench_branch_generator.py index 76bec8d0..3463653f 100644 --- a/src/instructlab/eval/mt_bench_branch_generator.py +++ b/src/instructlab/eval/mt_bench_branch_generator.py @@ -13,7 +13,12 @@ import yaml # Local -from .exceptions import GitRepoNotFoundError, InvalidGitBranchError, InvalidGitRepoError +from .exceptions import ( + EmptyTaxonomyError, + GitRepoNotFoundError, + InvalidGitBranchError, + InvalidGitRepoError, +) from .logger_config import setup_logger from .mt_bench_common import bench_dir @@ -57,6 +62,8 @@ def generate(judge_model_name, branch, taxonomy_dir, output_dir): taxonomy_repo.git.checkout(branch) qna_file_list = get_file_paths(taxonomy_dir) + if len(qna_file_list) == 0: + raise EmptyTaxonomyError question_lst = [] reference_answers = []