diff --git a/src/instructlab/eval/exceptions.py b/src/instructlab/eval/exceptions.py
index fb0c4a82..28b686df 100644
--- a/src/instructlab/eval/exceptions.py
+++ b/src/instructlab/eval/exceptions.py
@@ -134,3 +134,15 @@ class ModelServingAPIError(EvalError):
     def __init__(self) -> None:
         super().__init__()
         self.message = "Failed to receive a reply from model serving API."
+
+
+class EmptyTaxonomyError(EvalError):
+    """
+    Error raised when taxonomy doesn't contain any skill QNAs
+    Attributes
+        message              error message to be printed on raise
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.message = "Provided taxonomy doesn't contain any skill qna.yaml files"
diff --git a/src/instructlab/eval/mt_bench_branch_generator.py b/src/instructlab/eval/mt_bench_branch_generator.py
index 76bec8d0..3463653f 100644
--- a/src/instructlab/eval/mt_bench_branch_generator.py
+++ b/src/instructlab/eval/mt_bench_branch_generator.py
@@ -13,7 +13,12 @@
 import yaml
 
 # Local
-from .exceptions import GitRepoNotFoundError, InvalidGitBranchError, InvalidGitRepoError
+from .exceptions import (
+    EmptyTaxonomyError,
+    GitRepoNotFoundError,
+    InvalidGitBranchError,
+    InvalidGitRepoError,
+)
 from .logger_config import setup_logger
 from .mt_bench_common import bench_dir
 
@@ -57,6 +62,8 @@ def generate(judge_model_name, branch, taxonomy_dir, output_dir):
             taxonomy_repo.git.checkout(branch)
 
         qna_file_list = get_file_paths(taxonomy_dir)
+        if len(qna_file_list) == 0:
+            raise EmptyTaxonomyError
 
         question_lst = []
         reference_answers = []