diff --git a/src/instructlab/eval/mt_bench_common.py b/src/instructlab/eval/mt_bench_common.py
index fa334aff..86f99207 100644
--- a/src/instructlab/eval/mt_bench_common.py
+++ b/src/instructlab/eval/mt_bench_common.py
@@ -254,7 +254,7 @@ def chat_completion_openai(
         try:
             messages = conv.to_openai_api_messages()
             if (
-                merge_system_user_message
+                (merge_system_user_message or conv.name == "mistral")
                 and messages[0]["role"] == "system"
                 and messages[1]["role"] == "user"
             ):