Improve unknown command-line option guidance

src/Platform/Microsoft.Testing.Platform/CommandLine/CommandLineOptionsValidator.UnknownAndBootstrapValidation.cs

@@ -10,6 +10,105 @@ namespace Microsoft.Testing.Platform.CommandLine;
 
 internal static partial class CommandLineOptionsValidator
 {
+    // Keep in sync with public command-line providers that are automatically registered by the listed packages.
+    // Packages without public options, options requiring explicit framework registration, and hidden internal options
+    // are intentionally omitted.
+    private static readonly Dictionary<string, string[]> KnownExtensionOptionsByPackage = new(StringComparer.OrdinalIgnoreCase)
+    {
+        ["Microsoft.Testing.Extensions.AzureDevOpsReport"] =
+        [
+            "publish-azdo-run-name",
+            "publish-azdo-test-results",
+            "report-azdo",
+            "report-azdo-annotations",
+            "report-azdo-demote-known-flaky",
+            "report-azdo-flaky-history",
+            "report-azdo-groups",
+            "report-azdo-quarantine-file",
+            "report-azdo-severity",
+            "report-azdo-slow-test-history",
+            "report-azdo-slow-test-history-min-sample",
+            "report-azdo-slow-test-history-multiplier",
+            "report-azdo-stackframe-filter",
+            "report-azdo-summary",
+            "report-azdo-upload-artifact-exclude",
+            "report-azdo-upload-artifact-include",
+            "report-azdo-upload-artifact-name",
+            "report-azdo-upload-artifacts",
+        ],
+        ["Microsoft.Testing.Extensions.CodeCoverage"] =
+        [
+            "coverage",
+            "coverage-output",
+            "coverage-output-format",
+            "coverage-settings",
+        ],
+        ["Microsoft.Testing.Extensions.CrashDump"] =
+        [
+            "crash-report",
+            "crash-report-if-supported",
+            "crash-sequence",
+            "crashdump",
+            "crashdump-filename",
+            "crashdump-type",
+        ],
+        ["Microsoft.Testing.Extensions.CtrfReport"] =
+        [
+            "report-ctrf",
+            "report-ctrf-filename",
+        ],
+        ["Microsoft.Testing.Extensions.GitHubActionsReport"] =
+        [
+            "report-gh",
+            "report-gh-annotations",
+            "report-gh-failure-details",
+            "report-gh-groups",
+            "report-gh-slow-test-notices",
+            "report-gh-slow-test-threshold",
+            "report-gh-step-summary",
+            "report-gh-step-summary-sections",
+        ],
+        ["Microsoft.Testing.Extensions.HangDump"] =
+        [
+            "hangdump",
+            "hangdump-filename",
+            "hangdump-timeout",
+            "hangdump-type",
+            "hangdump-type-if-supported",
+        ],
+        ["Microsoft.Testing.Extensions.HtmlReport"] =
+        [
+            "report-html",
+            "report-html-filename",
+        ],
+        ["Microsoft.Testing.Extensions.JUnitReport"] =
+        [
+            "report-junit",
+            "report-junit-filename",
+        ],
+        ["Microsoft.Testing.Extensions.Retry"] =
+        [
+            "retry-failed-tests",
+            "retry-failed-tests-delay",
+            "retry-failed-tests-max-percentage",
+            "retry-failed-tests-max-tests",
+        ],
+        ["Microsoft.Testing.Extensions.TrxReport"] =
+        [
+            "report-trx",
+            "report-trx-filename",
+        ],
+        ["Microsoft.Testing.Extensions.VideoRecorder"] =
+        [
+            "capture-video",
+            "capture-video-args",
+            "capture-video-chapters",
+            "capture-video-granularity",
+            "capture-video-max-duration",
+            "capture-video-source",
+        ],
+    };
+
     private static ValidationResult ValidateNoUnknownOptions(
         CommandLineParseResult parseResult,
         IReadOnlyList<JsonCommandLineOptionEntry>? jsonCommandLineOptions,
@@ -21,11 +120,17 @@ private static ValidationResult ValidateNoUnknownOptions(
         // parsing is already case-sensitive but a case-insensitive lookup is a strict superset and
         // does not change CLI behavior.
         var validOptionNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        var visibleOptionNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+        bool includeKnownExtensionOptions = !parseResult.HasTool;
         foreach (KeyValuePair<ICommandLineOptionsProvider, IReadOnlyCollection<CommandLineOption>> provider in extensionOptionsByProvider)
         {
             foreach (CommandLineOption option in provider.Value)
             {
                 validOptionNames.Add(option.Name);
+                if (!option.IsHidden)
+                {
+                    visibleOptionNames.Add(option.Name);
+                }
             }
         }
 
@@ -34,6 +139,10 @@ private static ValidationResult ValidateNoUnknownOptions(
             foreach (CommandLineOption option in provider.Value)
             {
                 validOptionNames.Add(option.Name);
+                if (!option.IsHidden)
+                {
+                    visibleOptionNames.Add(option.Name);
+                }
             }
         }
 
@@ -43,7 +152,7 @@ private static ValidationResult ValidateNoUnknownOptions(
             if (!validOptionNames.Contains(optionRecord.Name))
             {
                 stringBuilder ??= new();
-                stringBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, PlatformResources.CommandLineUnknownOption, optionRecord.Name));
+                AppendUnknownOptionError(stringBuilder, optionRecord.Name, validOptionNames, visibleOptionNames, includeKnownExtensionOptions);
             }
         }
 
@@ -58,7 +167,9 @@ private static ValidationResult ValidateNoUnknownOptions(
                 if (!validOptionNames.Contains(entry.OptionName))
                 {
                     stringBuilder ??= new();
-                    string innerError = string.Format(CultureInfo.InvariantCulture, PlatformResources.CommandLineUnknownOption, entry.OptionName);
+                    StringBuilder innerErrorBuilder = new();
+                    AppendUnknownOptionError(innerErrorBuilder, entry.OptionName, validOptionNames, visibleOptionNames, includeKnownExtensionOptions);
+                    string innerError = innerErrorBuilder.ToTrimmedString();
                     stringBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, PlatformResources.JsonCommandLineOptionsValidationErrorPrefix, innerError));
                 }
             }
@@ -74,6 +185,136 @@ private static ValidationResult ValidateNoUnknownOptions(
             : ValidationResult.Valid();
     }
 
+    private static void AppendUnknownOptionError(
+        StringBuilder stringBuilder,
+        string unknownOptionName,
+        HashSet<string> validOptionNames,
+        HashSet<string> visibleOptionNames,
+        bool includeKnownExtensionOptions)
+    {
+        stringBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, PlatformResources.CommandLineUnknownOption, unknownOptionName));
+
+        if (includeKnownExtensionOptions
+            && GetKnownExtensionPackage(unknownOptionName) is { } packageName)
+        {
+            AppendMissingExtensionSuggestion(stringBuilder, unknownOptionName, packageName);
+            return;
+        }
+
+        IEnumerable<string> candidateOptionNames = includeKnownExtensionOptions
+            ? visibleOptionNames.Concat(KnownExtensionOptionsByPackage.Values.SelectMany(static optionNames => optionNames))
+            : visibleOptionNames;
+        string? suggestedOptionName = FindSuggestedOption(
+            unknownOptionName,
+            candidateOptionNames);
+        if (suggestedOptionName is null)
+        {
+            return;
+        }
+
+        stringBuilder.AppendLine(string.Format(CultureInfo.InvariantCulture, PlatformResources.CommandLineOptionSuggestion, suggestedOptionName));
+
+        if (includeKnownExtensionOptions
+            && GetKnownExtensionPackage(suggestedOptionName) is { } suggestedPackageName
+            && !validOptionNames.Contains(suggestedOptionName))
+        {
+            AppendMissingExtensionSuggestion(stringBuilder, suggestedOptionName, suggestedPackageName);
+        }
+    }
+
+    private static void AppendMissingExtensionSuggestion(StringBuilder stringBuilder, string optionName, string packageName)
+        => stringBuilder.AppendLine(string.Format(
+            CultureInfo.InvariantCulture,
+            PlatformResources.CommandLineOptionRequiresExtension,
+            optionName,
+            packageName));
+
+    private static string? GetKnownExtensionPackage(string optionName)
+    {
+        foreach (KeyValuePair<string, string[]> extensionOptions in KnownExtensionOptionsByPackage)
+        {
+            if (extensionOptions.Value.Contains(optionName, StringComparer.OrdinalIgnoreCase))
+            {
+                return extensionOptions.Key;
+            }
+        }
+
+        return null;
+    }
+
+    private static string? FindSuggestedOption(string unknownOptionName, IEnumerable<string> candidateOptionNames)
+    {
+        int maximumDistance = unknownOptionName.Length switch
+        {
+            <= 4 => 1,
+            <= 12 => 2,
+            _ => 3,
+        };
+
+        string? bestCandidate = null;
+        int bestDistance = maximumDistance + 1;
+        bool hasAmbiguousBestCandidate = false;
+
+        foreach (string candidateOptionName in candidateOptionNames.Distinct(StringComparer.OrdinalIgnoreCase))
+        {
+            if (Math.Abs(candidateOptionName.Length - unknownOptionName.Length) > maximumDistance)
+            {
+                continue;
+            }
+
+            int distance = CalculateEditDistance(unknownOptionName, candidateOptionName);
+            if (distance < bestDistance)
+            {
+                bestCandidate = candidateOptionName;
+                bestDistance = distance;
+                hasAmbiguousBestCandidate = false;
+            }
+            else if (distance == bestDistance)
+            {
+                hasAmbiguousBestCandidate = true;
+            }
+        }
+
+        return bestDistance <= maximumDistance && !hasAmbiguousBestCandidate
+            ? bestCandidate
+            : null;
+    }
+
+    private static int CalculateEditDistance(string source, string target)
+    {
+        int[,] distances = new int[source.Length + 1, target.Length + 1];
+        for (int i = 0; i <= source.Length; i++)
+        {
+            distances[i, 0] = i;
+        }
+
+        for (int j = 0; j <= target.Length; j++)
+        {
+            distances[0, j] = j;
+        }
+
+        for (int i = 1; i <= source.Length; i++)
+        {
+            for (int j = 1; j <= target.Length; j++)
+            {
+                int substitutionCost = char.ToUpperInvariant(source[i - 1]) == char.ToUpperInvariant(target[j - 1]) ? 0 : 1;
+                distances[i, j] = Math.Min(
+                    Math.Min(distances[i - 1, j] + 1, distances[i, j - 1] + 1),
+                    distances[i - 1, j - 1] + substitutionCost);
+
+                if (i > 1
+                    && j > 1
+                    && char.ToUpperInvariant(source[i - 1]) == char.ToUpperInvariant(target[j - 2])
+                    && char.ToUpperInvariant(source[i - 2]) == char.ToUpperInvariant(target[j - 1]))
+                {
+                    distances[i, j] = Math.Min(distances[i, j], distances[i - 2, j - 2] + 1);
+                }
+            }
+        }
+
+        return distances[source.Length, target.Length];
+    }
+
     private static ValidationResult ValidateNoBootstrapOnlyOptionsInJson(
         IReadOnlyList<JsonCommandLineOptionEntry>? jsonCommandLineOptions)
     {

src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx

@@ -467,10 +467,18 @@
     <value>Unknown option '--{0}'</value>
     <comment>{0} is the unknown option name without the leading dashes.</comment>
   </data>
+  <data name="CommandLineOptionSuggestion" xml:space="preserve">
+    <value>Did you mean '--{0}'?</value>
+    <comment>{0} is the suggested option name without the leading dashes.</comment>
+  </data>
   <data name="CommandLineUnknownOptionsHint" xml:space="preserve">
     <value>Run '--help' to see the options registered by this test application. If the option belongs to an extension, ensure its package is referenced and the extension is registered.</value>
     <comment>{Locked="'--help'"}</comment>
   </data>
+  <data name="CommandLineOptionRequiresExtension" xml:space="preserve">
+    <value>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</value>
+    <comment>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</comment>
+  </data>
   <data name="InvalidCommandLineArguments" xml:space="preserve">
     <value>Invalid command line arguments:</value>
   </data>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf

@@ -222,6 +222,16 @@
         <target state="translated">Možnost --{0} od zprostředkovatele {1} (UID: {2}) používá vyhrazenou předponu --internal.</target>
         <note>{0} is the option name without the leading dashes. {1} is the provider display name. {2} is the provider UID. {Locked="--internal"}</note>
       </trans-unit>
+      <trans-unit id="CommandLineOptionRequiresExtension">
+        <source>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</source>
+        <target state="new">Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</target>
+        <note>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</note>
+      </trans-unit>
+      <trans-unit id="CommandLineOptionSuggestion">
+        <source>Did you mean '--{0}'?</source>
+        <target state="new">Did you mean '--{0}'?</target>
+        <note>{0} is the suggested option name without the leading dashes.</note>
+      </trans-unit>
       <trans-unit id="CommandLineOptionsNotReady">
         <source>The ICommandLineOptions has not been built yet.</source>
         <target state="translated">Rozhraní ICommandLineOptions ještě není sestavené.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf

@@ -222,6 +222,16 @@
         <target state="translated">Die Option "--{0}" vom Anbieter "{1}" (UID: {2}) verwendet das reservierte Präfix "--internal".</target>
         <note>{0} is the option name without the leading dashes. {1} is the provider display name. {2} is the provider UID. {Locked="--internal"}</note>
       </trans-unit>
+      <trans-unit id="CommandLineOptionRequiresExtension">
+        <source>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</source>
+        <target state="new">Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</target>
+        <note>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</note>
+      </trans-unit>
+      <trans-unit id="CommandLineOptionSuggestion">
+        <source>Did you mean '--{0}'?</source>
+        <target state="new">Did you mean '--{0}'?</target>
+        <note>{0} is the suggested option name without the leading dashes.</note>
+      </trans-unit>
       <trans-unit id="CommandLineOptionsNotReady">
         <source>The ICommandLineOptions has not been built yet.</source>
         <target state="translated">ICommandLineOptions wurde noch nicht erstellt.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf

@@ -222,6 +222,16 @@
         <target state="translated">La opción “--{0}” del proveedor “{1}” (UID: {2}) usa el prefijo reservado “--internal”</target>
         <note>{0} is the option name without the leading dashes. {1} is the provider display name. {2} is the provider UID. {Locked="--internal"}</note>
       </trans-unit>
+      <trans-unit id="CommandLineOptionRequiresExtension">
+        <source>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</source>
+        <target state="new">Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</target>
+        <note>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</note>
+      </trans-unit>
+      <trans-unit id="CommandLineOptionSuggestion">
+        <source>Did you mean '--{0}'?</source>
+        <target state="new">Did you mean '--{0}'?</target>
+        <note>{0} is the suggested option name without the leading dashes.</note>
+      </trans-unit>
       <trans-unit id="CommandLineOptionsNotReady">
         <source>The ICommandLineOptions has not been built yet.</source>
         <target state="translated">ICommandLineOptions aún no se ha compilado.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf

@@ -222,6 +222,16 @@
         <target state="translated">L’option « --{0} » du fournisseur « {1} » (UID : {2}) utilise le préfixe réservé « --internal »</target>
         <note>{0} is the option name without the leading dashes. {1} is the provider display name. {2} is the provider UID. {Locked="--internal"}</note>
       </trans-unit>
+      <trans-unit id="CommandLineOptionRequiresExtension">
+        <source>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</source>
+        <target state="new">Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</target>
+        <note>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</note>
+      </trans-unit>
+      <trans-unit id="CommandLineOptionSuggestion">
+        <source>Did you mean '--{0}'?</source>
+        <target state="new">Did you mean '--{0}'?</target>
+        <note>{0} is the suggested option name without the leading dashes.</note>
+      </trans-unit>
       <trans-unit id="CommandLineOptionsNotReady">
         <source>The ICommandLineOptions has not been built yet.</source>
         <target state="translated">ICommandLineOptions n’a pas encore été généré.</target>

src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf

@@ -222,6 +222,16 @@
         <target state="translated">L'opzione '--{0}' del provider '{1}' (UID: {2}) usa il prefisso riservato '--internal'</target>
         <note>{0} is the option name without the leading dashes. {1} is the provider display name. {2} is the provider UID. {Locked="--internal"}</note>
       </trans-unit>
+      <trans-unit id="CommandLineOptionRequiresExtension">
+        <source>Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</source>
+        <target state="new">Option '--{0}' is provided by the '{1}' extension. Add a package reference to use it.</target>
+        <note>{0} is the option name without the leading dashes. {1} is the extension's NuGet package name.</note>
+      </trans-unit>
+      <trans-unit id="CommandLineOptionSuggestion">
+        <source>Did you mean '--{0}'?</source>
+        <target state="new">Did you mean '--{0}'?</target>
+        <note>{0} is the suggested option name without the leading dashes.</note>
+      </trans-unit>
       <trans-unit id="CommandLineOptionsNotReady">
         <source>The ICommandLineOptions has not been built yet.</source>
         <target state="translated">ICommandLineOptions non è stato ancora compilato.</target>