From a54c52c6d3be70df008934542c1be2ea1e624674 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 15:56:19 -0400 Subject: [PATCH 1/4] feat: add evidence-backed configuration onboarding workflow Signed-off-by: Tom Main --- README.md | 45 +- cmd/internationalizer/commands.go | 120 ++++ cmd/internationalizer/config.go | 182 ++++++ cmd/internationalizer/detect.go | 184 +++++- cmd/internationalizer/json.go | 209 ++++++ cmd/internationalizer/json_test.go | 154 +++++ cmd/internationalizer/main.go | 20 +- cmd/internationalizer/onboarding_test.go | 63 ++ cmd/internationalizer/schema.go | 197 ++++++ cmd/internationalizer/schema_test.go | 112 ++++ cmd/internationalizer/translate.go | 88 ++- cmd/internationalizer/validate.go | 107 +++- docs/cli-onboarding.md | 172 +++++ internal/onboarding/discovery.go | 564 ++++++++++++++++ internal/onboarding/discovery_test.go | 329 ++++++++++ internal/onboarding/plan.go | 748 ++++++++++++++++++++++ internal/onboarding/plan_test.go | 295 +++++++++ internal/translate/translate.go | 46 +- internal/validate/syntax.go | 28 +- internal/validate/syntax_guidance_test.go | 22 + internal/validate/validate.go | 2 +- test/acceptance/acceptance_test.go | 6 +- test/acceptance/onboarding_test.go | 158 +++++ 23 files changed, 3772 insertions(+), 79 deletions(-) create mode 100644 cmd/internationalizer/commands.go create mode 100644 cmd/internationalizer/config.go create mode 100644 cmd/internationalizer/json.go create mode 100644 cmd/internationalizer/json_test.go create mode 100644 cmd/internationalizer/onboarding_test.go create mode 100644 cmd/internationalizer/schema.go create mode 100644 cmd/internationalizer/schema_test.go create mode 100644 docs/cli-onboarding.md create mode 100644 internal/onboarding/discovery.go create mode 100644 internal/onboarding/discovery_test.go create mode 100644 internal/onboarding/plan.go create mode 100644 internal/onboarding/plan_test.go create mode 100644 internal/validate/syntax_guidance_test.go create mode 100644 test/acceptance/onboarding_test.go diff --git a/README.md b/README.md index ab0b8ba..bd92247 100644 --- a/README.md +++ b/README.md @@ -220,13 +220,44 @@ Human and JSON reports use stable finding codes: ### `detect` -Auto-detect the i18n framework and suggest a configuration. +Inspect existing configuration and discover catalogs, including nested apps. ```bash internationalizer detect +internationalizer detect --json +internationalizer config check --json ``` -Supports: react-i18next, next-intl, vue-i18n, vanilla JSON, markdown docs. +Discovery reports runtime evidence, uncovered catalogs, and unresolved choices. +i18next dependencies suggest `message_syntax: i18next`; ICU integration evidence +requires a runtime decision. JSON is a storage format, not a message grammar. + +### `config plan` and `config apply` + +Create a proposal, review its diff, then explicitly apply the saved plan: + +```bash +internationalizer config plan --json \ + --add-bundle web=exf-app/web/src/i18n/locales/en.json \ + --syntax web=i18next --syntax default=i18next \ + --confirm-source tmp/english-keys.json --out config-plan.json +internationalizer config apply --plan config-plan.json --no-input --json +internationalizer translate --dry-run --json +``` + +This example assumes an existing `source_path: tmp/english-keys.json` marketing +config. Select paths and syntax for your own runtime; discovery does not decide +which artifacts ship. Plan/apply preserves existing provider settings, locale +overrides, glossary paths, and bundle IDs. It rejects stale plans and recognizes +an already-applied configuration. `--no-input` disables prompts; it does not +authorize additional actions. + +See [the onboarding and JSON contract](docs/cli-onboarding.md) for initial setup, +filters, error codes, and retry behavior. `internationalizer commands --json` +describes installed workflow entry points and their effects. + +JSON compatibility: `validate --json` now emits a `schema_version: 1` envelope. +Consumers of its previous array output must read `data.reports` instead. ### `glossary` @@ -490,9 +521,13 @@ documents receive markers on their next successful update. `internationalizer detect` identifies your i18n setup by checking: -- `package.json` dependencies for react-i18next, next-intl, or vue-i18n -- Directory structures matching common locale patterns -- File extensions and naming conventions +- Existing configured bundles and source-locale filenames/directories +- Nested `package.json` dependencies and localization-module ICU references +- Runtime evidence separately from file format, with uncertainty made explicit + +Scanning is bounded and excludes dependency, build, hidden, and data directories. +Dynamic plugin registration and unconventional catalog paths need explicit +configuration; static detection cannot prove that ICU integration is absent. ## Architecture diff --git a/cmd/internationalizer/commands.go b/cmd/internationalizer/commands.go new file mode 100644 index 0000000..b692922 --- /dev/null +++ b/cmd/internationalizer/commands.go @@ -0,0 +1,120 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +type commandContract struct { + Argv []string `json:"argv"` + Description string `json:"description"` + Arguments []flagContract `json:"arguments"` + SideEffects []string `json:"side_effects"` + Network string `json:"network"` + InputSchema map[string]any `json:"input_schema,omitempty"` + OutputSchema map[string]any `json:"output_schema,omitempty"` + OutputFormat string `json:"output_format"` + Next [][]string `json:"next,omitempty"` +} +type flagContract struct { + Name string `json:"name"` + Type string `json:"type"` + Default string `json:"default"` + Description string `json:"description"` +} + +func newCommandsCmd(root *cobra.Command) *cobra.Command { + var asJSON bool + var selected string + var limit int + cmd := &cobra.Command{Use: "commands", Short: "Describe installed workflow entry points, arguments, and effects", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + if limit < 0 { + return codedError("invalid_arguments", fmt.Errorf("limit must be nonnegative")) + } + contracts := []commandContract{} + var visit func(*cobra.Command) + visit = func(c *cobra.Command) { + if c.Hidden { + return + } + if c.RunE != nil || c.Run != nil { + path := c.CommandPath() + item := commandContract{Argv: strings.Fields(path), Description: c.Short, Arguments: []flagContract{}, SideEffects: []string{}, Network: "none", OutputFormat: "human-readable"} + c.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Hidden { + item.Arguments = append(item.Arguments, flagContract{Name: "--" + f.Name, Type: f.Value.Type(), Default: f.DefValue, Description: f.Usage}) + } + }) + switch strings.TrimPrefix(path, root.Name()+" ") { + case "detect": + item.Next = [][]string{{root.Name(), "config", "check", "--json"}, {root.Name(), "config", "plan", "--help"}} + case "config check": + item.Next = [][]string{{root.Name(), "config", "plan", "--help"}, {root.Name(), "translate", "--dry-run", "--json"}} + case "config plan": + item.SideEffects = []string{"writes a new plan file only when --out is supplied"} + item.Next = [][]string{{root.Name(), "config", "apply", "--help"}} + case "config apply": + item.SideEffects = []string{"writes only the selected plan's configuration file after integrity and drift checks"} + item.Next = [][]string{{root.Name(), "config", "check", "--json"}, {root.Name(), "translate", "--dry-run", "--json"}} + case "translate": + item.Network = "provider calls except --dry-run and --adopt-existing" + item.SideEffects = []string{"may write catalogs, translation memory, and state; --dry-run writes nothing"} + case "validate": + item.Next = [][]string{{root.Name(), "review", "--help"}} + case "commands": + default: + item.SideEffects = []string{"mode-dependent; inspect command help before execution"} + item.Network = "mode-dependent; inspect command help" + } + if c.Flags().Lookup("json") != nil { + switch strings.TrimPrefix(path, root.Name()+" ") { + case "detect", "commands", "config check", "config plan", "config apply", "translate", "validate": + item.OutputFormat = "internationalizer.cli.v1 (--json)" + item.InputSchema = workflowInputSchema(c) + item.OutputSchema = workflowOutputSchema(path) + default: + item.OutputFormat = "command-specific JSON (--json)" + } + } + contracts = append(contracts, item) + } + for _, child := range c.Commands() { + visit(child) + } + } + visit(root) + total := len(contracts) + if selected != "" { + filtered := contracts[:0] + for _, item := range contracts { + if strings.Join(item.Argv[1:], " ") == selected { + filtered = append(filtered, item) + } + } + contracts = filtered + if len(contracts) == 0 { + return codedError("invalid_arguments", fmt.Errorf("unknown command selection %q", selected)) + } + } + matched := len(contracts) + if limit > 0 && len(contracts) > limit { + contracts = contracts[:limit] + } + if asJSON { + return emitJSON(cmd, "ok", map[string]any{"cli_version": version, "commands": contracts, "total": total, "matched": matched, "truncated": len(contracts) < matched, "exit_codes": map[string]string{"0": "command completed; inspect status and diagnostics for unresolved decisions", "1": "command failed or validation blocked"}, "states": []string{"configuration_checked", "planned", "applied", "generated", "structurally_valid", "human_approved"}, "authorization": "config apply is an explicit mutation request; --no-input disables prompts only"}, nil) + } + for _, c := range contracts { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s — %s\n", strings.Join(c.Argv, " "), c.Description); err != nil { + return err + } + } + return nil + }} + cmd.Flags().BoolVar(&asJSON, "json", false, "Emit versioned machine-readable command contracts") + cmd.Flags().StringVar(&selected, "command", "", "Return one command contract, for example 'config plan'") + cmd.Flags().IntVar(&limit, "limit", 50, "Maximum command contracts (0 for all)") + return cmd +} diff --git a/cmd/internationalizer/config.go b/cmd/internationalizer/config.go new file mode 100644 index 0000000..0682fac --- /dev/null +++ b/cmd/internationalizer/config.go @@ -0,0 +1,182 @@ +package main + +import ( + "encoding/json" + "fmt" + "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/message" + "github.com/Tom-R-Main/Internationalizer/internal/onboarding" + "github.com/spf13/cobra" + "os" + "sort" + "strings" +) + +func newConfigCmd() *cobra.Command { + cmd := &cobra.Command{Use: "config", Short: "Inspect, plan, and explicitly apply project configuration"} + f := &inspectionFlags{} + check := &cobra.Command{Use: "check", Short: "Check resolved configuration offline without changing files", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return runInspection(cmd, f, true) }} + f.bind(check) + cmd.AddCommand(check, newConfigPlanCmd(), newConfigApplyCmd()) + return cmd +} + +func assignments(values []string) (map[string]string, error) { + out := map[string]string{} + for _, value := range values { + key, v, ok := strings.Cut(value, "=") + if !ok || key == "" || v == "" { + return nil, fmt.Errorf("expected ID=value, got %q", value) + } + if _, exists := out[key]; exists { + return nil, fmt.Errorf("duplicate decision for %q", key) + } + out[key] = v + } + return out, nil +} + +func newConfigPlanCmd() *cobra.Command { + var path, out, sourceLocale string + var additions, syntaxes, targets, confirm, locales []string + var asJSON bool + cmd := &cobra.Command{Use: "plan", Short: "Propose a reviewable config change; never apply it", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + adds, err := assignments(additions) + if err != nil { + return err + } + modes, err := assignments(syntaxes) + if err != nil { + return err + } + targetMap, err := assignments(targets) + if err != nil { + return err + } + opts := onboarding.PlanOptions{Syntax: map[string]message.Syntax{}, ConfirmSources: confirm, SourceLocale: sourceLocale, TargetLocales: locales} + for id, mode := range modes { + opts.Syntax[id] = message.Syntax(mode) + } + for id := range targetMap { + if _, ok := adds[id]; !ok { + return fmt.Errorf("--target %s requires --add-bundle %s=source", id, id) + } + } + if len(adds) > 0 { + report, scanErr := onboarding.Scan(".", path) + if scanErr != nil { + return scanErr + } + ids := make([]string, 0, len(adds)) + for id := range adds { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + source := adds[id] + b := config.Bundle{ID: id, Source: source, Target: targetMap[id], MessageSyntax: opts.Syntax[id]} + for _, c := range report.Candidates { + if c.ID == source || c.Source == source { + b.Source = c.Source + b.Format = c.Format + if b.Target == "" { + b.Target = c.Target + } + break + } + } + if b.Target == "" { + return fmt.Errorf("source %q is not a discovered catalog; supply --target %s=path/{locale}.json", source, id) + } + opts.AddBundles = append(opts.AddBundles, b) + } + } + plan, err := onboarding.BuildPlan(".", path, opts) + if err != nil { + return err + } + if out != "" { + data, marshalErr := json.MarshalIndent(plan, "", " ") + if marshalErr != nil { + return marshalErr + } + file, openErr := os.OpenFile(out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if openErr != nil { + return fmt.Errorf("save plan (will not overwrite): %w", openErr) + } + _, writeErr := file.Write(append(data, '\n')) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + } + status := "planned" + if len(plan.RequiredDecisions) > 0 { + status = "needs_decision" + } + if asJSON { + return emitJSON(cmd, status, plan, nil) + } + if _, err := fmt.Fprintln(cmd.OutOrStdout(), plan.Diff); err != nil { + return err + } + for _, d := range plan.RequiredDecisions { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", d.Code, d.Message); err != nil { + return err + } + } + if out != "" { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Saved plan to %s. Review it, then: config apply --plan %s --no-input\n", out, out); err != nil { + return err + } + } + return nil + }} + cmd.Flags().StringVar(&path, "config", "", "Configuration path") + cmd.Flags().StringVar(&out, "out", "", "Save plan to a new file (never overwrites)") + cmd.Flags().StringArrayVar(&additions, "add-bundle", nil, "Explicit bundle ID=discovered-source-path (repeatable)") + cmd.Flags().StringArrayVar(&syntaxes, "syntax", nil, "Explicit bundle ID=plain|i18next|icu|auto (repeatable)") + cmd.Flags().StringArrayVar(&targets, "target", nil, "Target override for added bundle ID=path/{locale}.json") + cmd.Flags().StringArrayVar(&confirm, "confirm-source", nil, "Confirm authoritative source path, including tmp/ (repeatable)") + cmd.Flags().StringVar(&sourceLocale, "source-locale", "", "Explicit source locale (existing setting preserved when omitted)") + cmd.Flags().StringArrayVar(&locales, "locale", nil, "Explicit target locale set (repeatable; existing set preserved when omitted)") + cmd.Flags().BoolVar(&asJSON, "json", false, "Emit versioned JSON") + return cmd +} + +func newConfigApplyCmd() *cobra.Command { + var path string + var asJSON, noInput bool + cmd := &cobra.Command{Use: "apply", Short: "Apply an explicitly selected saved plan after drift checks", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { + if path == "" { + return fmt.Errorf("--plan is required; --no-input only disables prompts") + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + var plan onboarding.ConfigPlan + if err = json.Unmarshal(data, &plan); err != nil { + return fmt.Errorf("invalid plan: %w", err) + } + receipt, err := onboarding.ApplyPlan(&plan) + if asJSON { + status := "error" + if err == nil { + status = receipt.Status + } + return emitJSON(cmd, status, receipt, err) + } + if err != nil { + return err + } + return json.NewEncoder(cmd.OutOrStdout()).Encode(receipt) + }} + cmd.Flags().StringVar(&path, "plan", "", "Saved config plan to apply (explicit mutation request)") + cmd.Flags().BoolVar(&noInput, "no-input", false, "Disable prompting; does not authorize any additional action") + cmd.Flags().BoolVar(&asJSON, "json", false, "Emit versioned JSON receipt") + return cmd +} diff --git a/cmd/internationalizer/detect.go b/cmd/internationalizer/detect.go index 17e9ea8..5ca9dba 100644 --- a/cmd/internationalizer/detect.go +++ b/cmd/internationalizer/detect.go @@ -4,44 +4,168 @@ import ( "fmt" "github.com/Tom-R-Main/Internationalizer/internal/config" - "github.com/Tom-R-Main/Internationalizer/internal/detect" + "github.com/Tom-R-Main/Internationalizer/internal/onboarding" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" ) -func newDetectCmd() *cobra.Command { - return &cobra.Command{ - Use: "detect", - Short: "Auto-detect project type and suggest config", - Long: "Scan the current directory to detect the i18n framework and suggest a configuration.", - RunE: func(cmd *cobra.Command, args []string) error { - d := detect.Detect(".") +type inspectionFlags struct { + config, bundle, locale, code string + json bool + limit int +} - if d.Type == detect.Unknown { - fmt.Println("Could not detect project type.") - fmt.Println("Create a .internationalizer.yml config file manually.") - return nil - } +type inspectionJSON struct { + Total map[string]int `json:"total"` + Matched map[string]int `json:"matched"` + Offline bool `json:"offline"` + ProviderVerified bool `json:"provider_verified"` + Inspection *onboarding.Inspection `json:"inspection"` +} - fmt.Printf("Detected: %s (confidence: %.0f%%)\n\n", d.Type, d.Confidence*100) - fmt.Println("Suggested configuration:") +func (f *inspectionFlags) bind(cmd *cobra.Command) { + cmd.Flags().StringVar(&f.config, "config", "", "Configuration path") + cmd.Flags().BoolVar(&f.json, "json", false, "Emit versioned JSON") + cmd.Flags().StringVar(&f.bundle, "bundle", "", "Filter by bundle ID") + cmd.Flags().StringVar(&f.locale, "locale", "", "Filter resolved targets by locale") + cmd.Flags().StringVar(&f.code, "finding-code", "", "Filter diagnostics by stable code") + cmd.Flags().IntVar(&f.limit, "limit", 50, "Maximum entries per result section (0 for all)") +} - suggested := map[string]interface{}{ - "source_locale": d.SourceLocale, - "source_path": d.SourcePath, +func newDetectCmd() *cobra.Command { + f := &inspectionFlags{} + cmd := &cobra.Command{Use: "detect", Short: "Discover configured and uncovered catalogs with runtime evidence", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { return runInspection(cmd, f, false) }} + f.bind(cmd) + return cmd +} + +func runInspection(cmd *cobra.Command, f *inspectionFlags, check bool) error { + if f.limit < 0 { + return fmt.Errorf("limit must be nonnegative") + } + report, err := onboarding.Scan(".", f.config) + if err != nil { + return err + } + if f.bundle != "" { + found := false + for _, b := range report.Bundles { + if b.ID == f.bundle { + found = true + break + } + } + if !found { + return codedError("unknown_bundle", fmt.Errorf("bundle %q is not configured", f.bundle)) + } + } + if f.locale != "" { + cfg := config.Config{TargetLocales: report.TargetLocales} + locale, ok := cfg.ConfiguredTargetLocale(f.locale) + if !ok { + return codedError("unknown_locale", fmt.Errorf("locale %q is not configured", f.locale)) + } + f.locale = locale + } + totals := map[string]int{"candidates": len(report.Candidates), "bundles": len(report.Bundles), "diagnostics": len(report.Diagnostics), "credentials": len(report.Credentials)} + failed := !report.ConfigExists + for _, d := range report.Diagnostics { + if d.Severity == "error" { + failed = true + } + } + if f.bundle != "" { + bundles := report.Bundles[:0] + for _, b := range report.Bundles { + if b.ID == f.bundle { + bundles = append(bundles, b) + } + } + report.Bundles = bundles + candidates := report.Candidates[:0] + for _, c := range report.Candidates { + for _, id := range c.ConfiguredBundles { + if id == f.bundle { + candidates = append(candidates, c) + break + } } - if len(d.TargetLocales) > 0 { - suggested["target_locales"] = d.TargetLocales + } + report.Candidates = candidates + } + if f.locale != "" { + for i := range report.Bundles { + b := &report.Bundles[i] + target, ok := b.Targets[f.locale] + b.Targets = map[string]string{} + b.Locales = nil + if ok { + b.Targets[f.locale] = target + b.Locales = []string{f.locale} } - suggested["llm"] = map[string]string{ - "provider": "gemini", - "model": config.DefaultGeminiModel, - "api_key_env": "GOOGLE_AI_STUDIO_API_KEY", + } + credentials := report.Credentials[:0] + for _, c := range report.Credentials { + if c.Locale == f.locale { + credentials = append(credentials, c) } - - out, _ := yaml.Marshal(suggested) - fmt.Println(string(out)) - return nil - }, + } + report.Credentials = credentials + } + diagnostics := report.Diagnostics[:0] + for _, d := range report.Diagnostics { + if (f.bundle == "" || d.Bundle == "" || d.Bundle == f.bundle) && (f.code == "" || d.Code == f.code) { + diagnostics = append(diagnostics, d) + } + } + report.Diagnostics = diagnostics + matched := map[string]int{"candidates": len(report.Candidates), "bundles": len(report.Bundles), "diagnostics": len(report.Diagnostics), "credentials": len(report.Credentials)} + if f.limit > 0 { + if len(report.Bundles) > f.limit { + report.Bundles = report.Bundles[:f.limit] + report.Truncated = true + } + if len(report.Credentials) > f.limit { + report.Credentials = report.Credentials[:f.limit] + report.Truncated = true + } + if len(report.Candidates) > f.limit { + report.Candidates = report.Candidates[:f.limit] + report.Truncated = true + } + if len(report.Diagnostics) > f.limit { + report.Diagnostics = report.Diagnostics[:f.limit] + report.Truncated = true + } + } + status := "ok" + var resultErr error + if check && failed { + status = "error" + resultErr = codedError("config_invalid", fmt.Errorf("configuration check failed")) + } + if f.json { + return emitJSON(cmd, status, inspectionJSON{Total: totals, Matched: matched, Offline: true, ProviderVerified: false, Inspection: report}, resultErr) + } + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Config: %s (exists: %t)\n", report.ConfigPath, report.ConfigExists); err != nil { + return err + } + for _, b := range report.Bundles { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Bundle %s: %s -> %s; syntax=%s; framework=%s\n", b.ID, b.Source, b.Target, b.MessageSyntax, b.Framework); err != nil { + return err + } + } + for _, c := range report.Candidates { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "Catalog %s: framework=%s; suggested syntax=%s; configured=%v\n", c.Source, c.Framework, c.SuggestedSyntax, c.ConfiguredBundles); err != nil { + return err + } + } + for _, d := range report.Diagnostics { + if _, err := fmt.Fprintf(cmd.OutOrStdout(), "%s [%s]: %s\n", d.Severity, d.Code, d.Message); err != nil { + return err + } + } + if _, err := fmt.Fprintln(cmd.OutOrStdout(), "Offline inspection only; no provider call or translation. Next: config plan --help"); err != nil { + return err } + return resultErr } diff --git a/cmd/internationalizer/json.go b/cmd/internationalizer/json.go new file mode 100644 index 0000000..ae654e7 --- /dev/null +++ b/cmd/internationalizer/json.go @@ -0,0 +1,209 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/config" + localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" + "github.com/spf13/cobra" +) + +type recoveryAction struct { + Argv []string `json:"argv"` + SideEffects []string `json:"side_effects"` + RequiredDecisions []string `json:"required_decisions"` +} + +type jsonFailure struct { + Code string `json:"code"` + Message string `json:"message"` + Recovery []recoveryAction `json:"recovery"` +} + +type jsonEnvelope struct { + SchemaVersion int `json:"schema_version"` + Status string `json:"status"` + Data any `json:"data"` + Errors []jsonFailure `json:"errors"` +} + +type reportedError struct{ error } + +func (e reportedError) Unwrap() error { return e.error } + +type commandError struct { + code string + cause error +} + +func (e commandError) Error() string { return e.cause.Error() } +func (e commandError) Unwrap() error { return e.cause } +func (e commandError) JSONCode() string { return e.code } + +func codedError(code string, err error) error { return commandError{code: code, cause: err} } + +// yaml's type errors may quote source values. Configuration may contain +// credentials or other private values in unsupported fields; never echo them. +func safeErrorMessage(err error) string { + message := err.Error() + if strings.Contains(message, "yaml:") { + return "configuration YAML could not be parsed; check YAML syntax and field types" + } + return message +} + +func safeConfigLoadError(err error) error { + return codedError("config_invalid", errors.New(safeErrorMessage(err))) +} + +// emitJSON writes exactly one versioned result, including failures. The returned +// marker keeps the root boundary from writing a second failure envelope. +func emitJSON(cmd *cobra.Command, status string, data any, err error) error { + envelope := jsonEnvelope{SchemaVersion: 1, Status: status, Data: data, Errors: []jsonFailure{}} + if err != nil { + if status == "ok" || status == "applied" || status == "planned" || status == "generated" || status == "adopted" { + envelope.Status = "error" + } + code := "command_failed" + var coded interface{ JSONCode() string } + if errors.As(err, &coded) { + code = coded.JSONCode() + } + if errors.Is(err, errValidationFailed) { + code = "validation_failed" + } + envelope.Errors = append(envelope.Errors, jsonFailure{Code: code, Message: safeErrorMessage(err), Recovery: []recoveryAction{errorRecovery(cmd, code)}}) + } + enc := json.NewEncoder(cmd.OutOrStdout()) + enc.SetIndent("", " ") + if encodeErr := enc.Encode(envelope); encodeErr != nil { + return reportedError{encodeErr} + } + if err != nil { + return reportedError{err} + } + return nil +} + +func errorRecovery(cmd *cobra.Command, code string) recoveryAction { + action := recoveryAction{Argv: []string{"internationalizer", "config", "check", "--json"}, SideEffects: []string{}, RequiredDecisions: []string{}} + withConfig := true + switch strings.ToLower(code) { + case "invalid_arguments", "invalid_plan": + action.Argv = []string{"internationalizer", "commands", "--json"} + withConfig = false + case "stale_plan", "plan_stale", "decisions_required", "plan_tampered": + action.Argv = []string{"internationalizer", "config", "plan", "--help"} + withConfig = false + action.RequiredDecisions = []string{"review_current_configuration_and_create_a_new_plan"} + case "validation_failed": + action.Argv = []string{"internationalizer", "validate", "--json", "--limit", "100"} + action.RequiredDecisions = []string{"review_catalog_findings"} + for _, name := range []string{"bundle", "locale", "finding-code"} { + if cmd.Flags().Changed(name) { + values, _ := cmd.Flags().GetStringSlice(name) + for _, value := range values { + action.Argv = append(action.Argv, "--"+name, value) + } + } + } + for _, name := range []string{"strict", "require-state", "require-approved"} { + value, _ := cmd.Flags().GetBool(name) + if value { + action.Argv = append(action.Argv, "--"+name) + } + } + case "credentials_missing": + action.RequiredDecisions = []string{"configure_the_required_provider_credential_in_your_environment"} + case "translation_failed": + action.RequiredDecisions = []string{"inspect_failed_jobs_and_current_state_before_retrying_provider_requests"} + } + if withConfig { + if flag := cmd.Flags().Lookup("config"); flag != nil && flag.Value.String() != "" { + action.Argv = append(action.Argv, "--config", flag.Value.String()) + } + } + return action +} + +func jsonRequested(args []string) bool { + for _, arg := range args { + if arg == "--" { + break + } + if arg == "--json" || arg == "--json=true" { + return true + } + } + return false +} + +func execute(root *cobra.Command, args []string) error { + root.SetArgs(args) + cmd, err := root.ExecuteC() + if err == nil { + return nil + } + var reported reportedError + if errors.As(err, &reported) { + return err + } + if jsonRequested(args) { + if cmd == nil { + cmd = root + } + if strings.Contains(err.Error(), "unknown flag") || strings.Contains(err.Error(), "invalid argument") || strings.Contains(err.Error(), "unknown command") { + err = codedError("invalid_arguments", err) + } + return emitJSON(cmd, "error", nil, err) + } + return err +} + +func selectConfig(cfg *config.Config, bundles, locales []string) error { + if len(bundles) > 0 { + selected := make([]config.Bundle, 0, len(bundles)) + for _, id := range bundles { + found := false + for _, bundle := range cfg.EffectiveBundles() { + if bundle.ID == id { + selected = append(selected, bundle) + found = true + break + } + } + if !found { + return codedError("unknown_bundle", fmt.Errorf("bundle %q is not configured", id)) + } + } + cfg.Bundles = selected + } + if len(locales) > 0 { + selected := make([]string, 0, len(locales)) + for _, locale := range locales { + canonical, ok := cfg.ConfiguredTargetLocale(locale) + if !ok { + return codedError("unknown_locale", fmt.Errorf("locale %q is not in target_locales", locale)) + } + selected = append(selected, canonical) + } + cfg.TargetLocales = selected + // Filter only the in-memory execution configuration; on-disk overrides + // remain untouched. Validation rejects overrides for unselected locales. + filtered := make(map[string]config.LLMOverride) + for locale, override := range cfg.LLM.LocaleOverrides { + canonical, _ := localeid.Canonical(locale) + for _, selection := range selected { + selectedCanonical, _ := localeid.Canonical(selection) + if canonical == selectedCanonical { + filtered[locale] = override + } + } + } + cfg.LLM.LocaleOverrides = filtered + } + return nil +} diff --git a/cmd/internationalizer/json_test.go b/cmd/internationalizer/json_test.go new file mode 100644 index 0000000..cedd4f1 --- /dev/null +++ b/cmd/internationalizer/json_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/translate" + "github.com/Tom-R-Main/Internationalizer/internal/validate" +) + +func runJSONCommand(t *testing.T, args ...string) (jsonEnvelope, error) { + t.Helper() + root := newRootCmd() + var out, stderr bytes.Buffer + root.SetOut(&out) + root.SetErr(&stderr) + err := execute(root, args) + var result jsonEnvelope + if decodeErr := json.Unmarshal(out.Bytes(), &result); decodeErr != nil { + t.Fatalf("invalid JSON: %v; stdout=%s; stderr=%s; error=%v", decodeErr, out.String(), stderr.String(), err) + } + if result.SchemaVersion != 1 { + t.Fatalf("schema_version = %d", result.SchemaVersion) + } + if stderr.Len() != 0 { + t.Fatalf("unexpected stderr: %s", stderr.String()) + } + return result, err +} + +func TestJSONEarlyFailuresAreVersioned(t *testing.T) { + for _, tc := range []struct { + name string + args []string + code string + }{ + {"unknown flag", []string{"translate", "--json", "--nonexistent"}, "invalid_arguments"}, + {"invalid limit", []string{"validate", "--json", "--limit=-1"}, "invalid_arguments"}, + {"missing config", []string{"validate", "--json", "--config", filepath.Join(t.TempDir(), "missing.yml")}, "config_invalid"}, + {"malformed flag", []string{"translate", "--json", "--limit=abc"}, "invalid_arguments"}, + } { + t.Run(tc.name, func(t *testing.T) { + result, err := runJSONCommand(t, tc.args...) + if err == nil || result.Status != "error" || len(result.Errors) != 1 || result.Errors[0].Code != tc.code { + t.Fatalf("result=%+v error=%v", result, err) + } + if len(result.Errors[0].Recovery) == 0 || len(result.Errors[0].Recovery[0].Argv) == 0 { + t.Fatal("missing structured recovery") + } + }) + } +} + +func TestJSONConfigurationParseErrorsDoNotEchoValues(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yml") + const secret = "synthetic-private-value-never-echo" + if err := os.WriteFile(path, []byte("target_locales: "+secret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + result, err := runJSONCommand(t, "translate", "--dry-run", "--json", "--config", path) + data, _ := json.Marshal(result) + if err == nil || strings.Contains(string(data), secret) || strings.Contains(err.Error(), secret) { + t.Fatalf("unsafe parse failure: %s", data) + } +} + +func TestJSONRecoveryNeverAutomaticallyReappliesStalePlan(t *testing.T) { + cmd := newValidateCmd() + action := errorRecovery(cmd, "STALE_PLAN") + if strings.Join(action.Argv, " ") != "internationalizer config plan --help" || len(action.RequiredDecisions) == 0 || len(action.SideEffects) != 0 { + t.Fatalf("unsafe stale-plan recovery: %+v", action) + } +} + +func TestDryRunJSONHasPlannedNotGeneratedState(t *testing.T) { + configPath := writeValidateProject(t, `{"hello":"Hello","goodbye":"Goodbye"}`, `{}`, "message_syntax: plain\n") + dir := filepath.Dir(configPath) + before, err := os.ReadFile(filepath.Join(dir, "fr.json")) + if err != nil { + t.Fatal(err) + } + result, err := runJSONCommand(t, "translate", "--config", configPath, "--dry-run", "--json", "--limit=1") + if err != nil || result.Status != "planned" { + t.Fatalf("result=%+v error=%v", result, err) + } + data, _ := json.Marshal(result.Data) + var run translationJSON + if err := json.Unmarshal(data, &run); err != nil { + t.Fatal(err) + } + if !run.DryRun || run.ProviderCalled || run.HumanReviewApproved || run.Summary.PlannedKeys != 2 || run.Summary.GeneratedKeys != 0 { + t.Fatalf("incorrect planning state: %+v", run) + } + after, err := os.ReadFile(filepath.Join(dir, "fr.json")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(before, after) { + t.Fatal("dry-run changed target") + } +} + +func TestValidationFilteringNeverHidesFailureStatus(t *testing.T) { + configPath := writeValidateProject(t, `{"hello":"Hello"}`, `{}`, "") + result, err := runJSONCommand(t, "validate", "--config", configPath, "--json", "--finding-code=source_identical", "--limit=1") + if !errors.Is(err, errValidationFailed) || result.Status != "validation_failed" { + t.Fatalf("result=%+v error=%v", result, err) + } + data, _ := json.Marshal(result.Data) + var report validationJSON + if err := json.Unmarshal(data, &report); err != nil { + t.Fatal(err) + } + if report.MatchingFindings != 0 || report.ReportCount != 1 || report.HumanReviewApproved { + t.Fatalf("unexpected report %+v", report) + } +} + +func TestBoundedValidationCountsAllFindings(t *testing.T) { + reports := []validate.Report{{Bundle: "web", Locale: "fr", Findings: []validate.Finding{{Code: validate.CodeMissingKey}, {Code: validate.CodeMissingKey}, {Code: validate.CodeProtectedStructureMismatch}}}} + data := boundedValidation(reports, nil, 1) + if data.FindingCount != 3 || data.MatchingFindings != 3 || data.FindingCounts[validate.CodeMissingKey] != 2 || !data.Truncated || len(data.Reports[0].Findings) != 1 { + t.Fatalf("unexpected bounded report %+v", data) + } +} + +func TestProviderCallsAreNotPlannedBatches(t *testing.T) { + if summaryHasProviderCalls([]translate.Result{{Batches: 3, DryRun: true}}) { + t.Fatal("planned batch treated as executed provider call") + } + if !summaryHasProviderCalls([]translate.Result{{ProviderCalls: 1}}) { + t.Fatal("actual provider call omitted") + } +} + +func TestJSONAutoDiagnosticIsActionable(t *testing.T) { + configPath := writeValidateProject(t, `{"docs.tui.skills.desc":"Read {.sift,.claude}/skills"}`, `{}`, "message_syntax: auto\n") + for _, command := range []string{"validate", "translate"} { + args := []string{command, "--config", configPath, "--json"} + if command == "translate" { + args = append(args, "--dry-run") + } + result, err := runJSONCommand(t, args...) + data, _ := json.Marshal(result.Data) + if err == nil || !strings.Contains(string(data), "brace syntax inside HTML code") || !strings.Contains(string(data), "plain, i18next, or icu") { + t.Fatalf("missing actionable ambiguity: %s error=%v", data, err) + } + } +} diff --git a/cmd/internationalizer/main.go b/cmd/internationalizer/main.go index b9ac8da..59ab42b 100644 --- a/cmd/internationalizer/main.go +++ b/cmd/internationalizer/main.go @@ -11,6 +11,16 @@ import ( var version = "dev" func main() { + if err := execute(newRootCmd(), os.Args[1:]); err != nil { + var reported reportedError + if !errors.Is(err, errValidationFailed) && !errors.As(err, &reported) { + fmt.Fprintln(os.Stderr, err) + } + os.Exit(1) + } +} + +func newRootCmd() *cobra.Command { rootCmd := &cobra.Command{ Use: "internationalizer", Short: "AI-native i18n CLI tool", @@ -28,12 +38,8 @@ func main() { newValidateCmd(), newReviewCmd(), newPseudoCmd(), + newConfigCmd(), ) - - if err := rootCmd.Execute(); err != nil { - if !errors.Is(err, errValidationFailed) { - fmt.Fprintln(os.Stderr, err) - } - os.Exit(1) - } + rootCmd.AddCommand(newCommandsCmd(rootCmd)) + return rootCmd } diff --git a/cmd/internationalizer/onboarding_test.go b/cmd/internationalizer/onboarding_test.go new file mode 100644 index 0000000..8fde22c --- /dev/null +++ b/cmd/internationalizer/onboarding_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "testing" +) + +func TestInspectionRejectsUnknownScope(t *testing.T) { + root := t.TempDir() + t.Chdir(root) + if err := os.WriteFile(filepath.Join(root, ".internationalizer.yml"), []byte("source_locale: en\ntarget_locales: [fr]\nsource_path: en.json\nmessage_syntax: plain\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "en.json"), []byte(`{"hello":"Hello"}`), 0600); err != nil { + t.Fatal(err) + } + for _, flag := range []string{"--bundle", "--locale"} { + t.Run(flag, func(t *testing.T) { + command := newRootCmd() + var out bytes.Buffer + command.SetOut(&out) + command.SetErr(&out) + if err := execute(command, []string{"config", "check", flag, "typo", "--json"}); err == nil { + t.Fatal("unknown selection passed") + } + var envelope jsonEnvelope + if err := json.Unmarshal(out.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if len(envelope.Errors) != 1 || envelope.Status != "error" { + t.Fatalf("unexpected failure: %s", out.String()) + } + }) + } +} + +func TestCommandsContractFilter(t *testing.T) { + command := newRootCmd() + var out bytes.Buffer + command.SetOut(&out) + if err := execute(command, []string{"commands", "--command", "config apply", "--json"}); err != nil { + t.Fatal(err) + } + var envelope struct { + Data struct { + Commands []commandContract `json:"commands"` + Matched int `json:"matched"` + } + } + if err := json.Unmarshal(out.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if envelope.Data.Matched != 1 || len(envelope.Data.Commands) != 1 { + t.Fatalf("bad filter: %s", out.String()) + } + contract := envelope.Data.Commands[0] + if contract.InputSchema == nil || contract.OutputSchema == nil || len(contract.SideEffects) == 0 { + t.Fatalf("incomplete contract: %+v", contract) + } +} diff --git a/cmd/internationalizer/schema.go b/cmd/internationalizer/schema.go new file mode 100644 index 0000000..16a3c80 --- /dev/null +++ b/cmd/internationalizer/schema.go @@ -0,0 +1,197 @@ +package main + +import ( + "reflect" + "strconv" + "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/onboarding" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const jsonSchemaDialect = "https://json-schema.org/draft/2020-12/schema" + +// workflowOutputSchema describes JSON mode, not terminal text. Null data is +// permitted because argument/configuration failures happen before a result +// exists. Unknown object fields remain allowed for additive CLI evolution. +func workflowOutputSchema(path string) map[string]any { + path = strings.TrimPrefix(path, "internationalizer ") + var data map[string]any + switch path { + case "commands": + data = schemaObject(map[string]any{ + "cli_version": schemaType("string"), "commands": schemaArray(schemaForType(reflect.TypeFor[commandContract]())), + "total": schemaType("integer"), "matched": schemaType("integer"), "truncated": schemaType("boolean"), + "exit_codes": schemaForType(reflect.TypeFor[map[string]string]()), "states": schemaArray(schemaType("string")), + "authorization": schemaType("string"), + }, "cli_version", "commands", "total", "matched", "truncated", "exit_codes", "states", "authorization") + case "detect", "config check": + data = schemaObject(map[string]any{ + "inspection": schemaForType(reflect.TypeFor[onboarding.Inspection]()), + "total": schemaForType(reflect.TypeFor[map[string]int]()), "matched": schemaForType(reflect.TypeFor[map[string]int]()), + "offline": map[string]any{"type": "boolean", "const": true}, "provider_verified": map[string]any{"type": "boolean", "const": false}, + }, "inspection", "total", "matched", "offline", "provider_verified") + data["description"] = "Offline discovery and configuration evidence; credential presence is not a successful provider call." + case "config plan": + data = schemaForType(reflect.TypeFor[onboarding.ConfigPlan]()) + data["description"] = "Reviewable proposal only. required_decisions must be resolved before application; a plan does not grant authorization." + case "config apply": + data = schemaForType(reflect.TypeFor[onboarding.ApplyReceipt]()) + data["description"] = "Receipt tied to a plan and verified configuration fingerprint; application does not translate catalogs." + case "translate": + data = schemaForType(reflect.TypeFor[translationJSON]()) + data["description"] = "Planning, generation, adoption, and partial failure are distinct. Dry-run makes no provider call or file change; generation is not human approval." + case "validate": + data = schemaForType(reflect.TypeFor[validationJSON]()) + data["description"] = "Counts cover the full selected scope before presentation limits. Structural validation and checked human approval are distinct." + default: + return map[string]any{"$schema": jsonSchemaDialect, "description": "This command does not declare a versioned workflow JSON result; inspect its command-specific help."} + } + envelope := schemaObject(map[string]any{ + "schema_version": map[string]any{"type": "integer", "const": 1}, + "status": map[string]any{"type": "string", "description": "Workflow state; inspect errors and command-specific data. An exit code of zero does not imply configuration choices or human review are complete."}, + "data": map[string]any{"anyOf": []any{data, schemaType("null")}}, + "errors": schemaArray(schemaForType(reflect.TypeFor[jsonFailure]())), + }, "schema_version", "status", "data", "errors") + envelope["$schema"] = jsonSchemaDialect + envelope["title"] = "Internationalizer v1: " + path + return envelope +} + +// workflowInputSchema is a normalized invocation, not a claim that the CLI +// accepts JSON stdin. Each flags property maps to --; array values are +// repeated flags. The separate command argv is supplied by commands --json. +func workflowInputSchema(cmd *cobra.Command) map[string]any { + properties := map[string]any{} + required := []string{} + visit := func(flag *pflag.Flag) { + if flag.Hidden { + return + } + property := flagInputSchema(flag) + properties[flag.Name] = property + if len(flag.Annotations[cobra.BashCompOneRequiredFlag]) > 0 { + required = append(required, flag.Name) + } + } + cmd.InheritedFlags().VisitAll(visit) + cmd.Flags().VisitAll(visit) + path := strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name()+" ") + if path == "config apply" && cmd.Flags().Lookup("plan") != nil { + found := false + for _, name := range required { + found = found || name == "plan" + } + if !found { + required = append(required, "plan") + } + } + flags := schemaObject(properties, required...) + flags["additionalProperties"] = false + schema := schemaObject(map[string]any{ + "flags": flags, + "args": map[string]any{"type": "array", "items": schemaType("string")}, + }, "flags") + schema["$schema"] = jsonSchemaDialect + schema["description"] = "Normalized invocation: map flag names to --name arguments; repeat array-valued flags. This is not a JSON-stdin API. --no-input disables prompts, not authorization." + switch path { + case "commands", "detect", "config check", "config plan", "config apply", "translate", "validate": + schema["properties"].(map[string]any)["args"].(map[string]any)["maxItems"] = 0 + } + return schema +} + +func flagInputSchema(flag *pflag.Flag) map[string]any { + property := schemaType("string") + switch flag.Value.Type() { + case "bool": + property = schemaType("boolean") + if value, err := strconv.ParseBool(flag.DefValue); err == nil { + property["default"] = value + } + case "int", "int8", "int16", "int32", "int64", "uint", "uint8", "uint16", "uint32", "uint64", "count": + property = schemaType("integer") + if value, err := strconv.ParseInt(flag.DefValue, 10, 64); err == nil { + property["default"] = value + } + case "float32", "float64": + property = schemaType("number") + if value, err := strconv.ParseFloat(flag.DefValue, 64); err == nil { + property["default"] = value + } + case "stringArray", "stringSlice": + property = schemaArray(schemaType("string")) + case "string": + property["default"] = flag.DefValue + } + property["description"] = flag.Usage + property["x-cli-flag"] = "--" + flag.Name + property["x-cli-type"] = flag.Value.Type() + if flag.Name == "limit" { + property["minimum"] = 0 + } + return property +} + +func schemaType(kind string) map[string]any { return map[string]any{"type": kind} } +func schemaArray(items map[string]any) map[string]any { + return map[string]any{"type": "array", "items": items} +} +func schemaObject(properties map[string]any, required ...string) map[string]any { + if required == nil { + required = []string{} + } + return map[string]any{"type": "object", "properties": properties, "required": required, "additionalProperties": true} +} + +// Struct JSON tags are the authority for required keys. Nil slices/maps encode +// as null in the CLI today; schemas acknowledge that instead of promising []. +func schemaForType(t reflect.Type) map[string]any { + switch t.Kind() { + case reflect.Pointer: + return map[string]any{"anyOf": []any{schemaForType(t.Elem()), schemaType("null")}} + case reflect.Struct: + properties := map[string]any{} + required := []string{} + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + if !field.IsExported() { + continue + } + tag := strings.Split(field.Tag.Get("json"), ",") + if tag[0] == "-" { + continue + } + name := tag[0] + if name == "" { + name = field.Name + } + properties[name] = schemaForType(field.Type) + optional := false + for _, option := range tag[1:] { + optional = optional || option == "omitempty" || option == "omitzero" + } + if !optional { + required = append(required, name) + } + } + return schemaObject(properties, required...) + case reflect.Slice: + return map[string]any{"type": []string{"array", "null"}, "items": schemaForType(t.Elem())} + case reflect.Array: + return schemaArray(schemaForType(t.Elem())) + case reflect.Map: + return map[string]any{"type": []string{"object", "null"}, "additionalProperties": schemaForType(t.Elem())} + case reflect.String: + return schemaType("string") + case reflect.Bool: + return schemaType("boolean") + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return schemaType("integer") + case reflect.Float32, reflect.Float64: + return schemaType("number") + default: + return map[string]any{} + } +} diff --git a/cmd/internationalizer/schema_test.go b/cmd/internationalizer/schema_test.go new file mode 100644 index 0000000..6c6b6e5 --- /dev/null +++ b/cmd/internationalizer/schema_test.go @@ -0,0 +1,112 @@ +package main + +import ( + "encoding/json" + "reflect" + "slices" + "testing" + + "github.com/spf13/cobra" +) + +func TestWorkflowOutputSchemasDescribeActualData(t *testing.T) { + for _, tc := range []struct { + path string + required []string + }{ + {"commands", []string{"commands", "cli_version", "exit_codes"}}, + {"detect", []string{"inspection", "total", "matched", "offline", "provider_verified"}}, + {"config check", []string{"inspection", "total", "matched"}}, + {"config plan", []string{"id", "config_path", "proposed_yaml", "required_decisions", "observations"}}, + {"config apply", []string{"plan_id", "status", "changed_paths", "config_sha256", "observations_revalidated"}}, + {"translate", []string{"summary", "jobs", "dry_run", "provider_called", "human_review_approved"}}, + {"validate", []string{"report_count", "finding_count", "reports", "human_review_checked"}}, + } { + t.Run(tc.path, func(t *testing.T) { + schema := workflowOutputSchema("internationalizer " + tc.path) + if schema["$schema"] != jsonSchemaDialect || schema["additionalProperties"] != true { + t.Fatalf("invalid schema metadata: %+v", schema) + } + if _, err := json.Marshal(schema); err != nil { + t.Fatal(err) + } + properties := schema["properties"].(map[string]any) + if properties["schema_version"].(map[string]any)["const"] != 1 { + t.Fatal("missing schema version constraint") + } + alternatives := properties["data"].(map[string]any)["anyOf"].([]any) + if alternatives[1].(map[string]any)["type"] != "null" { + t.Fatal("early failures cannot return null data") + } + data := alternatives[0].(map[string]any) + for _, name := range tc.required { + if !slices.Contains(data["required"].([]string), name) { + t.Errorf("missing required key %s", name) + } + if _, exists := data["properties"].(map[string]any)[name]; !exists { + t.Errorf("missing property %s", name) + } + } + }) + } +} + +func TestWorkflowInputSchemaHasTypedFlags(t *testing.T) { + root := newRootCmd() + for _, path := range [][]string{{"translate"}, {"config", "plan"}, {"config", "apply"}} { + cmd, _, err := root.Find(path) + if err != nil { + t.Fatal(err) + } + schema := workflowInputSchema(cmd) + properties := schema["properties"].(map[string]any) + flags := properties["flags"].(map[string]any) + flagProps := flags["properties"].(map[string]any) + if flagProps["json"].(map[string]any)["type"] != "boolean" || properties["args"].(map[string]any)["maxItems"] != 0 { + t.Fatalf("incorrect flag/args shapes: %+v", schema) + } + switch cmd.Name() { + case "translate": + if flagProps["locale"].(map[string]any)["type"] != "array" || flagProps["limit"].(map[string]any)["type"] != "integer" { + t.Fatal("wrong translation flag types") + } + case "plan": + if flagProps["add-bundle"].(map[string]any)["type"] != "array" { + t.Fatal("repeatable add-bundle flag is not an array") + } + case "apply": + if !slices.Contains(flags["required"].([]string), "plan") { + t.Fatal("apply does not require an explicit plan") + } + } + } +} + +func TestSchemaTracksOptionalAndNullableJSONFields(t *testing.T) { + type payload struct { + Name string `json:"name"` + Optional string `json:"optional,omitempty"` + Items []string `json:"items"` + } + schema := schemaForType(reflect.TypeFor[payload]()) + if !reflect.DeepEqual(schema["required"], []string{"name", "items"}) { + t.Fatalf("required = %v", schema["required"]) + } + items := schema["properties"].(map[string]any)["items"].(map[string]any) + if !reflect.DeepEqual(items["type"], []string{"array", "null"}) { + t.Fatalf("slice type = %v", items["type"]) + } +} + +func TestInputSchemaUsesFlagDefaultsNotCurrentValues(t *testing.T) { + cmd := &cobra.Command{Use: "example"} + cmd.Flags().Int("count", 3, "count") + if err := cmd.Flags().Set("count", "9"); err != nil { + t.Fatal(err) + } + schema := workflowInputSchema(cmd) + flags := schema["properties"].(map[string]any)["flags"].(map[string]any)["properties"].(map[string]any) + if flags["count"].(map[string]any)["default"] != int64(3) { + t.Fatal("schema default changed with invocation state") + } +} diff --git a/cmd/internationalizer/translate.go b/cmd/internationalizer/translate.go index 2fa232e..8d95b16 100644 --- a/cmd/internationalizer/translate.go +++ b/cmd/internationalizer/translate.go @@ -14,13 +14,18 @@ import ( func newTranslateCmd() *cobra.Command { cmd := &cobra.Command{ Use: "translate", + Args: cobra.NoArgs, Short: "Translate missing keys using an LLM", Long: "Detect missing translation keys and generate translations via an LLM provider.", RunE: func(cmd *cobra.Command, args []string) error { + limit, _ := cmd.Flags().GetInt("limit") + if limit < 0 { + return codedError("invalid_arguments", fmt.Errorf("limit must be non-negative")) + } cfgPath, _ := cmd.Flags().GetString("config") cfg, err := config.Load(cfgPath) if err != nil { - return err + return safeConfigLoadError(err) } dryRun, _ := cmd.Flags().GetBool("dry-run") @@ -29,14 +34,18 @@ func newTranslateCmd() *cobra.Command { locales, _ := cmd.Flags().GetStringSlice("locale") batchSize, _ := cmd.Flags().GetInt("batch-size") concurrency, _ := cmd.Flags().GetInt("concurrency") + bundles, _ := cmd.Flags().GetStringSlice("bundle") + if err := selectConfig(cfg, bundles, nil); err != nil { + return err + } if err := cfg.ValidateProject(); err != nil { - return err + return codedError("config_invalid", err) } // Local inspection and explicit adoption do not need provider credentials. if !dryRun && !adoptExisting { if err := cfg.ValidateCredentialsForLocales(locales); err != nil { - return err + return codedError("credentials_missing", err) } } @@ -90,6 +99,49 @@ func newTranslateCmd() *cobra.Command { Concurrency: concurrency, LocaleProviders: localeProviders, }) + asJSON, _ := cmd.Flags().GetBool("json") + if asJSON { + status := "generated" + if dryRun { + status = "planned" + } else if adoptExisting { + status = "adopted" + } + if err != nil { + status = "blocked" + err = codedError("translation_failed", err) + } + summary := translationSummary{Jobs: len(results)} + for _, result := range results { + summary.ErrorCount += len(result.Errors) + if result.BlockedBySource || len(result.Errors) > 0 { + summary.BlockedJobs++ + } + if dryRun { + summary.PlannedKeys += result.KeysSkipped + } + summary.GeneratedKeys += result.KeysTranslated + } + if err != nil && summary.GeneratedKeys > 0 { + status = "partial_failure" + } + providerCalled := summaryHasProviderCalls(results) + truncated := false + if limit > 0 && len(results) > limit { + results = results[:limit] + truncated = true + } + remaining := limit + for i := range results { + if limit > 0 { + n := min(max(remaining, 0), len(results[i].Errors)) + truncated = truncated || n < len(results[i].Errors) + results[i].Errors = results[i].Errors[:n] + remaining -= n + } + } + return emitJSON(cmd, status, translationJSON{Summary: summary, Jobs: results, Returned: len(results), Truncated: truncated, DryRun: dryRun, ProviderCalled: providerCalled, HumanReviewApproved: false}, err) + } if len(results) > 0 { if _, outputErr := fmt.Fprint(cmd.OutOrStdout(), translate.FormatResults(results, time.Since(start))); outputErr != nil { return outputErr @@ -101,6 +153,9 @@ func newTranslateCmd() *cobra.Command { cmd.Flags().StringP("config", "c", "", "path to config file (default: .internationalizer.yml)") cmd.Flags().StringSliceP("locale", "l", nil, "target locale(s) to translate (default: all)") + cmd.Flags().StringSlice("bundle", nil, "bundle ID(s) to translate (default: all)") + cmd.Flags().Bool("json", false, "output a versioned JSON result, including failures") + cmd.Flags().Int("limit", 100, "maximum JSON jobs and error details returned; 0 returns all (does not limit execution)") cmd.Flags().Bool("dry-run", false, "show what would be translated without calling the LLM") cmd.Flags().Bool("adopt-existing", false, "record existing translations as the provenance baseline without calling the LLM") cmd.Flags().Bool("refresh-policy", false, "retranslate entries made stale by prompt, style-guide, glossary, provider, or model changes") @@ -111,3 +166,30 @@ func newTranslateCmd() *cobra.Command { return cmd } + +type translationSummary struct { + Jobs int `json:"jobs"` + BlockedJobs int `json:"blocked_jobs"` + PlannedKeys int `json:"planned_keys"` + GeneratedKeys int `json:"generated_keys"` + ErrorCount int `json:"error_count"` +} + +type translationJSON struct { + Summary translationSummary `json:"summary"` + Jobs []translate.Result `json:"jobs"` + Returned int `json:"returned"` + Truncated bool `json:"truncated"` + DryRun bool `json:"dry_run"` + ProviderCalled bool `json:"provider_called"` + HumanReviewApproved bool `json:"human_review_approved"` +} + +func summaryHasProviderCalls(results []translate.Result) bool { + for _, result := range results { + if result.ProviderCalls > 0 { + return true + } + } + return false +} diff --git a/cmd/internationalizer/validate.go b/cmd/internationalizer/validate.go index d22d04d..0d70e17 100644 --- a/cmd/internationalizer/validate.go +++ b/cmd/internationalizer/validate.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "errors" "fmt" @@ -15,6 +14,7 @@ var errValidationFailed = errors.New("validation failed") func newValidateCmd() *cobra.Command { cmd := &cobra.Command{ Use: "validate", + Args: cobra.NoArgs, Short: "Validate locale files against the source locale", Long: `Check target locale structure and interpolation against the source locale. @@ -23,9 +23,21 @@ structure, glossary, and configured plural rules. Use --require-state to verify that source, policy, and target content still match the translation manifest. Use --require-approved to additionally require explicit human approval.`, RunE: func(cmd *cobra.Command, args []string) error { + limit, _ := cmd.Flags().GetInt("limit") + if limit < 0 { + return codedError("invalid_arguments", fmt.Errorf("limit must be non-negative")) + } cfgPath, _ := cmd.Flags().GetString("config") cfg, err := config.Load(cfgPath) if err != nil { + return safeConfigLoadError(err) + } + if err := cfg.ValidateProject(); err != nil { + return codedError("config_invalid", err) + } + bundles, _ := cmd.Flags().GetStringSlice("bundle") + locales, _ := cmd.Flags().GetStringSlice("locale") + if err := selectConfig(cfg, bundles, locales); err != nil { return err } @@ -45,11 +57,17 @@ Use --require-approved to additionally require explicit human approval.`, quiet, _ := cmd.Flags().GetBool("quiet") if asJSON { - enc := json.NewEncoder(cmd.OutOrStdout()) - enc.SetIndent("", " ") - if err := enc.Encode(reports); err != nil { - return err + codes, _ := cmd.Flags().GetStringSlice("finding-code") + data := boundedValidation(reports, codes, limit) + status := "structural_validation_passed" + var failure error + if validate.HasFailures(reports) { + status = "validation_failed" + failure = errValidationFailed } + data.HumanReviewChecked = requireApproved + data.HumanReviewApproved = requireApproved && failure == nil + return emitJSON(cmd, status, data, failure) } else if !quiet { if _, err := fmt.Fprint(cmd.OutOrStdout(), validate.FormatHuman(reports)); err != nil { return err @@ -65,6 +83,10 @@ Use --require-approved to additionally require explicit human approval.`, cmd.Flags().StringP("config", "c", "", "path to config file (default: .internationalizer.yml)") cmd.Flags().Bool("json", false, "output report as JSON") + cmd.Flags().StringSlice("bundle", nil, "bundle ID(s) to validate (default: all)") + cmd.Flags().StringSliceP("locale", "l", nil, "target locale(s) to validate (default: all)") + cmd.Flags().StringSlice("finding-code", nil, "filter JSON findings by stable code; does not change failure status") + cmd.Flags().Int("limit", 100, "maximum JSON reports and detail items returned; 0 returns all") cmd.Flags().BoolP("quiet", "q", false, "exit code only, no output") cmd.Flags().Bool("strict", false, "fail on untranslated values and strict policy findings") cmd.Flags().Bool("require-state", false, "fail when translation manifest state is missing or stale") @@ -72,3 +94,78 @@ Use --require-approved to additionally require explicit human approval.`, return cmd } + +type validationJSON struct { + ReportCount int `json:"report_count"` + FindingCount int `json:"finding_count"` + MatchingFindings int `json:"matching_findings"` + FindingCounts map[validate.FindingCode]int `json:"finding_counts"` + Reports []validate.Report `json:"reports"` + Truncated bool `json:"truncated"` + HumanReviewApproved bool `json:"human_review_approved"` + HumanReviewChecked bool `json:"human_review_checked"` +} + +// Limits are presentation-only: aggregate status always reflects every selected +// bundle and locale, including findings hidden by a filter or output bound. +func boundedValidation(reports []validate.Report, codes []string, limit int) validationJSON { + data := validationJSON{ReportCount: len(reports), FindingCounts: map[validate.FindingCode]int{}, Reports: []validate.Report{}} + remaining := limit + for _, report := range reports { + selected := []validate.Finding{} + for _, finding := range report.Findings { + data.FindingCount++ + data.FindingCounts[finding.Code]++ + match := len(codes) == 0 + for _, code := range codes { + if code == string(finding.Code) { + match = true + } + } + if !match { + continue + } + data.MatchingFindings++ + if limit == 0 || remaining > 0 { + selected = append(selected, finding) + remaining-- + } else { + data.Truncated = true + } + } + report.Findings = selected + // Legacy detail arrays are bounded too, and omitted under a code filter + // because they do not carry codes. Counts above retain the full findings. + if len(codes) > 0 { + report.Missing = nil + report.Extra = nil + report.Mismatches = nil + report.Errors = nil + } else { + if limit > 0 { + n := min(max(remaining, 0), len(report.Missing)) + data.Truncated = data.Truncated || n < len(report.Missing) + report.Missing = report.Missing[:n] + remaining -= n + n = min(max(remaining, 0), len(report.Extra)) + data.Truncated = data.Truncated || n < len(report.Extra) + report.Extra = report.Extra[:n] + remaining -= n + n = min(max(remaining, 0), len(report.Mismatches)) + data.Truncated = data.Truncated || n < len(report.Mismatches) + report.Mismatches = report.Mismatches[:n] + remaining -= n + n = min(max(remaining, 0), len(report.Errors)) + data.Truncated = data.Truncated || n < len(report.Errors) + report.Errors = report.Errors[:n] + remaining -= n + } + } + if limit == 0 || len(data.Reports) < limit { + data.Reports = append(data.Reports, report) + } else { + data.Truncated = true + } + } + return data +} diff --git a/docs/cli-onboarding.md b/docs/cli-onboarding.md new file mode 100644 index 0000000..6b03c43 --- /dev/null +++ b/docs/cli-onboarding.md @@ -0,0 +1,172 @@ +# Configuration workflow and JSON contract + +Run setup commands from the project root. Inspection resolves relative catalog +paths against that root, matching translation; an explicit config file does not +change the base directory for its paths. + +## Discover and inspect + +```sh +internationalizer commands --json +internationalizer commands --command 'config plan' --json +internationalizer detect --json +internationalizer config check --json +internationalizer config check --bundle web --locale fr --json +``` + +`detect` reports source candidates, configured bundle IDs, runtime evidence, +suggested syntax, and uncertainty. It never edits configuration. A catalog under +`tmp/` is a source-ownership question, not automatically an error. + +`config check` adds resolved sources, target templates and locale-specific paths, +syntax provenance, diagnostics, and credential-presence checks. It is offline: +`provider_verified: false` means no provider request was made, even when a +credential is present. Neither command emits credential values. + +i18next dependency evidence suggests `i18next` syntax. An `i18next-icu` +dependency or localization-module reference leaves the profile unresolved until +you confirm the plugin registration. Discovery does not infer grammar from JSON +or claim that an absent static reference proves there is no ICU integration. + +Source-locale filenames such as `en.json` and paths such as +`locales/en/common.json` are catalog candidates. Other authoritative sources can +be supplied explicitly with `--add-bundle` and `--target`. + +Scan limits are 50,000 entries, depth 14, and 8 MiB per inspected catalog. +Dependency, build, hidden, and data directories and symlink traversals are +excluded. Large localization modules are skipped. A truncated scan is not proof +that all catalogs or integrations have been found. + +## Plan a change + +This example extends an existing marketing-only config whose `source_path` is +`tmp/english-keys.json`: + +```sh +internationalizer config plan --json \ + --add-bundle web=exf-app/web/src/i18n/locales/en.json \ + --syntax web=i18next \ + --syntax default=i18next \ + --confirm-source tmp/english-keys.json \ + --out config-plan.json +``` + +Each `--add-bundle ID=SOURCE` is an explicit catalog selection. For a discovered +source, the proposal uses its target template; use `--target ID=TEMPLATE` to +override it. New JSON bundles need an explicit `--syntax ID=PROFILE` decision. +Profiles are `plain`, `i18next`, and `icu`; an ambiguous `auto` choice remains +unresolved. Existing `source_path` configurations become a bundle named +`default`, preserving the existing translation-state identity. + +For a new project, select target locales too: + +```sh +internationalizer config plan --config .internationalizer.yml \ + --add-bundle web=src/locales/en.json \ + --syntax web=i18next --locale fr --locale ja --out config-plan.json --json +``` + +`--source-locale` and `--locale` also change existing locale settings when +explicitly supplied. Omitting them preserves the current locale set and overrides. + +Planning does not modify the config. `--out` creates a new, owner-readable plan +file and refuses to overwrite an existing file. Without `--out`, the proposal is +returned only in the command output. A `needs_decision` status means inspect +`required_decisions` and create a new plan with the missing selections. + +Plans contain proposed YAML, a diff, configuration fingerprints, and observations +of source/runtime evidence. Existing provider settings, locale overrides, +glossary paths, comments, and ordinary unknown settings are preserved. Configs +with YAML aliases/merge keys or possible inline credentials are rejected rather +than rewritten. Use environment-variable references for credentials. Treat +saved plans as private project configuration, not public attachments. + +Inspection can report a home-directory fallback config. Plan/apply only edits +files inside the project root; it does not silently replace a home config with a +new local config. Choose an explicit local `--config` path to start fresh. + +## Apply and verify + +```sh +internationalizer config apply --plan config-plan.json --no-input --json +internationalizer config check --json +internationalizer translate --dry-run --json +internationalizer validate --bundle web --locale fr --json +``` + +Review the saved plan before invoking `config apply`. That invocation requests +the exact config mutation; `--no-input` only disables prompts. A plan hash checks +integrity but is not a signature or authorization grant. Never apply an +unreviewed plan from an untrusted source. + +Apply checks fingerprints, rejects unresolved decisions and unsafe paths, and +uses a per-config lock and atomic replacement. Its receipt includes the plan ID, +changed paths, resulting config fingerprint, and `applied` or `already_applied` +status. It does not translate, approve, or edit catalog files. + +An `already_applied` receipt confirms the config fingerprint only; its +`observations_revalidated: false` is not a fresh source or provider check. Run +`config check` and a dry-run again if sources changed after the first application. + +After a stale-plan error, inspect current state and generate a new proposal. +Do not edit the saved JSON to bypass its fingerprint. After an interrupted apply, +inspect the lock and current config before retrying; do not delete another +process's lock. Cooperative locking protects concurrent CLI applications; it is +not a transaction against arbitrary filesystem writers. + +`translate --dry-run --json` performs no writes or provider calls. It reports +planned keys separately from generated keys and blocked jobs. An explicit syntax +profile resolves grammar ambiguity, but it does not waive placeholder, plural, +or protected-code validation. Explicit ICU remains strict; malformed ICU is not +silently reinterpreted as plain text. + +## JSON, filtering, and failures + +The onboarding commands, `commands`, `translate`, and `validate` use this envelope +when `--json` is supplied: + +```json +{ + "schema_version": 1, + "status": "planned", + "data": {}, + "errors": [] +} +``` + +Parse `schema_version` before consuming command-specific `data`. Errors use the +same envelope, including argument and configuration failures. Stdout contains +JSON only; command diagnostics must not be parsed from human prose. Each error +has a stable `code`, a message, and recovery actions with `argv`, `side_effects`, +and `required_decisions`. Review those decisions before executing a recovery +action. + +`detect` and `config check` put results under `data.inspection`, with `total` and +`matched` counts. Use `--bundle`, `--locale`, `--finding-code`, and `--limit` to +bound output. Their default limit is 50 entries per result section. Translation +defaults to 100 returned jobs; validation defaults to 100 reports/detail items. +Use `--limit 0` for unbounded presentation. Limits and finding filters never +turn an underlying failure into a pass. They do not reduce work already needed +to inspect selected catalogs. + +`validate --json` previously returned an array. It now returns reports under +`data.reports`, with counts and filtering metadata. Update existing consumers +before using this version. + +Exit code 0 means the command completed, not that every required decision was +made. A plan can exit 0 with `needs_decision`. Exit code 1 means execution failed, +configuration checks blocked, or validation failed. Inspect the structured +status and errors as well as the exit code. + +Common errors include `invalid_arguments`, `unknown_bundle`, `unknown_locale`, +`config_invalid`, `stale_plan`, `invalid_plan`, `decisions_required`, +`unsafe_path`, `apply_locked`, `credentials_missing`, and `translation_failed`. +Configuration findings include `UNCOVERED_CATALOG`, +`SOURCE_CONFIRMATION_REQUIRED`, and `AUTO_SYNTAX_AMBIGUOUS`. + +Configuration validity, planned translation, generated translation, structural +validation, and human approval are separate states. A successful provider call +does not establish approval. Translation can report `partial_failure` when some +work succeeded; inspect its jobs and retained state before retrying. Provider +requests are not inherently idempotent, even when completed local work can be +reused. diff --git a/internal/onboarding/discovery.go b/internal/onboarding/discovery.go new file mode 100644 index 0000000..b10f4f3 --- /dev/null +++ b/internal/onboarding/discovery.go @@ -0,0 +1,564 @@ +// Package onboarding exposes offline, evidence-backed configuration discovery. +package onboarding + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/formats" + "github.com/Tom-R-Main/Internationalizer/internal/message" + "github.com/Tom-R-Main/Internationalizer/internal/protectedtext" + "github.com/Tom-R-Main/Internationalizer/internal/validate" + "gopkg.in/yaml.v3" +) + +type Evidence struct { + Path string `json:"path"` + Kind string `json:"kind"` + Detail string `json:"detail"` +} + +type Recovery struct { + Argv []string `json:"argv"` + SideEffects []string `json:"side_effects"` + RequiredDecisions []string `json:"required_decisions,omitempty"` +} + +type Diagnostic struct { + Code string `json:"code"` + Severity string `json:"severity"` + Message string `json:"message"` + Bundle string `json:"bundle,omitempty"` + Key string `json:"key,omitempty"` + Evidence []Evidence `json:"evidence,omitempty"` + RequiredDecisions []string `json:"required_decisions,omitempty"` + Recovery []Recovery `json:"recovery,omitempty"` +} + +type Candidate struct { + ID string `json:"id"` + Source string `json:"source"` + Target string `json:"target"` + Format string `json:"format"` + Framework string `json:"framework"` + SuggestedSyntax message.Syntax `json:"suggested_syntax,omitempty"` + Evidence []Evidence `json:"evidence"` + Uncertainty string `json:"uncertainty"` + ConfiguredBundles []string `json:"configured_bundles"` + RequiresConfirmation bool `json:"requires_confirmation"` +} + +type ResolvedBundle struct { + ID string `json:"id"` + Source string `json:"source"` + Target string `json:"target"` + Format string `json:"format"` + Framework string `json:"framework"` + MessageSyntax message.Syntax `json:"message_syntax"` + Locales []string `json:"locales"` + Targets map[string]string `json:"targets"` + Provenance map[string]string `json:"provenance"` + Evidence []Evidence `json:"evidence"` +} + +type Credential struct { + Locale string `json:"locale"` + Provider string `json:"provider"` + EnvironmentVariable string `json:"environment_variable"` + Present bool `json:"present"` + ProviderVerified bool `json:"provider_verified"` +} + +type Inspection struct { + Root string `json:"root"` + ConfigPath string `json:"config_path"` + ConfigExists bool `json:"config_exists"` + SourceLocale string `json:"source_locale"` + TargetLocales []string `json:"target_locales"` + Candidates []Candidate `json:"candidates"` + Bundles []ResolvedBundle `json:"bundles"` + Diagnostics []Diagnostic `json:"diagnostics"` + Credentials []Credential `json:"credentials"` + Truncated bool `json:"truncated"` +} + +type runtimeEvidence struct { + i18next bool + icu bool + nextIntl bool + vueI18n bool + evidence []Evidence +} + +const maxDiscoveryBytes = 8 << 20 + +// Scan is read-only and never changes the process working directory. Relative +// config and bundle paths resolve against root, matching CLI execution there. +// Framework evidence is static; absence of an ICU import is not proof of absence. +func Scan(root, configPath string) (*Inspection, error) { + root, err := filepath.Abs(root) + if err != nil { + return nil, err + } + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("discovery root is not a directory") + } + root, err = filepath.EvalSymlinks(root) + if err != nil { + return nil, err + } + result := &Inspection{Root: root, Candidates: []Candidate{}, Bundles: []ResolvedBundle{}, Diagnostics: []Diagnostic{}, Credentials: []Credential{}} + result.ConfigPath = resolveConfigPath(root, configPath) + var raw config.Config + data, err := safeRead(result.ConfigPath) + if err == nil { + result.ConfigExists = true + if yaml.Unmarshal(data, &raw) != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "CONFIG_INVALID", Severity: "error", Message: "Configuration YAML could not be parsed; inspect the configuration file."}) + return result, fmt.Errorf("configuration YAML could not be parsed") + } + } else if !errors.Is(err, os.ErrNotExist) { + return result, fmt.Errorf("reading configuration: %w", err) + } else { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "CONFIG_NOT_FOUND", Severity: "warning", Message: "No configuration found; choose authoritative catalogs and target locales before planning configuration."}) + } + cfg := raw + cfg.ApplyDefaults() + result.SourceLocale, result.TargetLocales = cfg.SourceLocale, append([]string{}, cfg.TargetLocales...) + if result.ConfigExists { + if err := cfg.ValidateProject(); err != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "CONFIG_INVALID", Severity: "error", Message: err.Error()}) + } + } + files, runtimes := discoverFiles(root, result) + for _, path := range files { + if supportCatalogPath(root, path, cfg) { + continue + } + target, ok := catalogTarget(path, cfg.SourceLocale) + if !ok { + continue + } + format, formatErr := formats.FormatForFile(path) + if formatErr != nil { + continue + } + content, readErr := safeRead(filepath.Join(root, path)) + if readErr != nil { + result.Truncated = true + continue + } + if _, parseErr := formats.ParseUnits(format, content); parseErr != nil { + continue + } + candidate := Candidate{ID: path, Source: path, Target: target, Format: format.Name(), ConfiguredBundles: []string{}, RequiresConfirmation: temporaryPath(path)} + candidate.Evidence = []Evidence{{Path: path, Kind: "catalog_path", Detail: "Source-locale filename or directory; storage format does not establish message grammar."}} + applyRuntime(&candidate, nearestRuntime(filepath.Dir(path), runtimes)) + result.Candidates = append(result.Candidates, candidate) + } + for i, bundle := range cfg.EffectiveBundles() { + resolved := inspectBundle(root, bundle, cfg, raw, i, runtimes, result) + result.Bundles = append(result.Bundles, resolved) + source := relativePath(root, resolved.Source) + found := false + for j := range result.Candidates { + if result.Candidates[j].Source == source { + result.Candidates[j].ConfiguredBundles = append(result.Candidates[j].ConfiguredBundles, bundle.ID) + found = true + } + } + if !found { + candidate := Candidate{ID: source, Source: source, Target: relativePath(root, resolved.Target), Format: resolved.Format, ConfiguredBundles: []string{bundle.ID}, RequiresConfirmation: temporaryPath(source), Evidence: []Evidence{{Path: relativePath(root, result.ConfigPath), Kind: "configuration", Detail: "Existing configured source."}}} + applyRuntime(&candidate, nearestRuntime(filepath.Dir(source), runtimes)) + result.Candidates = append(result.Candidates, candidate) + } + } + for _, candidate := range result.Candidates { + if len(candidate.ConfiguredBundles) == 0 { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "UNCOVERED_CATALOG", Severity: "warning", Message: "Detected catalog is not covered by the current configuration: " + candidate.Source, Evidence: candidate.Evidence, RequiredDecisions: []string{"select_authoritative_source", "select_message_syntax"}, Recovery: []Recovery{{Argv: []string{"internationalizer", "config", "plan", "--help"}, SideEffects: []string{}}}}) + } + } + for _, locale := range cfg.TargetLocales { + provider := cfg.LLMForLocale(locale) + result.Credentials = append(result.Credentials, Credential{Locale: locale, Provider: provider.Provider, EnvironmentVariable: provider.APIKeyEnv, Present: os.Getenv(provider.APIKeyEnv) != "", ProviderVerified: false}) + } + sort.Slice(result.Candidates, func(i, j int) bool { return result.Candidates[i].ID < result.Candidates[j].ID }) + if result.Truncated { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "DISCOVERY_TRUNCATED", Severity: "warning", Message: "Discovery reached its file, depth, or size limit; narrow the project root. Unobserved integrations may exist."}) + } + for i := range result.Diagnostics { + diagnostic := &result.Diagnostics[i] + if len(diagnostic.RequiredDecisions) > 0 { + diagnostic.Recovery = []Recovery{{Argv: []string{"internationalizer", "config", "plan", "--help"}, SideEffects: []string{}, RequiredDecisions: append([]string{}, diagnostic.RequiredDecisions...)}} + } + } + return result, nil +} + +// Support documents inform translation policy; locale-shaped filenames alone +// do not make them translation catalogs. Explicit bundles are added separately. +func supportCatalogPath(root, path string, cfg config.Config) bool { + for _, dir := range []string{cfg.StyleGuidesDir, cfg.GlossaryDir} { + if dir == "" { + continue + } + relative, err := filepath.Rel(absolutePath(root, dir), absolutePath(root, path)) + if err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return true + } + } + return false +} + +func resolveConfigPath(root, explicit string) string { + if explicit != "" { + return absolutePath(root, explicit) + } + paths := []string{filepath.Join(root, ".internationalizer.yml"), filepath.Join(root, ".internationalizer.yaml")} + if home, err := os.UserHomeDir(); err == nil { + paths = append(paths, filepath.Join(home, ".internationalizer.yml"), filepath.Join(home, ".internationalizer.yaml")) + } + for _, path := range paths { + if _, err := os.Lstat(path); err == nil { + return path + } + } + return paths[0] +} + +func absolutePath(root, path string) string { + if filepath.IsAbs(path) { + return filepath.Clean(path) + } + return filepath.Join(root, path) +} + +func relativePath(root, path string) string { + rel, err := filepath.Rel(root, path) + if err != nil { + return filepath.ToSlash(path) + } + return filepath.ToSlash(rel) +} + +func safeRead(path string) ([]byte, error) { + name := strings.ToLower(filepath.Base(path)) + extension := strings.ToLower(filepath.Ext(path)) + if strings.HasPrefix(name, ".env") || extension == ".pem" || extension == ".p12" || extension == ".key" { + return nil, fmt.Errorf("secret-shaped files are not inspected: %s", path) + } + // Check every component: a regular leaf beneath a symlink is still a symlink + // traversal, and discovery must not escape through linked directories. + for current := filepath.Clean(path); ; current = filepath.Dir(current) { + info, err := os.Lstat(current) + if err != nil { + return nil, err + } + if info.Mode()&os.ModeSymlink != 0 { + return nil, fmt.Errorf("symlink paths are not inspected: %s", path) + } + if current == filepath.Dir(current) { + break + } + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("not a regular file: %s", path) + } + if info.Size() > maxDiscoveryBytes { + return nil, fmt.Errorf("file exceeds discovery size limit: %s", path) + } + return os.ReadFile(path) +} + +func skippedDirectory(name string) bool { + switch name { + case "node_modules", "build", "dist", "vendor", "coverage", "__pycache__", "data": + return true + } + return strings.HasPrefix(name, ".") +} + +func discoverFiles(root string, result *Inspection) ([]string, map[string]runtimeEvidence) { + var files []string + runtimes := map[string]runtimeEvidence{} + var imports []Evidence + count := 0 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "DISCOVERY_PATH_UNREADABLE", Severity: "warning", Message: "A project path could not be inspected.", Evidence: []Evidence{{Path: relativePath(root, path), Kind: "unreadable", Detail: "Check filesystem access."}}}) + return nil + } + if path == root { + return nil + } + count++ + if count > 50000 { + result.Truncated = true + return fs.SkipAll + } + rel := relativePath(root, path) + if entry.Type()&os.ModeSymlink != 0 { + return nil + } + if entry.IsDir() { + if skippedDirectory(entry.Name()) { + return fs.SkipDir + } + if strings.Count(rel, "/") >= 14 { + result.Truncated = true + return fs.SkipDir + } + return nil + } + if strings.HasPrefix(entry.Name(), ".") { + return nil + } + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".json", ".yaml", ".yml", ".ftl", ".md", ".mdx": + files = append(files, rel) + } + if entry.Name() == "package.json" { + content, readErr := safeRead(path) + if readErr != nil { + result.Truncated = true + return nil + } + var pkg struct { + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` + } + if json.Unmarshal(content, &pkg) != nil { + return nil + } + runtime := runtimeEvidence{evidence: []Evidence{{Path: rel, Kind: "package_manifest", Detail: "Nearest package dependency declarations inspected; runtime registration may be dynamic."}}} + for _, deps := range []map[string]string{pkg.Dependencies, pkg.DevDependencies, pkg.PeerDependencies} { + for _, name := range []string{"i18next", "react-i18next", "i18next-icu", "next-intl", "vue-i18n"} { + if _, ok := deps[name]; ok { + switch name { + case "i18next-icu": + runtime.icu = true + case "next-intl": + runtime.nextIntl = true + case "vue-i18n": + runtime.vueI18n = true + default: + runtime.i18next = true + } + runtime.evidence = append(runtime.evidence, Evidence{Path: rel, Kind: "dependency", Detail: name}) + } + } + } + runtimes[filepath.ToSlash(filepath.Dir(rel))] = runtime + } + // Keep source inspection bounded and targeted to localization modules. + localizationPath := strings.Contains(strings.ToLower(rel), "i18n") || strings.Contains(strings.ToLower(rel), "locale") + if localizationPath && (ext == ".ts" || ext == ".tsx" || ext == ".js" || ext == ".jsx" || ext == ".mjs") { + info, infoErr := entry.Info() + if infoErr != nil || info.Size() > 256<<10 { + result.Truncated = true + return nil + } + content, readErr := safeRead(path) + if readErr == nil && strings.Contains(string(content), "i18next-icu") { + imports = append(imports, Evidence{Path: rel, Kind: "runtime_reference", Detail: "i18next-icu reference; verify plugin registration for this catalog."}) + } + } + return nil + }) + if err != nil { + result.Truncated = true + } + for _, evidence := range imports { + dir := nearestRuntimeDirectory(filepath.Dir(evidence.Path), runtimes) + if dir != "" { + runtime := runtimes[dir] + runtime.icu = true + runtime.evidence = append(runtime.evidence, evidence) + runtimes[dir] = runtime + } + } + return files, runtimes +} + +func nearestRuntimeDirectory(dir string, runtimes map[string]runtimeEvidence) string { + if filepath.IsAbs(dir) || dir == ".." || strings.HasPrefix(filepath.ToSlash(dir), "../") { + return "" + } + for { + dir = filepath.ToSlash(dir) + if _, ok := runtimes[dir]; ok { + return dir + } + parent := filepath.ToSlash(filepath.Dir(dir)) + if parent == dir || dir == "." { + return "" + } + dir = parent + } +} + +func nearestRuntime(dir string, runtimes map[string]runtimeEvidence) runtimeEvidence { + return runtimes[nearestRuntimeDirectory(dir, runtimes)] +} + +func applyRuntime(candidate *Candidate, runtime runtimeEvidence) { + candidate.Framework = "unknown" + candidate.SuggestedSyntax = "" + candidate.Uncertainty = "high: storage and path evidence do not identify the runtime message grammar" + candidate.Evidence = append(candidate.Evidence, runtime.evidence...) + var frameworks []string + if runtime.i18next || runtime.icu { + frameworks = append(frameworks, "i18next") + } + if runtime.nextIntl { + frameworks = append(frameworks, "next-intl") + } + if runtime.vueI18n { + frameworks = append(frameworks, "vue-i18n") + } + if len(frameworks) > 1 { + candidate.Framework = "multiple" + candidate.Uncertainty = "high: multiple frameworks detected (" + strings.Join(frameworks, ", ") + "); explicitly choose this catalog's runtime and a compatible message syntax profile" + return + } + if runtime.nextIntl { + candidate.Framework = "next-intl" + candidate.SuggestedSyntax = message.ICU + candidate.Uncertainty = "medium: next-intl dependency suggests ICU messages; static dependency evidence does not prove that this catalog is consumed by next-intl" + } + if runtime.vueI18n { + candidate.Framework = "vue-i18n" + candidate.Uncertainty = "high: vue-i18n has its own message grammar; explicitly confirm compatibility and select a supported syntax profile before translation" + } + if runtime.i18next || runtime.icu { + candidate.Framework = "i18next" + candidate.SuggestedSyntax = message.I18next + candidate.Uncertainty = "medium: i18next dependency detected; static inspection cannot prove that ICU integration is absent" + if runtime.icu { + candidate.SuggestedSyntax = "" + candidate.Uncertainty = "high: i18next-icu evidence found; select i18next or icu after confirming runtime plugin registration" + } + } +} + +func catalogTarget(path, locale string) (string, bool) { + ext := filepath.Ext(path) + if strings.TrimSuffix(filepath.Base(path), ext) == locale { + return filepath.ToSlash(filepath.Join(filepath.Dir(path), "{locale}"+ext)), true + } + parts := strings.Split(filepath.ToSlash(path), "/") + for i := len(parts) - 2; i >= 0; i-- { + if parts[i] == locale { + parts[i] = "{locale}" + return strings.Join(parts, "/"), true + } + } + return "", false +} + +func temporaryPath(path string) bool { + for _, part := range strings.Split(filepath.ToSlash(path), "/") { + if part == "tmp" || part == "temp" { + return true + } + } + return false +} + +func inspectBundle(root string, bundle config.Bundle, cfg, raw config.Config, index int, runtimes map[string]runtimeEvidence, result *Inspection) ResolvedBundle { + source, target := absolutePath(root, bundle.Source), absolutePath(root, bundle.Target) + format, err := formats.FormatForFile(source) + if bundle.Format != "" { + format, err = formats.FormatByName(bundle.Format) + } + formatName := bundle.Format + if err == nil { + formatName = format.Name() + } + provenance := map[string]string{"source": "bundle.source", "target": "bundle.target", "locales": "target_locales", "message_syntax": "default:auto", "format": "source_extension", "path_base": "working_directory"} + if len(raw.Bundles) == 0 { + provenance["source"] = "source_path" + provenance["target"] = "source_path sibling convention" + } + if raw.MessageSyntax != "" { + provenance["message_syntax"] = "message_syntax" + } + if index < len(raw.Bundles) && raw.Bundles[index].MessageSyntax != "" { + provenance["message_syntax"] = "bundle.message_syntax" + } + if bundle.Format != "" { + provenance["format"] = "bundle.format" + } + candidate := Candidate{} + applyRuntime(&candidate, nearestRuntime(filepath.Dir(relativePath(root, source)), runtimes)) + resolved := ResolvedBundle{ID: bundle.ID, Source: source, Target: target, Format: formatName, Framework: candidate.Framework, MessageSyntax: bundle.MessageSyntax, Locales: append([]string{}, cfg.TargetLocales...), Targets: map[string]string{}, Provenance: provenance, Evidence: candidate.Evidence} + for _, locale := range cfg.TargetLocales { + if path, pathErr := bundle.TargetPath(locale); pathErr == nil { + resolved.Targets[locale] = absolutePath(root, path) + } + } + if temporaryPath(relativePath(root, source)) { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "SOURCE_CONFIRMATION_REQUIRED", Severity: "warning", Bundle: bundle.ID, Message: "Configured source is under tmp/ or temp/; confirm which pipeline owns this artifact before changing the configuration.", RequiredDecisions: []string{"confirm_authoritative_source"}, Evidence: []Evidence{{Path: relativePath(root, source), Kind: "temporary_path", Detail: "Temporary location requires confirmation; it is not automatically incorrect."}}}) + } + if err != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "SOURCE_FORMAT_INVALID", Severity: "error", Bundle: bundle.ID, Message: err.Error()}) + return resolved + } + data, err := safeRead(source) + if err != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "SOURCE_UNREADABLE", Severity: "error", Bundle: bundle.ID, Message: err.Error()}) + return resolved + } + units, err := formats.ParseSourceUnits(format, data, bundle.MessageSyntax) + if err != nil { + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "SOURCE_PARSE_FAILED", Severity: "error", Bundle: bundle.ID, Message: "Configured source could not be parsed using its storage format."}) + return resolved + } + for _, unit := range units { + findings := validate.SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax) + codeBraces := false + for _, span := range protectedtext.HTMLCode(unit.Value) { + if strings.Contains(span, "{") { + codeBraces = true + break + } + } + auto := bundle.MessageSyntax == message.Auto || bundle.MessageSyntax == "" + if len(findings) == 0 && (!auto || unit.Syntax != message.ICU || !codeBraces) { + continue + } + code, text := "SOURCE_SYNTAX_INVALID", "Source does not satisfy its explicit message syntax." + severity := "error" + if len(findings) == 0 { + severity = "warning" + } + var decisions []string + if auto { + code = "AUTO_SYNTAX_AMBIGUOUS" + text = "With message_syntax: auto, brace syntax was interpreted as ICU. Select the bundle's runtime syntax: plain, i18next, or icu." + if codeBraces { + text = "Brace syntax inside HTML code was interpreted as ICU with message_syntax: auto. Select the bundle's runtime syntax: plain, i18next, or icu." + } + decisions = []string{"select_message_syntax"} + } + result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: code, Severity: severity, Bundle: bundle.ID, Key: unit.ID, Message: text, RequiredDecisions: decisions, Evidence: []Evidence{{Path: relativePath(root, source), Kind: "source_syntax", Detail: "Source-only check; not repeated for each target locale."}}}) + } + return resolved +} diff --git a/internal/onboarding/discovery_test.go b/internal/onboarding/discovery_test.go new file mode 100644 index 0000000..ff0d840 --- /dev/null +++ b/internal/onboarding/discovery_test.go @@ -0,0 +1,329 @@ +package onboarding + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/message" +) + +func discoveryFile(t *testing.T, root, path, content string) { + t.Helper() + path = filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestScanMonorepo(t *testing.T) { + root := t.TempDir() + t.Setenv("ONBOARDING_TEST_KEY", "sensitive-value-must-not-appear") + discoveryFile(t, root, ".internationalizer.yml", `source_locale: en +target_locales: [fr, ja] +source_path: tmp/english-keys.json +llm: + provider: openai + api_key_env: ONBOARDING_TEST_KEY + locale_overrides: + ja: + provider: gemini + api_key_env: ONBOARDING_TEST_MISSING_KEY +`) + discoveryFile(t, root, "tmp/english-keys.json", `{"docs.tui.skills.desc":"<root>/{.sift,.claude,.codex,.agents}/skills"}`) + discoveryFile(t, root, "web/package.json", `{"dependencies":{"i18next":"1","react-i18next":"1"}}`) + discoveryFile(t, root, "web/src/i18n/locales/en.json", `{"welcome":"Hello {{name}}"}`) + discoveryFile(t, root, "node_modules/fake/en.json", `{"bad":"must not be found"}`) + inspection, err := Scan(root, ".internationalizer.yml") + if err != nil { + t.Fatal(err) + } + if !inspection.ConfigExists || len(inspection.Candidates) != 2 || len(inspection.Bundles) != 1 { + t.Fatalf("unexpected inspection: %+v", inspection) + } + bundle := inspection.Bundles[0] + if bundle.ID != "default" || bundle.MessageSyntax != message.Auto || bundle.Provenance["source"] != "source_path" { + t.Fatalf("bundle = %+v", bundle) + } + if !filepath.IsAbs(bundle.Source) || !strings.HasSuffix(bundle.Targets["ja"], filepath.Join("tmp", "ja.json")) { + t.Fatalf("paths = %+v", bundle) + } + var web Candidate + for _, candidate := range inspection.Candidates { + if strings.HasPrefix(candidate.Source, "web/") { + web = candidate + } + } + if web.Framework != "i18next" || web.SuggestedSyntax != message.I18next || len(web.ConfiguredBundles) != 0 || len(web.Evidence) < 2 { + t.Fatalf("web = %+v", web) + } + counts := map[string]int{} + for _, diagnostic := range inspection.Diagnostics { + counts[diagnostic.Code]++ + } + for _, code := range []string{"SOURCE_CONFIRMATION_REQUIRED", "AUTO_SYNTAX_AMBIGUOUS", "UNCOVERED_CATALOG"} { + if counts[code] != 1 { + t.Fatalf("%s count = %d; diagnostics = %+v", code, counts[code], inspection.Diagnostics) + } + } + if !inspection.Credentials[0].Present || inspection.Credentials[1].Present || inspection.Credentials[0].ProviderVerified { + t.Fatalf("credentials = %+v", inspection.Credentials) + } + encoded, err := json.Marshal(inspection) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), "sensitive-value") { + t.Fatal("credential value leaked") + } +} + +func TestScanICUIntegrationAndNamespace(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: app/locales/en/common.json\nmessage_syntax: icu\n") + discoveryFile(t, root, "app/package.json", `{"dependencies":{"i18next":"1","i18next-icu":"1"}}`) + discoveryFile(t, root, "app/locales/en/common.json", `{"count":"{count, plural, one {One} other {Many}}"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + if len(inspection.Candidates) != 1 { + t.Fatalf("candidates = %+v", inspection.Candidates) + } + candidate := inspection.Candidates[0] + if candidate.Target != "app/locales/{locale}/common.json" || candidate.SuggestedSyntax != "" || !strings.Contains(candidate.Uncertainty, "plugin registration") { + t.Fatalf("candidate = %+v", candidate) + } + if inspection.Bundles[0].MessageSyntax != message.ICU { + t.Fatal("explicit ICU changed") + } +} + +func TestScanNoConfigAndRuntimeReference(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, "package.json", `{"dependencies":{"i18next":"1"}}`) + discoveryFile(t, root, "src/i18n.ts", `import ICU from 'i18next-icu';`) + discoveryFile(t, root, "locales/en.json", `{"hello":"Hello"}`) + inspection, err := Scan(root, "absent.yml") + if err != nil { + t.Fatal(err) + } + if inspection.ConfigExists || inspection.SourceLocale != "en" || len(inspection.Candidates) != 1 { + t.Fatalf("inspection = %+v", inspection) + } + if inspection.Candidates[0].SuggestedSyntax != "" { + t.Fatal("runtime ICU reference ignored") + } +} + +func TestScanInvalidConfigDoesNotEchoValues(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, "broken.yml", "batch_size: super-secret-invalid-value\n") + inspection, err := Scan(root, "broken.yml") + if err == nil || strings.Contains(err.Error(), "super-secret") { + t.Fatalf("error = %v", err) + } + if len(inspection.Diagnostics) != 1 || inspection.Diagnostics[0].Code != "CONFIG_INVALID" { + t.Fatalf("inspection = %+v", inspection) + } +} + +func TestScanSymlinksExcluded(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: linked/en.json\n") + other := t.TempDir() + discoveryFile(t, other, "en.json", `{"hello":"Hello"}`) + if err := os.Symlink(other, filepath.Join(root, "linked")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + var unreadable bool + for _, diagnostic := range inspection.Diagnostics { + unreadable = unreadable || diagnostic.Code == "SOURCE_UNREADABLE" + } + if !unreadable { + t.Fatal("configured symlink should not be read") + } +} + +func TestScanExplicitICUStrict(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr, ja]\nsource_path: en.json\nmessage_syntax: icu\n") + discoveryFile(t, root, "en.json", `{"bad":"{oops, invalid}"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + count := 0 + for _, diagnostic := range inspection.Diagnostics { + if diagnostic.Code == "SOURCE_SYNTAX_INVALID" { + count++ + } + if diagnostic.Code == "AUTO_SYNTAX_AMBIGUOUS" { + t.Fatal("explicit ICU downgraded") + } + } + if count != 1 { + t.Fatalf("diagnostics = %+v", inspection.Diagnostics) + } +} + +func TestScanAutoCodeBracesWarnEvenWhenICUParses(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "target_locales: [fr]\nsource_path: en.json\n") + discoveryFile(t, root, "en.json", `{"code":"Run {command}"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + for _, diagnostic := range inspection.Diagnostics { + if diagnostic.Code == "AUTO_SYNTAX_AMBIGUOUS" { + if diagnostic.Severity != "warning" || diagnostic.Key != "code" { + t.Fatalf("diagnostic = %+v", diagnostic) + } + return + } + } + t.Fatalf("missing ambiguity warning: %+v", inspection.Diagnostics) +} + +func TestScanUsesRootNotConfigDirectory(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, "configuration/project.yml", "target_locales: [fr]\nmessage_syntax: plain\nbundles:\n - id: web\n source: catalogs/en.json\n target: catalogs/{locale}.json\n message_syntax: i18next\n") + discoveryFile(t, root, "catalogs/en.json", `{"hello":"Hello {{name}}"}`) + inspection, err := Scan(root, "configuration/project.yml") + if err != nil { + t.Fatal(err) + } + if len(inspection.Bundles) != 1 { + t.Fatalf("bundles = %+v", inspection.Bundles) + } + bundle := inspection.Bundles[0] + if bundle.Source != filepath.Join(inspection.Root, "catalogs", "en.json") || bundle.Provenance["message_syntax"] != "bundle.message_syntax" || bundle.MessageSyntax != message.I18next { + t.Fatalf("bundle = %+v", bundle) + } + for _, diagnostic := range inspection.Diagnostics { + if diagnostic.Severity == "error" { + t.Fatalf("unexpected diagnostic = %+v", diagnostic) + } + } +} + +func TestScanRejectsSecretShapedSource(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "target_locales: [fr]\nbundles:\n - id: bad\n source: .env.json\n target: fr.json\n format: json\n") + discoveryFile(t, root, ".env.json", `{"TOKEN":"must-not-be-read"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + for _, diagnostic := range inspection.Diagnostics { + if diagnostic.Code == "SOURCE_UNREADABLE" { + return + } + } + t.Fatal("secret-shaped configured source was not rejected") +} + +func TestScanInvalidRoot(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, "file", "not a directory") + if _, err := Scan(filepath.Join(root, "file"), ""); err == nil { + t.Fatal("expected non-directory error") + } + if _, err := Scan(filepath.Join(root, "missing"), ""); err == nil { + t.Fatal("expected missing root error") + } +} + +func TestScanExcludesConfiguredSupportDirectoriesButKeepsExplicitBundles(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "target_locales: [fr]\nstyle_guides_dir: docs/i18n-style-guides\nglossary_dir: terms\nbundles:\n - id: intentional-guide\n source: docs/i18n-style-guides/published/en.md\n target: docs/i18n-style-guides/published/{locale}.md\n") + discoveryFile(t, root, "docs/i18n-style-guides/en.md", "# English style guide\n") + discoveryFile(t, root, "docs/i18n-style-guides/microsoft/en.md", "# Microsoft English guide\n") + discoveryFile(t, root, "docs/i18n-style-guides/published/en.md", "# Intentionally translated guide\n") + discoveryFile(t, root, "terms/en.json", `{"term":"Definition"}`) + discoveryFile(t, root, "terms-extra/en.json", `{"hello":"Hello"}`) + discoveryFile(t, root, "app/locales/en.json", `{"hello":"Hello"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + if len(inspection.Candidates) != 3 { + t.Fatalf("candidates = %+v", inspection.Candidates) + } + expected := map[string]bool{"docs/i18n-style-guides/published/en.md": true, "terms-extra/en.json": true, "app/locales/en.json": true} + for _, candidate := range inspection.Candidates { + if !expected[candidate.Source] { + t.Fatalf("unexpected inferred support catalog = %+v", candidate) + } + if candidate.Source == "docs/i18n-style-guides/published/en.md" && (len(candidate.ConfiguredBundles) != 1 || candidate.ConfiguredBundles[0] != "intentional-guide") { + t.Fatalf("explicit bundle missing = %+v", candidate) + } + } +} + +func TestScanConfigurationDecisionsHaveSafeRecovery(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "target_locales: [fr]\nsource_path: tmp/english-keys.json\n") + discoveryFile(t, root, "tmp/english-keys.json", `{"code":"{a,b,c}"}`) + discoveryFile(t, root, "web/locales/en.json", `{"hello":"Hello"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + count := 0 + for _, diagnostic := range inspection.Diagnostics { + if len(diagnostic.RequiredDecisions) == 0 { + continue + } + count++ + if len(diagnostic.Recovery) != 1 { + t.Fatalf("missing recovery = %+v", diagnostic) + } + recovery := diagnostic.Recovery[0] + if strings.Join(recovery.Argv, " ") != "internationalizer config plan --help" || len(recovery.SideEffects) != 0 || strings.Join(recovery.RequiredDecisions, ",") != strings.Join(diagnostic.RequiredDecisions, ",") { + t.Fatalf("unsafe or incomplete recovery = %+v", diagnostic) + } + } + if count != 3 { + t.Fatalf("expected 3 decisions, got %d: %+v", count, inspection.Diagnostics) + } +} + +func TestScanOtherFrameworksAndMixedRuntimeEvidence(t *testing.T) { + for _, tc := range []struct { + name, dependencies, framework, uncertainty string + syntax message.Syntax + }{ + {name: "next-intl", dependencies: `"next-intl":"1"`, framework: "next-intl", syntax: message.ICU, uncertainty: "static"}, + {name: "vue-i18n", dependencies: `"vue-i18n":"1"`, framework: "vue-i18n", uncertainty: "compatibility"}, + {name: "mixed-next-i18next", dependencies: `"next-intl":"1","i18next":"1"`, framework: "multiple", uncertainty: "multiple frameworks"}, + {name: "mixed-vue-next", dependencies: `"vue-i18n":"1","next-intl":"1"`, framework: "multiple", uncertainty: "multiple frameworks"}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, "app/package.json", `{"dependencies":{`+tc.dependencies+`}}`) + discoveryFile(t, root, "app/messages/en.json", `{"hello":"Hello"}`) + inspection, err := Scan(root, "absent.yml") + if err != nil { + t.Fatal(err) + } + if len(inspection.Candidates) != 1 { + t.Fatalf("candidates = %+v", inspection.Candidates) + } + candidate := inspection.Candidates[0] + if candidate.Framework != tc.framework || candidate.SuggestedSyntax != tc.syntax || !strings.Contains(candidate.Uncertainty, tc.uncertainty) { + t.Fatalf("candidate = %+v", candidate) + } + }) + } +} diff --git a/internal/onboarding/plan.go b/internal/onboarding/plan.go new file mode 100644 index 0000000..a8ad9a4 --- /dev/null +++ b/internal/onboarding/plan.go @@ -0,0 +1,748 @@ +package onboarding + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/message" + "gopkg.in/yaml.v3" +) + +const planVersion = 1 +const maxPlanConfigSize = 2 << 20 + +// PlanOptions contains explicit user decisions; discovery alone never selects a +// new bundle or overwrites an existing bundle's runtime profile. +type PlanOptions struct { + AddBundles []config.Bundle + Syntax map[string]message.Syntax + ConfirmSources []string + SourceLocale string + TargetLocales []string +} + +type PlanDecision struct { + Code string `json:"code"` + Bundle string `json:"bundle,omitempty"` + Source string `json:"source,omitempty"` + Message string `json:"message"` +} + +// Observation binds the proposal to the source and runtime evidence it used. +// Missing paths are recorded too, so newly-created evidence invalidates a plan. +type Observation struct { + Path string `json:"path"` + Exists bool `json:"exists"` + SHA256 string `json:"sha256,omitempty"` +} + +// ConfigPlan is a reviewable, versioned proposal, not an authorization token. +// Its content hash detects accidental modification, not a malicious signer. +type ConfigPlan struct { + SchemaVersion int `json:"schema_version"` + ID string `json:"id"` + Root string `json:"root"` + ConfigPath string `json:"config_path"` + BeforeExists bool `json:"before_exists"` + BeforeSHA256 string `json:"before_sha256,omitempty"` + AfterSHA256 string `json:"after_sha256"` + DiscoverySHA256 string `json:"discovery_sha256"` + ProposedYAML string `json:"proposed_yaml"` + Diff string `json:"diff"` + Observations []Observation `json:"observations"` + RequiredDecisions []PlanDecision `json:"required_decisions"` +} + +type ApplyReceipt struct { + SchemaVersion int `json:"schema_version"` + PlanID string `json:"plan_id"` + Status string `json:"status"` + ChangedPaths []string `json:"changed_paths"` + ConfigPath string `json:"config_path"` + ConfigSHA256 string `json:"config_sha256"` + ObservationsRevalidated bool `json:"observations_revalidated"` +} + +type PlanError struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (e *PlanError) Error() string { return e.Message } +func (e *PlanError) JSONCode() string { return e.Code } + +func planError(code, message string) error { return &PlanError{Code: code, Message: message} } + +// BuildPlan reads project state and returns an immutable proposal. It performs +// no writes, provider calls or credential materialization. +func BuildPlan(root, configPath string, options PlanOptions) (*ConfigPlan, error) { + root, err := canonicalPlanRoot(root) + if err != nil { + return nil, err + } + configPath, err = planConfigPath(root, configPath) + if err != nil { + return nil, err + } + before, exists, err := readPlanFile(root, configPath) + if err != nil { + return nil, err + } + if len(before) > maxPlanConfigSize { + return nil, planError("config_too_large", "configuration exceeds the plan size limit") + } + doc := &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Kind: yaml.MappingNode, Tag: "!!map"}}} + if exists { + if err := yaml.Unmarshal(before, doc); err != nil { + return nil, planError("invalid_config", "configuration is not valid YAML") + } + } + if len(doc.Content) != 1 || doc.Content[0].Kind != yaml.MappingNode { + return nil, planError("invalid_config", "configuration must be a YAML mapping") + } + if err := inspectPlanYAML(doc); err != nil { + return nil, err + } + var cfg config.Config + if err := doc.Decode(&cfg); err != nil { + return nil, planError("invalid_config", "configuration fields have invalid types") + } + cfg.ApplyDefaults() + inspection, err := Scan(root, configPath) + if err != nil { + return nil, err + } + content := doc.Content[0] + if options.SourceLocale != "" { + setPlanScalar(content, "source_locale", options.SourceLocale) + } + if len(options.TargetLocales) > 0 { + values := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for _, locale := range options.TargetLocales { + values.Content = append(values.Content, planScalar(locale)) + } + setPlanNode(content, "target_locales", values) + } + if !exists && options.SourceLocale == "" { + setPlanScalar(content, "source_locale", "en") + } + if len(options.AddBundles) > 0 || len(options.Syntax) > 0 { + if len(cfg.Bundles) == 0 && cfg.SourcePath != "" { + bundles := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for _, b := range cfg.EffectiveBundles() { + node := planBundleNode(b) + // Carry source_path comments onto its explicit bundle equivalent. + if source := getPlanNode(content, "source_path"); source != nil { + copy := *source + setPlanNode(node, "source", ©) + getPlanNode(node, "source").HeadComment = source.HeadComment + getPlanNode(node, "source").LineComment = source.LineComment + getPlanNode(node, "source").FootComment = source.FootComment + if key := getPlanKeyNode(content, "source_path"); key != nil { + newKey := getPlanKeyNode(node, "source") + newKey.HeadComment, newKey.LineComment, newKey.FootComment = key.HeadComment, key.LineComment, key.FootComment + } + } + bundles.Content = append(bundles.Content, node) + } + setPlanNode(content, "bundles", bundles) + removePlanKey(content, "source_path") + } + } + bundleNodes := getPlanNode(content, "bundles") + if bundleNodes == nil && len(options.AddBundles) > 0 { + bundleNodes = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + setPlanNode(content, "bundles", bundleNodes) + } + seen := map[string]*yaml.Node{} + if bundleNodes != nil { + if bundleNodes.Kind != yaml.SequenceNode { + return nil, planError("invalid_config", "bundles must be a sequence") + } + for _, b := range bundleNodes.Content { + id := getPlanNode(b, "id") + if id == nil || id.Value == "" || seen[id.Value] != nil { + return nil, planError("invalid_config", "existing bundle identities must be nonempty and unique") + } + seen[id.Value] = b + } + } + additions := append([]config.Bundle(nil), options.AddBundles...) + sort.Slice(additions, func(i, j int) bool { return additions[i].ID < additions[j].ID }) + for _, b := range additions { + if b.ID == "" || seen[b.ID] != nil { + return nil, planError("invalid_decision", "added bundle identity must be nonempty and not already configured") + } + if b.Source == "" || b.Target == "" { + return nil, planError("invalid_decision", "added bundles require explicit source and target paths") + } + if _, err := safePlanPath(root, b.Source); err != nil { + return nil, err + } + if _, err := safePlanPath(root, b.Target); err != nil { + return nil, err + } + node := planBundleNode(b) + bundleNodes.Content = append(bundleNodes.Content, node) + seen[b.ID] = node + } + for id, syntax := range options.Syntax { + if seen[id] == nil { + return nil, planError("invalid_decision", "syntax selection references an unknown bundle: "+id) + } + if err := message.ValidateSyntax(syntax); err != nil || syntax == "" { + return nil, planError("invalid_decision", "syntax selection must be auto, plain, i18next, or icu") + } + setPlanScalar(seen[id], "message_syntax", string(syntax)) + } + var proposed config.Config + if err := doc.Decode(&proposed); err != nil { + return nil, planError("invalid_config", "proposed configuration has invalid field types") + } + proposed.ApplyDefaults() + plan := &ConfigPlan{SchemaVersion: planVersion, Root: root, ConfigPath: configPath, BeforeExists: exists, Observations: []Observation{}, RequiredDecisions: []PlanDecision{}} + plan.DiscoverySHA256, err = discoveryPlanDigest(inspection) + if err != nil { + return nil, err + } + if exists { + plan.BeforeSHA256 = planHash(before) + } + if len(proposed.TargetLocales) == 0 { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "TARGET_LOCALES_REQUIRED", Message: "Select target locales before applying the configuration."}) + } + if len(proposed.EffectiveBundles()) == 0 { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "BUNDLE_SELECTION_REQUIRED", Message: "Select an authoritative source catalog and target template."}) + } + if len(proposed.TargetLocales) > 0 && len(proposed.EffectiveBundles()) > 0 { + if err := proposed.ValidateProject(); err != nil { + return nil, planError("invalid_config", "proposed configuration: "+err.Error()) + } + } + confirmed := map[string]bool{} + for _, source := range options.ConfirmSources { + path, err := safePlanPath(root, source) + if err != nil { + return nil, err + } + confirmed[path] = true + } + paths := map[string]bool{} + for _, b := range proposed.EffectiveBundles() { + source, err := safePlanPath(root, b.Source) + if err != nil { + return nil, err + } + paths[source] = true + if _, exists, err := readPlanFile(root, source); err != nil { + return nil, err + } else if !exists { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "SOURCE_NOT_FOUND", Bundle: b.ID, Source: b.Source, Message: "The selected source catalog does not exist; choose an existing authoritative path."}) + } + for _, locale := range proposed.TargetLocales { + target, err := b.TargetPath(locale) + if err != nil { + return nil, planError("invalid_config", err.Error()) + } + if _, err := safePlanPath(root, target); err != nil { + return nil, err + } + } + if temporaryPlanPath(root, source) && !confirmed[source] { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "SOURCE_CONFIRMATION_REQUIRED", Bundle: b.ID, Source: b.Source, Message: "Confirm that this temporary-path catalog is the authoritative source; temporary paths are not automatically invalid."}) + } + for _, d := range inspection.Diagnostics { + if d.Code == "AUTO_SYNTAX_AMBIGUOUS" && d.Bundle == b.ID && b.MessageSyntax == message.Auto { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "SYNTAX_SELECTION_REQUIRED", Bundle: b.ID, Source: b.Source, Message: "Select plain, i18next, or icu for ambiguous automatic message syntax."}) + break + } + } + } + for _, b := range additions { + selected := b.MessageSyntax + if syntax, ok := options.Syntax[b.ID]; ok { + selected = syntax + } + intrinsicFluent := strings.EqualFold(b.Format, "fluent") || strings.EqualFold(filepath.Ext(b.Source), ".ftl") + if !intrinsicFluent && (selected == "" || selected == message.Auto) { + plan.RequiredDecisions = append(plan.RequiredDecisions, PlanDecision{Code: "SYNTAX_SELECTION_REQUIRED", Bundle: b.ID, Source: b.Source, Message: "Explicitly choose the new bundle's runtime syntax from the discovery evidence."}) + } + } + for _, candidate := range inspection.Candidates { + paths[candidate.Source] = true + for _, evidence := range candidate.Evidence { + if evidence.Path != "" { + paths[evidence.Path] = true + } + } + } + for _, b := range inspection.Bundles { + for _, evidence := range b.Evidence { + if evidence.Path != "" { + paths[evidence.Path] = true + } + } + } + // Runtime integration may be installed after discovery. Record absent package + // manifests between each catalog and root as well as the evidence we found. + for _, b := range proposed.EffectiveBundles() { + source, _ := safePlanPath(root, b.Source) + for dir := filepath.Dir(source); ; dir = filepath.Dir(dir) { + paths[filepath.Join(dir, "package.json")] = true + if dir == root { + break + } + } + } + for path := range paths { + abs, err := safePlanPath(root, path) + if err != nil { + return nil, err + } + if abs == configPath { + continue + } + data, present, err := readPlanFile(root, abs) + if err != nil { + return nil, err + } + ob := Observation{Path: abs, Exists: present} + if present { + ob.SHA256 = planHash(data) + } + plan.Observations = append(plan.Observations, ob) + } + sort.Slice(plan.Observations, func(i, j int) bool { return plan.Observations[i].Path < plan.Observations[j].Path }) + sort.Slice(plan.RequiredDecisions, func(i, j int) bool { + a, b := plan.RequiredDecisions[i], plan.RequiredDecisions[j] + return a.Bundle+"\x00"+a.Code < b.Bundle+"\x00"+b.Code + }) + var encoded bytes.Buffer + encoder := yaml.NewEncoder(&encoded) + encoder.SetIndent(2) + if err := encoder.Encode(doc); err != nil { + return nil, err + } + if err := encoder.Close(); err != nil { + return nil, err + } + plan.ProposedYAML = encoded.String() + // No-op planning must not rewrite formatting or comments. + if exists && len(options.AddBundles) == 0 && len(options.Syntax) == 0 && options.SourceLocale == "" && len(options.TargetLocales) == 0 { + plan.ProposedYAML = string(before) + } + plan.AfterSHA256 = planHash([]byte(plan.ProposedYAML)) + plan.Diff = planDiff(configPath, string(before), plan.ProposedYAML) + plan.ID, err = planDigest(plan) + return plan, err +} + +// ApplyPlan rechecks all observations under a cooperative per-config lock and +// replaces only the exact config file. It never executes commands from a plan. +func ApplyPlan(plan *ConfigPlan) (*ApplyReceipt, error) { + if plan == nil || plan.SchemaVersion != planVersion { + return nil, planError("invalid_plan", "unsupported or missing configuration plan") + } + digest, err := planDigest(plan) + if err != nil || plan.ID != digest || plan.AfterSHA256 != planHash([]byte(plan.ProposedYAML)) { + return nil, planError("invalid_plan", "plan integrity check failed; create a new plan") + } + if len(plan.RequiredDecisions) > 0 { + return nil, planError("decisions_required", "plan has unresolved decisions; create a new plan with explicit selections") + } + if len(plan.ProposedYAML) > maxPlanConfigSize { + return nil, planError("invalid_plan", "proposed configuration exceeds the plan size limit") + } + root, err := canonicalPlanRoot(plan.Root) + if err != nil { + return nil, err + } + path, err := safePlanPath(root, plan.ConfigPath) + if err != nil { + return nil, err + } + var doc yaml.Node + if err := yaml.Unmarshal([]byte(plan.ProposedYAML), &doc); err != nil { + return nil, planError("invalid_plan", "proposed configuration is not valid YAML") + } + if err := inspectPlanYAML(&doc); err != nil { + return nil, err + } + var cfg config.Config + if err := doc.Decode(&cfg); err != nil { + return nil, planError("invalid_plan", "proposed configuration fields have invalid types") + } + cfg.ApplyDefaults() + if err := cfg.ValidateProject(); err != nil { + return nil, planError("invalid_plan", "proposed configuration is invalid: "+err.Error()) + } + for _, b := range cfg.EffectiveBundles() { + if _, err := safePlanPath(root, b.Source); err != nil { + return nil, err + } + for _, locale := range cfg.TargetLocales { + target, _ := b.TargetPath(locale) + if _, err := safePlanPath(root, target); err != nil { + return nil, err + } + } + } + lockPath := path + ".apply-lock" + if _, err := safePlanPath(root, lockPath); err != nil { + return nil, err + } + lock, err := os.OpenFile(lockPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if errors.Is(err, os.ErrExist) { + return nil, planError("apply_locked", "another config application owns the lock; inspect it before retrying") + } + if err != nil { + return nil, planError("apply_io", "cannot create the configuration application lock") + } + defer func() { _ = lock.Close(); _ = os.Remove(lockPath) }() + before, exists, err := readPlanFile(root, path) + if err != nil { + return nil, err + } + receipt := &ApplyReceipt{SchemaVersion: planVersion, PlanID: plan.ID, ConfigPath: path, ConfigSHA256: plan.AfterSHA256, ChangedPaths: []string{}} + // Replay attests only that config bytes already match the plan. Source and + // framework evidence can evolve after a successful application; callers must + // run config check/dry-run to establish current translation readiness. + if exists && planHash(before) == plan.AfterSHA256 { + receipt.Status = "already_applied" + return receipt, nil + } + if exists != plan.BeforeExists || (exists && planHash(before) != plan.BeforeSHA256) { + return nil, planError("stale_plan", "configuration changed; create a new configuration plan") + } + for _, observation := range plan.Observations { + data, exists, err := readPlanFile(root, observation.Path) + if err != nil { + return nil, err + } + if exists != observation.Exists || (exists && planHash(data) != observation.SHA256) { + return nil, planError("stale_plan", "source or runtime evidence changed; create a new configuration plan") + } + } + inspection, err := Scan(root, path) + if err != nil { + return nil, planError("stale_plan", "project discovery can no longer reproduce the saved plan; create a new plan") + } + discoveryDigest, err := discoveryPlanDigest(inspection) + if err != nil || discoveryDigest != plan.DiscoverySHA256 { + return nil, planError("stale_plan", "discovered catalogs or runtime evidence changed; create a new configuration plan") + } + receipt.ObservationsRevalidated = true + mode := os.FileMode(0600) + if exists { + info, err := os.Stat(path) + if err != nil { + return nil, planError("apply_io", "cannot inspect configuration permissions") + } + mode = info.Mode().Perm() + } + tmp, err := os.CreateTemp(filepath.Dir(path), ".internationalizer-apply-*") + if err != nil { + return nil, planError("apply_io", "cannot prepare the configuration replacement") + } + tmpPath := tmp.Name() + defer func() { _ = tmp.Close(); _ = os.Remove(tmpPath) }() + if err := tmp.Chmod(mode); err != nil { + return nil, planError("apply_io", "cannot preserve configuration permissions") + } + if _, err := tmp.WriteString(plan.ProposedYAML); err != nil { + return nil, planError("apply_io", "cannot write the configuration replacement") + } + if err := tmp.Sync(); err != nil { + return nil, planError("apply_io", "cannot flush the configuration replacement") + } + if err := tmp.Close(); err != nil { + return nil, planError("apply_io", "cannot close the configuration replacement") + } + // Recheck immediately before commit as protection against ordinary editors + // that do not participate in the cooperative lock. + current, currentExists, err := readPlanFile(root, path) + if err != nil { + return nil, err + } + if currentExists != exists || !bytes.Equal(current, before) { + return nil, planError("stale_plan", "configuration changed during application; create a new plan") + } + if err := os.Rename(tmpPath, path); err != nil { + return nil, planError("apply_io", "cannot commit the configuration replacement") + } + readback, present, err := readPlanFile(root, path) + if err != nil || !present || planHash(readback) != plan.AfterSHA256 { + return nil, planError("apply_verification_failed", "configuration was written but readback did not match; inspect current state before retrying") + } + receipt.Status = "applied" + receipt.ChangedPaths = []string{path} + return receipt, nil +} + +func planDigest(plan *ConfigPlan) (string, error) { + copy := *plan + copy.ID = "" + data, err := json.Marshal(copy) + if err != nil { + return "", err + } + return planHash(data), nil +} + +func discoveryPlanDigest(inspection *Inspection) (string, error) { + // Credential presence is neither source evidence nor provider verification. + // Excluding it keeps otherwise identical offline plans deterministic. + copy := *inspection + copy.Credentials = nil + data, err := json.Marshal(copy) + if err != nil { + return "", err + } + return planHash(data), nil +} + +func planHash(data []byte) string { + digest := sha256.Sum256(data) + return hex.EncodeToString(digest[:]) +} + +func canonicalPlanRoot(root string) (string, error) { + abs, err := filepath.Abs(root) + if err != nil { + return "", planError("unsafe_path", "cannot resolve project root") + } + abs, err = filepath.EvalSymlinks(abs) + if err != nil { + return "", planError("unsafe_path", "project root must exist") + } + info, err := os.Stat(abs) + if err != nil || !info.IsDir() { + return "", planError("unsafe_path", "project root must be a directory") + } + return abs, nil +} + +func planConfigPath(root, path string) (string, error) { + if path != "" { + return safePlanPath(root, path) + } + resolved := resolveConfigPath(root, "") + rel, err := filepath.Rel(root, resolved) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", planError("external_config_scope", "the active default configuration is outside the project; use --config with an explicit project-local path to create a local configuration intentionally") + } + return safePlanPath(root, resolved) +} + +// Reject links in every project-relative component, including dangling links. +// Root itself is canonicalized to support platform aliases such as /var. +func safePlanPath(root, path string) (string, error) { + if !filepath.IsAbs(path) { + path = filepath.Join(root, path) + } + path = filepath.Clean(path) + if sensitivePlanPath(path) { + return "", planError("unsafe_path", "configuration plans do not read or write credential-shaped files") + } + rel, err := filepath.Rel(root, path) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", planError("unsafe_path", "configuration plans may only access files inside the project root") + } + current := root + parts := strings.Split(rel, string(filepath.Separator)) + for i, part := range parts { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return "", planError("unsafe_path", "cannot inspect a project path safely") + } + if info.Mode()&os.ModeSymlink != 0 { + return "", planError("unsafe_path", "configuration plans do not follow symlinks") + } + if i < len(parts)-1 && !info.IsDir() { + return "", planError("unsafe_path", "project path has a non-directory ancestor") + } + if i == len(parts)-1 && !info.Mode().IsRegular() { + return "", planError("unsafe_path", "configuration plans require regular files") + } + } + return path, nil +} + +func sensitivePlanPath(path string) bool { + name := strings.ToLower(filepath.Base(path)) + return name == ".env" || strings.HasPrefix(name, ".env.") || strings.HasSuffix(name, ".key") || strings.HasSuffix(name, ".pem") || strings.HasSuffix(name, ".p12") +} + +func readPlanFile(root, path string) ([]byte, bool, error) { + path, err := safePlanPath(root, path) + if err != nil { + return nil, false, err + } + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil, false, nil + } + if err != nil { + return nil, false, planError("apply_io", "cannot read a project file required by the plan") + } + defer func() { _ = file.Close() }() + data, err := io.ReadAll(io.LimitReader(file, maxDiscoveryBytes+1)) + if err != nil { + return nil, false, planError("apply_io", "cannot read a project file required by the plan") + } + if len(data) > maxDiscoveryBytes { + return nil, false, planError("file_too_large", "a project file exceeds the plan observation size limit") + } + return data, true, nil +} + +func temporaryPlanPath(root, path string) bool { + rel, _ := filepath.Rel(root, path) + for _, part := range strings.Split(filepath.ToSlash(rel), "/") { + if part == "tmp" || part == "temp" { + return true + } + } + return false +} + +// Plans preserve unknown settings, but cannot safely publish opaque inline +// secrets. Reject such configs before serializing either YAML or a diff. +func inspectPlanYAML(node *yaml.Node) error { + if node.Kind == yaml.AliasNode { + return planError("unsupported_config", "configuration plan editing does not support YAML aliases; expand aliases before planning") + } + if node.Kind == yaml.MappingNode { + seen := map[string]bool{} + for i := 0; i+1 < len(node.Content); i += 2 { + key := node.Content[i].Value + if seen[key] { + return planError("invalid_config", "configuration contains duplicate YAML keys") + } + seen[key] = true + lower := strings.ToLower(key) + if lower == "<<" { + return planError("unsupported_config", "configuration plan editing does not support YAML merge keys") + } + if !strings.HasSuffix(lower, "_env") && (strings.Contains(lower, "password") || strings.Contains(lower, "secret") || strings.Contains(lower, "token") || lower == "api_key" || strings.HasSuffix(lower, "_api_key") || lower == "private_key") { + return planError("inline_secret", "configuration contains a possible inline credential; use environment variable references before creating a saved plan") + } + } + } + if node.Kind == yaml.ScalarNode { + if parsed, err := url.Parse(node.Value); err == nil && parsed.Scheme != "" && parsed.Host != "" { + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return planError("inline_secret", "configuration contains a URL with possible embedded credentials; use a credential-free endpoint before creating a saved plan") + } + } + } + for _, child := range node.Content { + if err := inspectPlanYAML(child); err != nil { + return err + } + } + return nil +} + +func planScalar(value string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value} +} +func getPlanNode(mapping *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + return mapping.Content[i+1] + } + } + return nil +} +func getPlanKeyNode(mapping *yaml.Node, key string) *yaml.Node { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + return mapping.Content[i] + } + } + return nil +} +func setPlanNode(mapping *yaml.Node, key string, value *yaml.Node) { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + old := mapping.Content[i+1] + value.HeadComment = old.HeadComment + value.LineComment = old.LineComment + value.FootComment = old.FootComment + mapping.Content[i+1] = value + return + } + } + mapping.Content = append(mapping.Content, planScalar(key), value) +} +func setPlanScalar(mapping *yaml.Node, key, value string) { + if node := getPlanNode(mapping, key); node != nil { + node.Kind = yaml.ScalarNode + node.Tag = "!!str" + node.Value = value + node.Content = nil + return + } + setPlanNode(mapping, key, planScalar(value)) +} +func removePlanKey(mapping *yaml.Node, key string) { + for i := 0; i+1 < len(mapping.Content); i += 2 { + if mapping.Content[i].Value == key { + mapping.Content = append(mapping.Content[:i], mapping.Content[i+2:]...) + return + } + } +} +func planBundleNode(b config.Bundle) *yaml.Node { + node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + setPlanScalar(node, "id", b.ID) + setPlanScalar(node, "source", b.Source) + setPlanScalar(node, "target", b.Target) + if b.Format != "" { + setPlanScalar(node, "format", b.Format) + } + if b.MessageSyntax != "" { + setPlanScalar(node, "message_syntax", string(b.MessageSyntax)) + } + return node +} + +func planDiff(path, before, after string) string { + if before == after { + return "" + } + var out strings.Builder + fmt.Fprintf(&out, "--- %s\n+++ %s\n", path, path) + oldLines := strings.Split(strings.TrimSuffix(before, "\n"), "\n") + newLines := strings.Split(strings.TrimSuffix(after, "\n"), "\n") + if before == "" { + oldLines = nil + } + if after == "" { + newLines = nil + } + fmt.Fprintf(&out, "@@ -1,%d +1,%d @@\n", len(oldLines), len(newLines)) + for _, line := range oldLines { + out.WriteString("-" + line + "\n") + } + for _, line := range newLines { + out.WriteString("+" + line + "\n") + } + return out.String() +} diff --git a/internal/onboarding/plan_test.go b/internal/onboarding/plan_test.go new file mode 100644 index 0000000..9325686 --- /dev/null +++ b/internal/onboarding/plan_test.go @@ -0,0 +1,295 @@ +package onboarding + +import ( + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/message" +) + +func planFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + planWrite(t, root, ".internationalizer.yml", "# retain me\nsource_locale: en\ntarget_locales: [fr]\n# marketing owner\nsource_path: tmp/en.json # staging source\nllm:\n provider: openai\n api_key_env: TEST_KEY\n locale_overrides:\n fr:\n model: custom\nglossary_dir: custom/glossary\nfuture_setting: retained\n") + planWrite(t, root, "tmp/en.json", `{"code":"{.sift,.agents}"}`) + planWrite(t, root, "web/locales/en.json", `{"hello":"Hello {{name}}"}`) + return root +} + +func planWrite(t *testing.T, root, path, data string) { + t.Helper() + p := filepath.Join(root, path) + if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(data), 0600); err != nil { + t.Fatal(err) + } +} + +func planOptions() PlanOptions { + return PlanOptions{AddBundles: []config.Bundle{{ID: "web", Source: "web/locales/en.json", Target: "web/locales/{locale}.json", MessageSyntax: message.I18next}}, Syntax: map[string]message.Syntax{"default": message.Plain}, ConfirmSources: []string{"tmp/en.json"}} +} + +func TestPlanPreservesConfigAndReplay(t *testing.T) { + root := planFixture(t) + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + if len(p.RequiredDecisions) != 0 { + t.Fatalf("decisions: %+v", p.RequiredDecisions) + } + p2, err := BuildPlan(root, "", planOptions()) + if err != nil || p.ID != p2.ID { + t.Fatalf("plan not deterministic: %v", err) + } + for _, retained := range []string{"# retain me", "# marketing owner", "# staging source", "future_setting: retained", "glossary_dir: custom/glossary", "model: custom", "id: default", "id: web", "message_syntax: plain", "message_syntax: i18next"} { + if !strings.Contains(p.ProposedYAML, retained) { + t.Errorf("missing %q in %s", retained, p.ProposedYAML) + } + } + before, _ := os.ReadFile(filepath.Join(root, ".internationalizer.yml")) + if strings.Contains(string(before), "id: web") { + t.Fatal("planning wrote config") + } + r, err := ApplyPlan(p) + if err != nil || r.Status != "applied" || len(r.ChangedPaths) != 1 { + t.Fatalf("apply: %+v %v", r, err) + } + r, err = ApplyPlan(p) + if err != nil || r.Status != "already_applied" || len(r.ChangedPaths) != 0 { + t.Fatalf("replay: %+v %v", r, err) + } +} + +func TestPlanRequiresDecisions(t *testing.T) { + root := planFixture(t) + p, err := BuildPlan(root, "", PlanOptions{}) + if err != nil { + t.Fatal(err) + } + if len(p.RequiredDecisions) == 0 { + t.Fatal("missing source/syntax decisions") + } + _, err = ApplyPlan(p) + assertPlanCode(t, err, "decisions_required") +} + +func TestApplyRejectsDriftTamperingAndLocks(t *testing.T) { + for _, kind := range []string{"source", "config", "tamper", "lock", "symlink"} { + t.Run(kind, func(t *testing.T) { + root := planFixture(t) + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + code := "stale_plan" + switch kind { + case "source": + planWrite(t, root, "tmp/en.json", `{"new":"changed"}`) + case "config": + planWrite(t, root, ".internationalizer.yml", "changed: true\n") + case "tamper": + p.ProposedYAML += "tampered: true\n" + code = "invalid_plan" + case "lock": + planWrite(t, root, ".internationalizer.yml.apply-lock", "owned") + code = "apply_locked" + case "symlink": + if err := os.Remove(filepath.Join(root, "tmp/en.json")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "web/locales/en.json"), filepath.Join(root, "tmp/en.json")); err != nil { + t.Skip(err) + } + code = "unsafe_path" + } + _, err = ApplyPlan(p) + assertPlanCode(t, err, code) + }) + } +} + +func TestPlanRejectsUnsafePathsAndSecrets(t *testing.T) { + root := planFixture(t) + _, err := BuildPlan(root, "../outside.yml", planOptions()) + assertPlanCode(t, err, "unsafe_path") + planWrite(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: tmp/en.json\napi_key: never-serialize-this\n") + p, err := BuildPlan(root, "", planOptions()) + assertPlanCode(t, err, "inline_secret") + if p != nil || strings.Contains(err.Error(), "never-serialize-this") { + t.Fatal("secret exposed") + } +} + +func TestPlanCreatesConfigAndObservesAbsentRuntimeEvidence(t *testing.T) { + root := t.TempDir() + planWrite(t, root, "locales/en.json", `{"hello":"Hello {{name}}"}`) + options := PlanOptions{TargetLocales: []string{"fr"}, AddBundles: []config.Bundle{{ID: "web", Source: "locales/en.json", Target: "locales/{locale}.json", MessageSyntax: message.I18next}}} + plan, err := BuildPlan(root, "", options) + if err != nil || plan.BeforeExists || len(plan.RequiredDecisions) > 0 { + t.Fatalf("fresh plan: %+v %v", plan, err) + } + planWrite(t, root, "package.json", `{"dependencies":{"i18next-icu":"1.0.0"}}`) + _, err = ApplyPlan(plan) + assertPlanCode(t, err, "stale_plan") + plan, err = BuildPlan(root, "", options) + if err != nil { + t.Fatal(err) + } + receipt, err := ApplyPlan(plan) + if err != nil || receipt.Status != "applied" { + t.Fatalf("fresh apply: %+v %v", receipt, err) + } + info, err := os.Stat(filepath.Join(root, ".internationalizer.yml")) + if err != nil { + t.Fatal(err) + } + if runtime.GOOS != "windows" && info.Mode().Perm() != 0600 { + t.Fatalf("unexpected permissions %v", info.Mode().Perm()) + } +} + +func TestPlanPreservesExistingBundleSettings(t *testing.T) { + root := t.TempDir() + planWrite(t, root, ".internationalizer.yaml", "source_locale: en\ntarget_locales: [fr]\nmessage_syntax: auto\nbundles:\n - id: marketing\n source: locales/en.json\n target: locales/{locale}.json\n message_syntax: auto # profile note\n future_bundle_option: keep\nllm:\n provider: openai\n locale_overrides:\n fr:\n provider: gemini\n api_key_env: FRENCH_KEY\n") + planWrite(t, root, "locales/en.json", `{"hello":"Hello"}`) + p, err := BuildPlan(root, "", PlanOptions{Syntax: map[string]message.Syntax{"marketing": message.Plain}}) + if err != nil { + t.Fatal(err) + } + for _, fragment := range []string{"id: marketing", "future_bundle_option: keep", "# profile note", "provider: gemini", "api_key_env: FRENCH_KEY"} { + if !strings.Contains(p.ProposedYAML, fragment) { + t.Errorf("lost %q", fragment) + } + } + if !strings.HasSuffix(p.ConfigPath, ".internationalizer.yaml") { + t.Fatalf("wrong config path %s", p.ConfigPath) + } + if _, err = ApplyPlan(p); err != nil { + t.Fatal(err) + } +} + +func TestPlanMissingChoicesAndInvalidDecisions(t *testing.T) { + t.Run("empty project", func(t *testing.T) { + p, err := BuildPlan(t.TempDir(), "", PlanOptions{}) + if err != nil || len(p.RequiredDecisions) != 2 { + t.Fatalf("empty plan: %+v %v", p, err) + } + }) + t.Run("missing syntax", func(t *testing.T) { + root := t.TempDir() + planWrite(t, root, "en.json", `{"hello":"hello"}`) + p, err := BuildPlan(root, "", PlanOptions{TargetLocales: []string{"fr"}, AddBundles: []config.Bundle{{ID: "web", Source: "en.json", Target: "{locale}.json"}}}) + if err != nil || len(p.RequiredDecisions) != 1 || p.RequiredDecisions[0].Code != "SYNTAX_SELECTION_REQUIRED" { + t.Fatalf("missing syntax: %+v %v", p, err) + } + }) + t.Run("missing source", func(t *testing.T) { + p, err := BuildPlan(t.TempDir(), "", PlanOptions{TargetLocales: []string{"fr"}, AddBundles: []config.Bundle{{ID: "web", Source: "en.json", Target: "{locale}.json", MessageSyntax: message.Plain}}}) + if err != nil || len(p.RequiredDecisions) != 1 || p.RequiredDecisions[0].Code != "SOURCE_NOT_FOUND" { + t.Fatalf("missing source: %+v %v", p, err) + } + }) + for _, options := range []PlanOptions{ + {Syntax: map[string]message.Syntax{"absent": message.Plain}}, + {Syntax: map[string]message.Syntax{"default": "unsupported"}}, + {AddBundles: []config.Bundle{{ID: "default", Source: "tmp/en.json", Target: "tmp/{locale}.json"}}}, + {AddBundles: []config.Bundle{{ID: "other", Source: "tmp/en.json"}}}, + } { + root := planFixture(t) + _, err := BuildPlan(root, "", options) + assertPlanCode(t, err, "invalid_decision") + } +} + +func TestPlanRejectsYAMLAliasesDuplicateKeysAndCredentialURLs(t *testing.T) { + for _, tc := range []struct{ extra, code string }{ + {"future: &alias {one: two}\nother: *alias\n", "unsupported_config"}, + {"future: one\nfuture: two\n", "invalid_config"}, + {"llm:\n base_url: https://example.com/v1?key=secret\n", "inline_secret"}, + } { + root := t.TempDir() + planWrite(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: en.json\n"+tc.extra) + _, err := BuildPlan(root, "", PlanOptions{}) + assertPlanCode(t, err, tc.code) + } +} + +func TestApplyReplayReportsConfigMatchWithoutRevalidatingSource(t *testing.T) { + root := planFixture(t) + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + if _, err = ApplyPlan(p); err != nil { + t.Fatal(err) + } + planWrite(t, root, "tmp/en.json", `{"new":"new"}`) + receipt, err := ApplyPlan(p) + if err != nil || receipt.Status != "already_applied" || receipt.ObservationsRevalidated { + t.Fatalf("replay: %+v %v", receipt, err) + } +} + +func TestApplyRejectsNewDiscoveryEvidence(t *testing.T) { + root := planFixture(t) + planWrite(t, root, "web/package.json", `{"dependencies":{"i18next":"1.0.0"}}`) + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + planWrite(t, root, "web/i18n.ts", `import ICU from "i18next-icu";`) + _, err = ApplyPlan(p) + assertPlanCode(t, err, "stale_plan") +} + +func TestPlanDoesNotSilentlyShadowHomeConfiguration(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + planWrite(t, home, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: en.json\n") + root := t.TempDir() + _, err := BuildPlan(root, "", PlanOptions{}) + assertPlanCode(t, err, "external_config_scope") + p, err := BuildPlan(root, ".internationalizer.yml", PlanOptions{}) + if err != nil || p.BeforeExists { + t.Fatalf("explicit local config: %+v %v", p, err) + } +} + +func TestPlanRejectsCredentialShapedFiles(t *testing.T) { + for _, name := range []string{".env", ".env.production", "private.pem", "private.key", "certificate.p12"} { + root := t.TempDir() + planWrite(t, root, name, `{"hello":"private material"}`) + _, err := BuildPlan(root, ".internationalizer.yml", PlanOptions{TargetLocales: []string{"fr"}, AddBundles: []config.Bundle{{ID: "web", Source: name, Target: "{locale}.json", Format: "json", MessageSyntax: message.Plain}}}) + assertPlanCode(t, err, "unsafe_path") + } +} + +func TestPlanAllowsIntrinsicFluentGrammar(t *testing.T) { + root := t.TempDir() + planWrite(t, root, "en.ftl", "hello = Hello\n") + p, err := BuildPlan(root, ".internationalizer.yml", PlanOptions{TargetLocales: []string{"fr"}, AddBundles: []config.Bundle{{ID: "fluent", Source: "en.ftl", Target: "{locale}.ftl", MessageSyntax: message.Auto}}}) + if err != nil || len(p.RequiredDecisions) > 0 { + t.Fatalf("Fluent plan: %+v %v", p, err) + } + if _, err = ApplyPlan(p); err != nil { + t.Fatal(err) + } +} + +func assertPlanCode(t *testing.T, err error, want string) { + t.Helper() + var e *PlanError + if !errors.As(err, &e) || e.Code != want { + t.Fatalf("got %v, want code %s", err, want) + } +} diff --git a/internal/translate/translate.go b/internal/translate/translate.go index 2fa9f5e..b81ed43 100644 --- a/internal/translate/translate.go +++ b/internal/translate/translate.go @@ -35,27 +35,28 @@ type Options struct { // Result holds the outcome of one bundle and locale. Lifecycle counters are // pre-run observations; manual, source-stale, and policy-stale may overlap. type Result struct { - BlockedBySource bool - SourcePath string - BlockedLocales []string - DryRun bool - Bundle string - Locale string - TargetPath string - KeysTotal int - KeysMissing int - KeysSourceStale int - KeysPolicyStale int - KeysManualEdit int - KeysUntracked int - KeysCurrent int - KeysCached int - KeysTranslated int - KeysSkipped int - Batches int - TokensIn int - TokensOut int - Errors []string + BlockedBySource bool `json:"blocked_by_source"` + SourcePath string `json:"source_path,omitempty"` + BlockedLocales []string `json:"blocked_locales,omitempty"` + DryRun bool `json:"dry_run"` + Bundle string `json:"bundle"` + Locale string `json:"locale"` + TargetPath string `json:"target_path"` + KeysTotal int `json:"keys_total"` + KeysMissing int `json:"keys_missing"` + KeysSourceStale int `json:"keys_source_stale"` + KeysPolicyStale int `json:"keys_policy_stale"` + KeysManualEdit int `json:"keys_manual_edit"` + KeysUntracked int `json:"keys_untracked"` + KeysCurrent int `json:"keys_current"` + KeysCached int `json:"keys_cached"` + KeysTranslated int `json:"keys_translated"` + KeysSkipped int `json:"keys_skipped"` + Batches int `json:"batches"` + ProviderCalls int `json:"provider_calls"` + TokensIn int `json:"tokens_in"` + TokensOut int `json:"tokens_out"` + Errors []string `json:"errors,omitempty"` } // RunError reports that one or more locale jobs failed. Results remain @@ -139,7 +140,7 @@ func Run(ctx context.Context, cfg *config.Config, provider llm.Provider, opts Op } for i := range bundles { for _, unit := range bundles[i].sourceUnits { - for _, finding := range validation.SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax) { + for _, finding := range validation.SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax, bundles[i].bundle.MessageSyntax) { bundles[i].sourceErrors = append(bundles[i].sourceErrors, fmt.Sprintf("source %q: %s", unit.ID, finding.Message)) } } @@ -430,6 +431,7 @@ func translateLocale( entries[j] = llm.Entry{Key: plan.key, Value: plan.source, Context: plan.context} } + result.ProviderCalls++ response, err := provider.Translate(ctx, llm.TranslateRequest{ SourceLocale: cfg.SourceLocale, TargetLocale: locale, diff --git a/internal/validate/syntax.go b/internal/validate/syntax.go index e6a6cf2..998efdb 100644 --- a/internal/validate/syntax.go +++ b/internal/validate/syntax.go @@ -1,11 +1,33 @@ package validate -import "github.com/Tom-R-Main/Internationalizer/internal/message" +import ( + "fmt" + "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/message" + "github.com/Tom-R-Main/Internationalizer/internal/protectedtext" +) // SyntaxSourceFindings checks a source with its already resolved runtime grammar. -func SyntaxSourceFindings(key, source, sourceLocale string, syntax message.Syntax) []Finding { +func SyntaxSourceFindings(key, source, sourceLocale string, syntax message.Syntax, requested ...message.Syntax) []Finding { if syntax == message.ICU { - return messageFindings(key, message.CompareICU(source, source, sourceLocale)) + findings := messageFindings(key, message.CompareICU(source, source, sourceLocale)) + if len(requested) > 0 && (requested[0] == message.Auto || requested[0] == "") { + for i := range findings { + if findings[i].Code != CodeICUMessageSyntax { + continue + } + context := "contains ambiguous brace syntax" + for _, code := range protectedtext.HTMLCode(source) { + if strings.Contains(code, "{") { + context = "contains brace syntax inside HTML code" + break + } + } + findings[i].Message = fmt.Sprintf("%s %s. With message_syntax: auto, it was interpreted as ICU. Select the bundle's runtime syntax: plain, i18next, or icu. %s", key, context, findings[i].Message) + } + } + return findings } return nil } diff --git a/internal/validate/syntax_guidance_test.go b/internal/validate/syntax_guidance_test.go new file mode 100644 index 0000000..f30e3a3 --- /dev/null +++ b/internal/validate/syntax_guidance_test.go @@ -0,0 +1,22 @@ +package validate + +import ( + "strings" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/message" +) + +func TestSourceSyntaxGuidanceOnlyForAuto(t *testing.T) { + source := `Read {.sift,.claude}/skills` + for _, policy := range []message.Syntax{message.Auto, message.ICU} { + findings := SyntaxSourceFindings("docs.tui.skills.desc", source, "en", message.ICU, policy) + if len(findings) != 1 || findings[0].Code != CodeICUMessageSyntax { + t.Fatalf("policy %s: %+v", policy, findings) + } + guidance := strings.Contains(findings[0].Message, "Select the bundle's runtime syntax") + if guidance != (policy == message.Auto) { + t.Fatalf("policy %s: %s", policy, findings[0].Message) + } + } +} diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 97fbc69..8f5b784 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -96,7 +96,7 @@ func ValidateWithOptions(cfg *config.Config, opts Options) ([]Report, error) { sourceKeys := formats.UnitValues(sourceUnits) sourceFindings := make(map[string][]Finding) for _, unit := range sourceUnits { - if findings := SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax); len(findings) > 0 { + if findings := SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax, bundle.MessageSyntax); len(findings) > 0 { sourceFindings[unit.ID] = findings } } diff --git a/test/acceptance/acceptance_test.go b/test/acceptance/acceptance_test.go index 0c6f069..6b6e80a 100644 --- a/test/acceptance/acceptance_test.go +++ b/test/acceptance/acceptance_test.go @@ -61,9 +61,9 @@ func TestDetectUsesProjectFixture(t *testing.T) { result.requireSuccess(t) normalizedStdout := strings.ReplaceAll(result.stdout, `\`, "/") for _, want := range []string{ - "Detected: react-i18next (confidence: 90%)", - "source_path: public/locales/en.json", - "model: gemini-3.8-flash", + "Catalog public/locales/en.json: framework=i18next", + "suggested syntax=i18next", + "UNCOVERED_CATALOG", } { if !strings.Contains(normalizedStdout, want) { t.Fatalf("detect stdout does not contain %q:\n%s", want, result.stdout) diff --git a/test/acceptance/onboarding_test.go b/test/acceptance/onboarding_test.go new file mode 100644 index 0000000..2313643 --- /dev/null +++ b/test/acceptance/onboarding_test.go @@ -0,0 +1,158 @@ +package acceptance_test + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// This trajectory deliberately starts with a valid marketing-only config and +// a nested app. No network, real credentials, or repository catalogs are used. +func onboardingFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + files := map[string]string{ + ".internationalizer.yml": `# Marketing pipeline; preserve this comment. +source_locale: en +target_locales: [fr, ja] +source_path: tmp/english-keys.json +llm: + provider: openai + api_key_env: ONBOARDING_TEST_KEY + locale_overrides: + ja: + provider: gemini + model: custom-japanese-model + api_key_env: ONBOARDING_JA_KEY +glossary_dir: custom/glossary +style_guides_dir: custom/guides +future_setting: preserve-me +`, + "tmp/english-keys.json": `{"docs.tui.skills.desc":"Use {.sift,.claude,.codex,.agents}/skills","hello":"Hello {{name}}"}`, + "tmp/fr.json": `{"docs.tui.skills.desc":"Utiliser {.sift,.claude,.codex,.agents}/skills","hello":"Bonjour {{name}}"}`, + "tmp/ja.json": `{"docs.tui.skills.desc":"使用 {.sift,.claude,.codex,.agents}/skills","hello":"こんにちは {{name}}"}`, + "exf-app/web/package.json": `{"dependencies":{"i18next":"25.0.0","react-i18next":"16.0.0"}}`, + "exf-app/web/src/i18n/index.ts": `import i18next from 'i18next'; import en from './locales/en.json'; i18next.init({resources:{en}});`, + "exf-app/web/src/i18n/locales/en.json": `{"hello":"Hello {{name}}","count_one":"{{count}} item","count_other":"{{count}} items"}`, + } + for name, contents := range files { + path := filepath.Join(root, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + } + return root +} + +func decodeOnboardingJSON(t *testing.T, result cliResult) map[string]any { + t.Helper() + var doc map[string]any + if err := json.Unmarshal([]byte(result.stdout), &doc); err != nil { + t.Fatalf("invalid JSON: %v\nstdout=%s\nstderr=%s", err, result.stdout, result.stderr) + } + if doc["schema_version"] != float64(1) { + t.Fatalf("unversioned output: %s", result.stdout) + } + return doc +} + +func TestOnboardingDiscoverPlanApplyVerify(t *testing.T) { + root := onboardingFixture(t) + original := mustReadFile(t, filepath.Join(root, ".internationalizer.yml")) + discovery := runCLI(t, root, nil, "detect", "--json") + discovery.requireSuccess(t) + decodeOnboardingJSON(t, discovery) + for _, want := range []string{"UNCOVERED_CATALOG", "SOURCE_CONFIRMATION_REQUIRED", "AUTO_SYNTAX_AMBIGUOUS", "exf-app/web/src/i18n/locales/en.json", `"suggested_syntax": "i18next"`} { + if !strings.Contains(discovery.stdout, want) { + t.Fatalf("missing %q: %s", want, discovery.stdout) + } + } + check := runCLI(t, root, nil, "config", "check", "--json") + decodeOnboardingJSON(t, check) + if !strings.Contains(check.stdout, `"provider_verified": false`) { + t.Fatal("offline readiness missing") + } + blocked := runCLI(t, root, nil, "config", "plan", "--json") + blocked.requireSuccess(t) + if decodeOnboardingJSON(t, blocked)["status"] != "needs_decision" { + t.Fatalf("should need decisions: %s", blocked.stdout) + } + if !bytes.Equal(original, mustReadFile(t, filepath.Join(root, ".internationalizer.yml"))) { + t.Fatal("inspection or planning modified config") + } + planArgs := []string{"config", "plan", "--json", "--add-bundle", "web=exf-app/web/src/i18n/locales/en.json", "--syntax", "web=i18next", "--syntax", "default=i18next", "--confirm-source", "tmp/english-keys.json", "--out", "onboarding-plan.json"} + plan := runCLI(t, root, nil, planArgs...) + plan.requireSuccess(t) + if decodeOnboardingJSON(t, plan)["status"] != "planned" { + t.Fatalf("explicit choices should resolve plan: %s", plan.stdout) + } + apply := runCLI(t, root, nil, "config", "apply", "--plan", "onboarding-plan.json", "--no-input", "--json") + apply.requireSuccess(t) + decodeOnboardingJSON(t, apply) + configured := string(mustReadFile(t, filepath.Join(root, ".internationalizer.yml"))) + for _, want := range []string{"# Marketing pipeline", "custom-japanese-model", "ONBOARDING_JA_KEY", "custom/glossary", "custom/guides", "future_setting: preserve-me", "id: default", "id: web"} { + if !strings.Contains(configured, want) { + t.Fatalf("lost %q: %s", want, configured) + } + } + replay := runCLI(t, root, nil, "config", "apply", "--plan", "onboarding-plan.json", "--no-input", "--json") + replay.requireSuccess(t) + if !strings.Contains(replay.stdout, "already_applied") { + t.Fatalf("retry not recognized: %s", replay.stdout) + } + dry := runCLI(t, root, nil, "translate", "--dry-run", "--json") + dry.requireSuccess(t) + if decodeOnboardingJSON(t, dry)["status"] != "planned" { + t.Fatalf("dry run did not plan: %s", dry.stdout) + } + for _, want := range []string{`"provider_called": false`, `"generated_keys": 0`} { + if !strings.Contains(dry.stdout, want) { + t.Fatalf("missing %q: %s", want, dry.stdout) + } + } + if _, err := os.Stat(filepath.Join(root, ".internationalizer.lock")); !os.IsNotExist(err) { + t.Fatalf("dry run wrote state: %v", err) + } + // A configured runtime is not a waiver for catalog damage. + target := filepath.Join(root, "tmp/fr.json") + if err := os.WriteFile(target, []byte(`{"docs.tui.skills.desc":"different-command","hello":"Bonjour {{other}}"}`), 0600); err != nil { + t.Fatal(err) + } + invalid := runCLI(t, root, nil, "validate", "--bundle", "default", "--locale", "fr", "--json") + if invalid.exitCode == 0 { + t.Fatal("damaged placeholders and code passed validation") + } + decodeOnboardingJSON(t, invalid) + if !strings.Contains(invalid.stdout, "HTML code mismatch") || !strings.Contains(invalid.stdout, "interpolation") { + t.Fatalf("expected code and interpolation findings: %s", invalid.stdout) + } +} + +func TestOnboardingJSONFailuresAndPlanDrift(t *testing.T) { + root := onboardingFixture(t) + invalid := runCLI(t, root, nil, "config", "plan", "--unknown-option", "--json") + if invalid.exitCode == 0 { + t.Fatal("unknown option accepted") + } + decodeOnboardingJSON(t, invalid) + planned := runCLI(t, root, nil, "config", "plan", "--syntax", "default=i18next", "--confirm-source", "tmp/english-keys.json", "--out", "plan.json", "--json") + planned.requireSuccess(t) + source := filepath.Join(root, "tmp/english-keys.json") + if err := os.WriteFile(source, []byte(`{"changed":"new source"}`), 0600); err != nil { + t.Fatal(err) + } + applied := runCLI(t, root, nil, "config", "apply", "--plan", "plan.json", "--no-input", "--json") + if applied.exitCode == 0 { + t.Fatal("stale plan applied") + } + decodeOnboardingJSON(t, applied) + if !strings.Contains(applied.stdout, `"code": "stale_plan"`) { + t.Fatalf("missing drift code: %s", applied.stdout) + } +} From bd897a51586a8be46b63b3aba4feae6c28ecce41 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 16:03:24 -0400 Subject: [PATCH 2/4] test: verify onboarding apply transaction postconditions Signed-off-by: Tom Main --- internal/onboarding/plan_test.go | 75 ++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/internal/onboarding/plan_test.go b/internal/onboarding/plan_test.go index 9325686..5d9aed0 100644 --- a/internal/onboarding/plan_test.go +++ b/internal/onboarding/plan_test.go @@ -1,6 +1,7 @@ package onboarding import ( + "bytes" "errors" "os" "path/filepath" @@ -286,6 +287,80 @@ func TestPlanAllowsIntrinsicFluentGrammar(t *testing.T) { } } +func TestApplyFailurePreservesConfigAndReleasesOwnedLock(t *testing.T) { + for _, drift := range []string{"source", "config", "runtime"} { + t.Run(drift, func(t *testing.T) { + root := planFixture(t) + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + switch drift { + case "source": + planWrite(t, root, "tmp/en.json", `{"hello":"Changed"}`) + case "config": + planWrite(t, root, ".internationalizer.yml", "# concurrent edit\nsource_locale: en\ntarget_locales: [fr]\nsource_path: tmp/en.json\n") + case "runtime": + planWrite(t, root, "web/package.json", `{"dependencies":{"i18next-icu":"1"}}`) + } + configPath := filepath.Join(root, ".internationalizer.yml") + before, err := os.ReadFile(configPath) + if err != nil { + t.Fatal(err) + } + receipt, err := ApplyPlan(p) + assertPlanCode(t, err, "stale_plan") + if receipt != nil { + t.Fatalf("failed application returned a success receipt: %+v", receipt) + } + after, err := os.ReadFile(configPath) + if err != nil || !bytes.Equal(before, after) { + t.Fatalf("failed application changed existing config: %v", err) + } + if _, err := os.Lstat(configPath + ".apply-lock"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("owned lock remains after failure: %v", err) + } + artifacts, err := filepath.Glob(filepath.Join(root, ".internationalizer-apply-*")) + if err != nil || len(artifacts) != 0 { + t.Fatalf("temporary replacements remain: %v, %v", artifacts, err) + } + }) + } +} + +func TestApplyPreservesPermissionsAndAttestsExactReadback(t *testing.T) { + root := planFixture(t) + configPath := filepath.Join(root, ".internationalizer.yml") + if err := os.Chmod(configPath, 0640); err != nil { + t.Fatal(err) + } + p, err := BuildPlan(root, "", planOptions()) + if err != nil { + t.Fatal(err) + } + receipt, err := ApplyPlan(p) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(configPath) + if err != nil { + t.Fatal(err) + } + if string(data) != p.ProposedYAML || planHash(data) != receipt.ConfigSHA256 || receipt.PlanID != p.ID || !receipt.ObservationsRevalidated { + t.Fatalf("receipt does not attest committed bytes and evidence: %+v", receipt) + } + info, err := os.Stat(configPath) + if err != nil { + t.Fatal(err) + } + if runtime.GOOS != "windows" && info.Mode().Perm() != 0640 { + t.Fatalf("existing config permissions changed: %v", info.Mode().Perm()) + } + if _, err := os.Lstat(configPath + ".apply-lock"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("owned lock remains after success: %v", err) + } +} + func assertPlanCode(t *testing.T, err error, want string) { t.Helper() var e *PlanError From b0e36ae60a2f39bac5c4fd136231a20350dbbc6b Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 16:03:24 -0400 Subject: [PATCH 3/4] chore: release onboarding workflow as 0.2.0 Signed-off-by: Tom Main --- CHANGELOG.md | 23 +++++++++++++++++++++++ package.json | 14 ++++++++------ packages/darwin-arm64/package.json | 2 +- packages/darwin-x64/package.json | 2 +- packages/linux-arm64/package.json | 2 +- packages/linux-x64/package.json | 2 +- packages/win32-x64/package.json | 2 +- 7 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..195bc73 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +## 0.2.0 - 2026-09-04 + +### Added + +- Discover configured and uncovered catalogs across nested apps with `detect --json`, including runtime evidence, syntax suggestions, and unresolved source choices. +- Inspect resolved bundles, locale targets, syntax provenance, and offline credential presence with `config check --json`. +- Review and apply saved configuration proposals with `config plan` and `config apply`. Plans preserve provider settings, locale overrides, glossary paths, and bundle identities; application checks drift and returns a receipt tied to the plan. +- Discover command arguments, schemas, side effects, and next steps with `commands --json`. +- Preview translation work with `translate --dry-run --json`, separating planned work, blocked jobs, generated translations, and provider calls. + +### Changed + +- **JSON compatibility:** `validate --json` now returns a versioned envelope. Read reports from `data.reports` instead of the previous top-level array. Check `schema_version` before consuming command-specific data. +- Discovery and configuration diagnostics, translation results, and validation reports support bounded JSON output and scope filters. Structured errors include recovery argument arrays. +- Framework detection is advisory: i18next ICU integrations and mixed frameworks require explicit decisions. Temporary marketing catalogs are flagged for confirmation rather than replaced. + +### Fixed + +- Explain when automatic syntax detection interprets literal code braces as ICU, and point to explicit runtime profiles. Explicit ICU parsing and placeholder/protected-code validation remain strict. + +See [the configuration workflow and JSON contract](docs/cli-onboarding.md) for setup, migration, and retry behavior. diff --git a/package.json b/package.json index e88741f..c05646a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer", - "version": "0.1.3", + "version": "0.2.0", "description": "AI-native internationalization CLI for software projects", "license": "AGPL-3.0-only", "bin": { @@ -10,17 +10,19 @@ "bin", "assets", "README.md", + "CHANGELOG.md", + "docs/cli-onboarding.md", "LICENSE" ], "engines": { "node": ">=18" }, "optionalDependencies": { - "internationalizer-darwin-arm64": "0.1.3", - "internationalizer-darwin-x64": "0.1.3", - "internationalizer-linux-arm64": "0.1.3", - "internationalizer-linux-x64": "0.1.3", - "internationalizer-win32-x64": "0.1.3" + "internationalizer-darwin-arm64": "0.2.0", + "internationalizer-darwin-x64": "0.2.0", + "internationalizer-linux-arm64": "0.2.0", + "internationalizer-linux-x64": "0.2.0", + "internationalizer-win32-x64": "0.2.0" }, "scripts": { "test:npm-wrapper": "node --test test/npm-wrapper.test.cjs", diff --git a/packages/darwin-arm64/package.json b/packages/darwin-arm64/package.json index a6bfeb5..4d678ec 100644 --- a/packages/darwin-arm64/package.json +++ b/packages/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer-darwin-arm64", - "version": "0.1.3", + "version": "0.2.0", "description": "darwin arm64 binary for the internationalizer CLI", "license": "AGPL-3.0-only", "os": [ diff --git a/packages/darwin-x64/package.json b/packages/darwin-x64/package.json index 8ced90b..81947be 100644 --- a/packages/darwin-x64/package.json +++ b/packages/darwin-x64/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer-darwin-x64", - "version": "0.1.3", + "version": "0.2.0", "description": "darwin x64 binary for the internationalizer CLI", "license": "AGPL-3.0-only", "os": [ diff --git a/packages/linux-arm64/package.json b/packages/linux-arm64/package.json index e3c5467..285571a 100644 --- a/packages/linux-arm64/package.json +++ b/packages/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer-linux-arm64", - "version": "0.1.3", + "version": "0.2.0", "description": "linux arm64 binary for the internationalizer CLI", "license": "AGPL-3.0-only", "os": [ diff --git a/packages/linux-x64/package.json b/packages/linux-x64/package.json index 9a997b1..8781b81 100644 --- a/packages/linux-x64/package.json +++ b/packages/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer-linux-x64", - "version": "0.1.3", + "version": "0.2.0", "description": "linux x64 binary for the internationalizer CLI", "license": "AGPL-3.0-only", "os": [ diff --git a/packages/win32-x64/package.json b/packages/win32-x64/package.json index eeef543..b185534 100644 --- a/packages/win32-x64/package.json +++ b/packages/win32-x64/package.json @@ -1,6 +1,6 @@ { "name": "internationalizer-win32-x64", - "version": "0.1.3", + "version": "0.2.0", "description": "win32 x64 binary for the internationalizer CLI", "license": "AGPL-3.0-only", "os": [ From cfc12bef0160e1aec3f5fcd0be6ce6cc3dce81ca Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 16:12:58 -0400 Subject: [PATCH 4/4] fix: report partial translation failure from persisted work Signed-off-by: Tom Main --- CHANGELOG.md | 1 + cmd/internationalizer/schema.go | 2 +- cmd/internationalizer/schema_test.go | 16 ++ cmd/internationalizer/translate.go | 6 +- .../translate_persistence_test.go | 137 ++++++++++++++++++ docs/cli-onboarding.md | 8 + internal/translate/persistence_test.go | 96 ++++++++++++ internal/translate/translate.go | 8 + 8 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 cmd/internationalizer/translate_persistence_test.go create mode 100644 internal/translate/persistence_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 195bc73..1700e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,5 +19,6 @@ ### Fixed - Explain when automatic syntax detection interprets literal code braces as ICU, and point to explicit runtime profiles. Explicit ICU parsing and placeholder/protected-code validation remain strict. +- Base partial-failure reporting on persisted catalog or manifest updates, not staged translations from a failed batch sequence. Job results distinguish catalog writes from manifest updates. See [the configuration workflow and JSON contract](docs/cli-onboarding.md) for setup, migration, and retry behavior. diff --git a/cmd/internationalizer/schema.go b/cmd/internationalizer/schema.go index 16a3c80..1fc7607 100644 --- a/cmd/internationalizer/schema.go +++ b/cmd/internationalizer/schema.go @@ -41,7 +41,7 @@ func workflowOutputSchema(path string) map[string]any { data["description"] = "Receipt tied to a plan and verified configuration fingerprint; application does not translate catalogs." case "translate": data = schemaForType(reflect.TypeFor[translationJSON]()) - data["description"] = "Planning, generation, adoption, and partial failure are distinct. Dry-run makes no provider call or file change; generation is not human approval." + data["description"] = "Planning, generation, adoption, and persistence are distinct. Partial failure requires retained catalog or manifest updates. Dry-run makes no provider call or file change; generation is not human approval or persistence." case "validate": data = schemaForType(reflect.TypeFor[validationJSON]()) data["description"] = "Counts cover the full selected scope before presentation limits. Structural validation and checked human approval are distinct." diff --git a/cmd/internationalizer/schema_test.go b/cmd/internationalizer/schema_test.go index 6c6b6e5..56f38ac 100644 --- a/cmd/internationalizer/schema_test.go +++ b/cmd/internationalizer/schema_test.go @@ -82,6 +82,22 @@ func TestWorkflowInputSchemaHasTypedFlags(t *testing.T) { } } +func TestTranslationSchemaDistinguishesPersistence(t *testing.T) { + schema := workflowOutputSchema("translate") + data := schema["properties"].(map[string]any)["data"].(map[string]any)["anyOf"].([]any)[0].(map[string]any) + properties := data["properties"].(map[string]any) + summary := properties["summary"].(map[string]any)["properties"].(map[string]any) + if _, ok := summary["persisted_jobs"]; !ok { + t.Fatal("summary schema omits persisted jobs") + } + job := properties["jobs"].(map[string]any)["items"].(map[string]any) + for _, name := range []string{"catalog_written", "manifest_updated"} { + if !slices.Contains(job["required"].([]string), name) { + t.Errorf("job schema omits required persistence flag %s", name) + } + } +} + func TestSchemaTracksOptionalAndNullableJSONFields(t *testing.T) { type payload struct { Name string `json:"name"` diff --git a/cmd/internationalizer/translate.go b/cmd/internationalizer/translate.go index 8d95b16..53d2617 100644 --- a/cmd/internationalizer/translate.go +++ b/cmd/internationalizer/translate.go @@ -121,8 +121,11 @@ func newTranslateCmd() *cobra.Command { summary.PlannedKeys += result.KeysSkipped } summary.GeneratedKeys += result.KeysTranslated + if result.CatalogWritten || result.ManifestUpdated { + summary.PersistedJobs++ + } } - if err != nil && summary.GeneratedKeys > 0 { + if err != nil && summary.PersistedJobs > 0 { status = "partial_failure" } providerCalled := summaryHasProviderCalls(results) @@ -172,6 +175,7 @@ type translationSummary struct { BlockedJobs int `json:"blocked_jobs"` PlannedKeys int `json:"planned_keys"` GeneratedKeys int `json:"generated_keys"` + PersistedJobs int `json:"persisted_jobs"` ErrorCount int `json:"error_count"` } diff --git a/cmd/internationalizer/translate_persistence_test.go b/cmd/internationalizer/translate_persistence_test.go new file mode 100644 index 0000000..bea323c --- /dev/null +++ b/cmd/internationalizer/translate_persistence_test.go @@ -0,0 +1,137 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/message" + "gopkg.in/yaml.v3" +) + +func persistenceCLIProject(t *testing.T, source, target, endpoint string, locales []string) (string, *config.Config) { + t.Helper() + dir := t.TempDir() + cfg := &config.Config{ + SourceLocale: "en", TargetLocales: locales, MessageSyntax: message.Plain, + SourcePath: filepath.Join(dir, "en.json"), BatchSize: 1, Concurrency: 1, + TMPath: filepath.Join(dir, "tm.jsonl"), ManifestPath: filepath.Join(dir, "manifest.json"), + StyleGuidesDir: filepath.Join(dir, "guides"), GlossaryDir: filepath.Join(dir, "glossary"), + LLM: config.LLM{Provider: "openai", Model: "test-model", BaseURL: endpoint, APIKeyEnv: "PERSISTENCE_TEST_KEY"}, + } + t.Setenv("PERSISTENCE_TEST_KEY", "synthetic-test-credential") + for name, content := range map[string]string{"en.json": source, "fr.json": target} { + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } + data, err := yaml.Marshal(cfg) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "config.yml") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + return path, cfg +} + +func persistenceResponse(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{\"a\":\"Un\"}"}}]}`)) +} + +func TestTranslationJSONLaterBatchFailureDoesNotClaimRetainedWork(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + persistenceResponse(w) + return + } + http.Error(w, "synthetic batch failure", http.StatusBadRequest) + })) + defer server.Close() + path, cfg := persistenceCLIProject(t, `{"a":"A","b":"B"}`, `{}`, server.URL, []string{"fr"}) + result, err := runJSONCommand(t, "translate", "--config", path, "--json") + if err == nil || result.Status != "blocked" { + t.Fatalf("unpersisted batches must be blocked, not partial success: status=%s error=%v", result.Status, err) + } + data, _ := json.Marshal(result.Data) + var run translationJSON + if err := json.Unmarshal(data, &run); err != nil { + t.Fatal(err) + } + if run.Summary.GeneratedKeys != 1 || !run.ProviderCalled || run.Summary.PersistedJobs != 0 { + t.Fatalf("generation must remain distinct from persistence: %+v", run) + } + if len(run.Jobs) != 1 || run.Jobs[0].CatalogWritten || run.Jobs[0].ManifestUpdated { + t.Fatalf("uncommitted batch claimed persistence: %+v", run.Jobs) + } + target, err := os.ReadFile(filepath.Join(filepath.Dir(path), "fr.json")) + if err != nil || string(target) != `{}` { + t.Fatalf("failed locale wrote its staged batch: %s %v", target, err) + } + if _, err := os.Stat(cfg.ManifestPath); !os.IsNotExist(err) { + t.Fatalf("unexpected manifest after unpersisted failure: %v", err) + } +} + +func TestTranslationJSONRetainedCachedAndAdoptedJobsArePartialFailures(t *testing.T) { + for _, mode := range []string{"cached", "adopted"} { + t.Run(mode, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + persistenceResponse(w) + })) + defer server.Close() + path, cfg := persistenceCLIProject(t, `{"a":"A"}`, `{"a":"Un"}`, server.URL, []string{"fr", "de"}) + args := []string{"translate", "--config", path, "--json"} + if mode == "cached" { + if err := os.WriteFile(filepath.Join(filepath.Dir(path), "fr.json"), []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if _, err := runJSONCommand(t, "translate", "--config", path, "--locale", "fr", "--json"); err != nil { + t.Fatal(err) + } + for _, stale := range []string{filepath.Join(filepath.Dir(path), "fr.json"), cfg.ManifestPath} { + if err := os.Remove(stale); err != nil { + t.Fatal(err) + } + } + calls.Store(0) + } else { + args = append(args, "--adopt-existing") + } + if err := os.WriteFile(filepath.Join(filepath.Dir(path), "de.json"), []byte(`invalid target`), 0o600); err != nil { + t.Fatal(err) + } + result, err := runJSONCommand(t, args...) + if err == nil || result.Status != "partial_failure" { + t.Fatalf("retained %s job must report partial_failure: status=%s error=%v", mode, result.Status, err) + } + data, _ := json.Marshal(result.Data) + var run translationJSON + if err := json.Unmarshal(data, &run); err != nil { + t.Fatal(err) + } + if run.Summary.PersistedJobs != 1 || run.Summary.GeneratedKeys != 0 || run.ProviderCalled || len(run.Jobs) != 2 { + t.Fatalf("incorrect retained-work summary: %+v", run) + } + if !run.Jobs[0].ManifestUpdated || run.Jobs[0].CatalogWritten != (mode == "cached") || run.Jobs[1].ManifestUpdated || run.Jobs[1].CatalogWritten { + t.Fatalf("persistence was attributed to the wrong jobs: %+v", run.Jobs) + } + if calls.Load() != 0 { + t.Fatalf("%s execution unexpectedly called provider", mode) + } + if _, err := os.Stat(cfg.ManifestPath); err != nil { + t.Fatalf("successful %s job was not retained: %v", mode, err) + } + }) + } +} diff --git a/docs/cli-onboarding.md b/docs/cli-onboarding.md index 6b03c43..bb03854 100644 --- a/docs/cli-onboarding.md +++ b/docs/cli-onboarding.md @@ -170,3 +170,11 @@ does not establish approval. Translation can report `partial_failure` when some work succeeded; inspect its jobs and retained state before retrying. Provider requests are not inherently idempotent, even when completed local work can be reused. + +Translation jobs report `catalog_written` and `manifest_updated` only after +those writes succeed. `summary.persisted_jobs` counts jobs with either kind of +retained update; an error yields `partial_failure` only when this count is +positive. Generated and cached key counters can include staged work discarded +after a later batch fails. They are not persistence receipts. A catalog write +can succeed before a later state update fails, so inspect both flags before +retrying; the catalog, translation memory, and manifest are not one transaction. diff --git a/internal/translate/persistence_test.go b/internal/translate/persistence_test.go new file mode 100644 index 0000000..d8f1c25 --- /dev/null +++ b/internal/translate/persistence_test.go @@ -0,0 +1,96 @@ +package translate + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/llm" +) + +type persistenceProvider struct { + beforeResponse func() +} + +func (*persistenceProvider) Name() string { return "persistence-test" } + +func (p *persistenceProvider) Translate(_ context.Context, _ llm.TranslateRequest) (*llm.TranslateResponse, error) { + if p.beforeResponse != nil { + p.beforeResponse() + } + return &llm.TranslateResponse{Translations: map[string]string{"a": "Un"}}, nil +} + +func TestRunPersistenceFlagsFollowActualWriteBoundaries(t *testing.T) { + for _, tc := range []struct { + name string + catalogWritten bool + manifestUpdated bool + }{ + {name: "success", catalogWritten: true, manifestUpdated: true}, + {name: "catalog failure"}, + {name: "TM failure", catalogWritten: true, manifestUpdated: true}, + {name: "manifest failure", catalogWritten: true}, + {name: "cancellation after catalog", catalogWritten: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + source := filepath.Join(dir, "en.json") + if err := os.WriteFile(source, []byte(`{"a":"A"}`), 0o600); err != nil { + t.Fatal(err) + } + cfg := testConfig(dir, source) + cfg.TMPath = filepath.Join(dir, "tm.jsonl") + cfg.ManifestPath = filepath.Join(dir, "manifest.json") + target := filepath.Join(dir, "fr.json") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + provider := &persistenceProvider{beforeResponse: func() { + var blocked string + switch tc.name { + case "catalog failure": + blocked = target + case "TM failure": + blocked = cfg.TMPath + case "manifest failure": + blocked = cfg.ManifestPath + case "cancellation after catalog": + // The provider completed despite cancellation. Its result is + // committed before Run observes cancellation and skips Save. + cancel() + } + if blocked != "" { + if err := os.Mkdir(blocked, 0o700); err != nil { + t.Error(err) + } + } + }} + results, err := Run(ctx, cfg, provider, Options{}) + if (err != nil) != (tc.name != "success") || len(results) != 1 { + t.Fatalf("results=%+v error=%v", results, err) + } + if tc.name == "cancellation after catalog" && !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + result := results[0] + if result.CatalogWritten != tc.catalogWritten || result.ManifestUpdated != tc.manifestUpdated { + t.Fatalf("persistence flags=%+v, want catalog=%t manifest=%t", result, tc.catalogWritten, tc.manifestUpdated) + } + if result.KeysTranslated != 1 { + t.Fatalf("provider-generated count should not depend on persistence: %+v", result) + } + if tc.catalogWritten { + if info, err := os.Stat(target); err != nil || !info.Mode().IsRegular() { + t.Fatalf("catalog flag has no retained file: %v", err) + } + } + if tc.manifestUpdated { + if info, err := os.Stat(cfg.ManifestPath); err != nil || !info.Mode().IsRegular() { + t.Fatalf("manifest flag has no retained file: %v", err) + } + } + }) + } +} diff --git a/internal/translate/translate.go b/internal/translate/translate.go index b81ed43..067bdfc 100644 --- a/internal/translate/translate.go +++ b/internal/translate/translate.go @@ -34,6 +34,8 @@ type Options struct { // Result holds the outcome of one bundle and locale. Lifecycle counters are // pre-run observations; manual, source-stale, and policy-stale may overlap. +// KeysTranslated counts validated provider output, even when a later failure +// prevents its commit. Persistence flags attest only completed local writes. type Result struct { BlockedBySource bool `json:"blocked_by_source"` SourcePath string `json:"source_path,omitempty"` @@ -54,6 +56,8 @@ type Result struct { KeysSkipped int `json:"keys_skipped"` Batches int `json:"batches"` ProviderCalls int `json:"provider_calls"` + CatalogWritten bool `json:"catalog_written"` + ManifestUpdated bool `json:"manifest_updated"` TokensIn int `json:"tokens_in"` TokensOut int `json:"tokens_out"` Errors []string `json:"errors,omitempty"` @@ -220,6 +224,9 @@ enqueue: if err := manifest.Save(cfg.ManifestPath); err != nil { return results, err } + for i := range results { + results[i].ManifestUpdated = len(outputs[i].updates) > 0 + } } failed := 0 @@ -525,6 +532,7 @@ func translateLocale( result.Errors = append(result.Errors, err.Error()) return jobOutput{result: result} } + result.CatalogWritten = true targetExists = true }