From fddc0e800e7f112ad3d47ed35bafad3f82e99cad Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 17:48:58 -0400 Subject: [PATCH 1/3] fix: reject lossy JSON decoding and serialization Detect decoded duplicate members and flattened leaf collisions before values can be lost. Preserve valid catalog identities and make malformed LLM responses terminal. Signed-off-by: Tom Main --- internal/formats/json.go | 87 ++++++++++++--- internal/formats/json_test.go | 152 ++++++++++++++++++++++++++ internal/jsonintegrity/decode.go | 147 +++++++++++++++++++++++++ internal/jsonintegrity/decode_test.go | 100 +++++++++++++++++ internal/llm/parse.go | 41 ++++++- internal/llm/parse_test.go | 33 +++++- 6 files changed, 533 insertions(+), 27 deletions(-) create mode 100644 internal/jsonintegrity/decode.go create mode 100644 internal/jsonintegrity/decode_test.go diff --git a/internal/formats/json.go b/internal/formats/json.go index 9ab2b66..a8e8afb 100644 --- a/internal/formats/json.go +++ b/internal/formats/json.go @@ -7,6 +7,8 @@ import ( "sort" "strconv" "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" ) type JSONFormat struct{} @@ -15,8 +17,8 @@ func (f *JSONFormat) Name() string { return "json" } func (f *JSONFormat) Extensions() []string { return []string{".json"} } func (f *JSONFormat) Parse(data []byte) (map[string]string, error) { - var raw interface{} - if err := json.Unmarshal(data, &raw); err != nil { + raw, err := jsonintegrity.Decode(data) + if err != nil { return nil, fmt.Errorf("json parse: %w", err) } result := make(map[string]string) @@ -54,10 +56,8 @@ func (f *JSONFormat) Serialize(entries map[string]string, original []byte) ([]by } func (f *JSONFormat) RemoveEntries(original []byte, keys map[string]struct{}) ([]byte, error) { - var raw interface{} - dec := json.NewDecoder(bytes.NewReader(original)) - dec.UseNumber() - if err := dec.Decode(&raw); err != nil { + raw, err := jsonintegrity.Decode(original) + if err != nil { return nil, fmt.Errorf("json parse original: %w", err) } removeJSONEntries("", raw, keys) @@ -68,6 +68,9 @@ func (f *JSONFormat) RemoveEntries(original []byte, keys map[string]struct{}) ([ if err := enc.Encode(raw); err != nil { return nil, err } + if _, err := jsonintegrity.Decode(buf.Bytes()); err != nil { + return nil, err + } return bytes.TrimRight(buf.Bytes(), "\n"), nil } @@ -80,8 +83,10 @@ func removeJSONEntries(prefix string, value interface{}, keys map[string]struct{ path = prefix + "." + key } if _, remove := keys[path]; remove { - delete(node, key) - continue + if _, isString := child.(string); isString { + delete(node, key) + continue + } } removeJSONEntries(path, child, keys) } @@ -96,12 +101,13 @@ func removeJSONEntries(prefix string, value interface{}, keys map[string]struct{ // serializePreservingOrder walks the original JSON structure and replaces // leaf values from the entries map, preserving key ordering. func serializePreservingOrder(entries map[string]string, original []byte) ([]byte, error) { - var raw interface{} - dec := json.NewDecoder(bytes.NewReader(original)) - dec.UseNumber() - if err := dec.Decode(&raw); err != nil { + raw, err := jsonintegrity.Decode(original) + if err != nil { return nil, fmt.Errorf("json parse original: %w", err) } + if raw == nil && len(entries) > 0 { + return nil, fmt.Errorf("json insertion would discard root null metadata") + } replaced := make(map[string]struct{}, len(entries)) if _, rootString := raw.(string); rootString { if replacement, ok := entries[""]; ok { @@ -110,11 +116,19 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt } } replaceLeaves("", raw, entries, replaced) - for key, value := range entries { + keys := make([]string, 0, len(entries)) + for key := range entries { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { if _, ok := replaced[key]; ok { continue } - if err := setPath(&raw, strings.Split(key, "."), value); err != nil { + if strings.Count(key, ".") >= jsonintegrity.MaxDepth { + return nil, &jsonintegrity.Error{Code: "json_nesting_limit"} + } + if err := setPath(&raw, strings.Split(key, "."), entries[key]); err != nil { return nil, fmt.Errorf("json set path %q: %w", key, err) } } @@ -125,6 +139,9 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt if err := enc.Encode(raw); err != nil { return nil, err } + if err := checkSerializedEntries(buf.Bytes(), entries); err != nil { + return nil, err + } // json.Encoder adds a trailing newline; trim then add exactly one. return bytes.TrimRight(buf.Bytes(), "\n"), nil } @@ -140,7 +157,7 @@ func replaceLeaves(prefix string, val interface{}, entries map[string]string, re switch child.(type) { case map[string]interface{}, []interface{}: replaceLeaves(p, child, entries, replaced) - default: + case string: if replacement, ok := entries[p]; ok { v[key] = replacement replaced[p] = struct{}{} @@ -153,7 +170,7 @@ func replaceLeaves(prefix string, val interface{}, entries map[string]string, re switch child.(type) { case map[string]interface{}, []interface{}: replaceLeaves(p, child, entries, replaced) - default: + case string: if replacement, ok := entries[p]; ok { v[i] = replacement replaced[p] = struct{}{} @@ -165,12 +182,21 @@ func replaceLeaves(prefix string, val interface{}, entries map[string]string, re func setPath(target *interface{}, parts []string, value string) error { if len(parts) == 0 { + if *target != nil { + return fmt.Errorf("replacement would discard existing content") + } *target = value return nil } if idx, err := strconv.Atoi(parts[0]); err == nil { if arr, ok := (*target).([]interface{}); ok { + if idx < 0 || idx > len(arr) { + return fmt.Errorf("array index %q is outside existing structure", parts[0]) + } + if idx < len(arr) && (len(parts) == 1 || arr[idx] == nil) { + return fmt.Errorf("replacement would discard existing array content") + } if len(arr) <= idx { expanded := make([]interface{}, idx+1) copy(expanded, arr) @@ -198,7 +224,10 @@ func setPath(target *interface{}, parts []string, value string) error { default: return fmt.Errorf("object segment %q conflicts with existing array or scalar", parts[0]) } - child := obj[parts[0]] + child, exists := obj[parts[0]] + if exists && (len(parts) == 1 || child == nil) { + return fmt.Errorf("replacement would discard existing content") + } if err := setPath(&child, parts[1:], value); err != nil { return err } @@ -217,6 +246,9 @@ func serializeFromScratch(entries map[string]string) ([]byte, error) { sort.Strings(keys) for _, key := range keys { + if strings.Count(key, ".") >= jsonintegrity.MaxDepth { + return nil, &jsonintegrity.Error{Code: "json_nesting_limit"} + } if err := setPath(&root, strings.Split(key, "."), entries[key]); err != nil { return nil, fmt.Errorf("json set path %q: %w", key, err) } @@ -229,5 +261,26 @@ func serializeFromScratch(entries map[string]string) ([]byte, error) { if err := enc.Encode(root); err != nil { return nil, err } + if err := checkSerializedEntries(buf.Bytes(), entries); err != nil { + return nil, err + } return bytes.TrimRight(buf.Bytes(), "\n"), nil } + +func checkSerializedEntries(data []byte, entries map[string]string) error { + parsed, err := (&JSONFormat{}).Parse(data) + if err != nil { + return err + } + keys := make([]string, 0, len(entries)) + for key := range entries { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if value, exists := parsed[key]; !exists || value != entries[key] { + return fmt.Errorf("json set path %q cannot preserve the requested identity", key) + } + } + return nil +} diff --git a/internal/formats/json_test.go b/internal/formats/json_test.go index 557dd42..fa01499 100644 --- a/internal/formats/json_test.go +++ b/internal/formats/json_test.go @@ -2,9 +2,161 @@ package formats import ( "encoding/json" + "errors" + "reflect" + "strings" "testing" ) +func TestJSONRejectsIntegrityLoss(t *testing.T) { + cases := []struct{ name, input, code string }{ + {"duplicate", `{"key":"{{name}}","key":"Hello"}`, "json_duplicate_member"}, + {"escaped duplicate", `{"key":"one","\u006bey":"two"}`, "json_duplicate_member"}, + {"nested duplicate", `{"a":[{"key":"one","key":"two"}]}`, "json_duplicate_member"}, + {"flattened collision", `{"a.b":"one","a":{"b":"two"}}`, "json_flattened_key_collision"}, + {"array collision", `{"a.0":"one","a":["two"]}`, "json_flattened_key_collision"}, + {"metadata collision", `{"a.b":false,"a":{"b":"two"}}`, "json_flattened_key_collision"}, + {"empty key collision", `{"":{"a":"one"},"a":"two"}`, "json_flattened_key_collision"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := &JSONFormat{} + operations := []func() error{ + func() error { _, err := f.Parse([]byte(tc.input)); return err }, + func() error { _, err := f.Serialize(map[string]string{}, []byte(tc.input)); return err }, + func() error { _, err := f.RemoveEntries([]byte(tc.input), map[string]struct{}{}); return err }, + } + for i, operation := range operations { + var last string + for range 100 { + err := operation() + var coded interface{ JSONCode() string } + if !errors.As(err, &coded) || coded.JSONCode() != tc.code { + t.Fatalf("operation %d: got %v, want %s", i, err, tc.code) + } + if last != "" && last != err.Error() { + t.Fatalf("nondeterministic error: %q != %q", last, err.Error()) + } + last = err.Error() + } + } + }) + } +} + +func TestJSONRejectsTrailingContent(t *testing.T) { + f := &JSONFormat{} + for _, input := range []string{`{"key":"one"} {"key":"two"}`, `{"key":"one"} trailing`} { + if _, err := f.Serialize(nil, []byte(input)); err == nil { + t.Fatal("Serialize accepted trailing JSON") + } + if _, err := f.RemoveEntries([]byte(input), nil); err == nil { + t.Fatal("RemoveEntries accepted trailing JSON") + } + } +} + +func TestJSONRejectsUnboundedNesting(t *testing.T) { + if _, err := (&JSONFormat{}).Parse([]byte(strings.Repeat("[", 300) + `"leaf"` + strings.Repeat("]", 300))); err == nil { + t.Fatal("accepted excessive nesting") + } +} + +func TestJSONSerializeRejectsDestructiveInsertion(t *testing.T) { + cases := []struct { + original string + entries map[string]string + }{ + {`{"a":{"b":"Keep"}}`, map[string]string{"a": "Lose child"}}, + {`{"a":null}`, map[string]string{"a": "Lose metadata"}}, + {`{"a":null}`, map[string]string{"a.b": "Lose metadata"}}, + {`{"a":42}`, map[string]string{"a": "Lose metadata"}}, + {`{"a":[null]}`, map[string]string{"a.0": "Lose metadata"}}, + {`{"a":[null]}`, map[string]string{"a.0.b": "Lose metadata"}}, + {`null`, map[string]string{"a": "Lose root metadata"}}, + {`{"a":["Keep"]}`, map[string]string{"a.-1": "Invalid index"}}, + {`{"a":["Keep"]}`, map[string]string{"a.999999999": "Invalid index"}}, + {`{"a":["Keep"]}`, map[string]string{"a.00": "Alias"}}, + {`{}`, map[string]string{".a": "Lost identity"}}, + {"", map[string]string{"a": "one", "a.b": "two"}}, + } + for _, tc := range cases { + if _, err := (&JSONFormat{}).Serialize(tc.entries, []byte(tc.original)); err == nil { + t.Fatalf("accepted destructive insertion into %s with keys %v", tc.original, tc.entries) + } + } +} + +func TestJSONValidDottedContainersRoundTripAndRemoval(t *testing.T) { + f := &JSONFormat{} + for _, original := range []string{`{"a.b":{"c":"X"},"a":{"b":{"d":"Y"}}}`, `{"":{"":"X"}}`, `{"a.b":"X","a":{"b":{"d":"Y"}}}`} { + entries, err := f.Parse([]byte(original)) + if err != nil { + t.Fatal(err) + } + output, err := f.Serialize(entries, []byte(original)) + if err != nil { + t.Fatal(err) + } + reparsed, err := f.Parse(output) + if err != nil || !reflect.DeepEqual(entries, reparsed) { + t.Fatalf("roundtrip: %v", err) + } + } + output, err := f.RemoveEntries([]byte(`{"a.b":"X","a":{"b":{"d":"Y"}},"metadata":true}`), map[string]struct{}{"a.b": {}, "metadata": {}}) + if err != nil { + t.Fatal(err) + } + entries, err := f.Parse(output) + if err != nil || !reflect.DeepEqual(entries, map[string]string{"a.b.d": "Y"}) { + t.Fatalf("removed unrelated content: %s %v", output, err) + } + var raw map[string]any + if err := json.Unmarshal(output, &raw); err != nil { + t.Fatal(err) + } + if raw["metadata"] != true { + t.Fatal("removed nonstring metadata") + } +} + +func FuzzJSONCatalogRoundTrip(f *testing.F) { + for _, input := range []string{`{"a.b":"dotted","a":{"c":"nested"}}`, `{"n":9007199254740993,"ok":true,"items":["a",null]}`, `{"":"empty"}`, `"root"`, `["root array"]`, `{"a":"one","a":"two"}`} { + f.Add(input) + } + f.Fuzz(func(t *testing.T, input string) { + if len(input) > 65536 { + t.Skip() + } + format := &JSONFormat{} + entries, err := format.Parse([]byte(input)) + if err != nil { + return + } + output, err := format.Serialize(entries, []byte(input)) + if err != nil { + t.Fatalf("cannot serialize valid input: %v", err) + } + reparsed, err := format.Parse(output) + if err != nil || !reflect.DeepEqual(entries, reparsed) { + t.Fatalf("catalog identities changed: %v", err) + } + // Compare with json.Number decoding to preserve integers beyond float64. + decode := func(data []byte) any { + decoder := json.NewDecoder(strings.NewReader(string(data))) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + t.Fatal(err) + } + return value + } + if !reflect.DeepEqual(decode([]byte(input)), decode(output)) { + t.Fatal("roundtrip changed non-string metadata or structure") + } + }) +} + func TestJSONParse(t *testing.T) { input := `{ "common": { diff --git a/internal/jsonintegrity/decode.go b/internal/jsonintegrity/decode.go new file mode 100644 index 0000000..85f7aa9 --- /dev/null +++ b/internal/jsonintegrity/decode.go @@ -0,0 +1,147 @@ +// Package jsonintegrity decodes JSON without discarding duplicate members or +// ambiguous dotted identities used by localization catalogs. +package jsonintegrity + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" +) + +// MaxDepth bounds recursive catalog processing, including downstream walkers. +const MaxDepth = 256 + +// Error describes an input integrity failure without including catalog values. +// Paths are RFC 6901 JSON pointers. Offsets distinguish duplicate members whose +// decoded pointers are necessarily identical. +type Error struct { + Code string `json:"code"` + Key string `json:"key"` + Path string `json:"path"` + OtherPath string `json:"other_path"` + Offset int64 `json:"offset"` + OtherOffset int64 `json:"other_offset"` +} + +func (e *Error) JSONCode() string { return e.Code } + +func (e *Error) Error() string { + switch e.Code { + case "json_duplicate_member": + return fmt.Sprintf("duplicate JSON member at %q (byte offsets %d and %d)", e.Path, e.OtherOffset, e.Offset) + case "json_flattened_key_collision": + return fmt.Sprintf("flattened JSON key %q collides between %q and %q", e.Key, e.OtherPath, e.Path) + case "json_nesting_limit": + return fmt.Sprintf("JSON nesting exceeds %d levels", MaxDepth) + default: + return "unexpected content after JSON document" + } +} + +type location struct { + path string + offset int64 +} + +// Decode returns maps, slices, strings, bools, nil and json.Number values. It +// checks leaf identities, including metadata and empty containers, before a map +// insertion could erase information. The first error follows source order. +func Decode(data []byte) (any, error) { + d := json.NewDecoder(bytes.NewReader(data)) + d.UseNumber() + seen := make(map[string]location) + value, err := decodeValue(d, "", "", 0, seen) + if err != nil { + return nil, err + } + if _, err := d.Token(); err != io.EOF { + return nil, &Error{Code: "json_trailing_content", Offset: d.InputOffset()} + } + return value, nil +} + +func decodeValue(d *json.Decoder, key, path string, depth int, seen map[string]location) (any, error) { + if depth > MaxDepth { + return nil, &Error{Code: "json_nesting_limit", Path: path} + } + offset := d.InputOffset() + token, err := d.Token() + if err != nil { + return nil, err + } + switch token { + case json.Delim('{'): + object := make(map[string]any) + members := make(map[string]int64) + if !d.More() && depth > 0 { + if err := registerLeaf(key, path, offset, seen); err != nil { + return nil, err + } + } + for d.More() { + token, err := d.Token() + if err != nil { + return nil, err + } + member, ok := token.(string) + if !ok { + return nil, fmt.Errorf("expected JSON object member") + } + childPath := path + "/" + strings.ReplaceAll(strings.ReplaceAll(member, "~", "~0"), "/", "~1") + childKey := member + if key != "" { + childKey = key + "." + member + } + if offset, exists := members[member]; exists { + return nil, &Error{Code: "json_duplicate_member", Key: childKey, Path: childPath, OtherPath: childPath, Offset: d.InputOffset(), OtherOffset: offset} + } + members[member] = d.InputOffset() + child, err := decodeValue(d, childKey, childPath, depth+1, seen) + if err != nil { + return nil, err + } + object[member] = child + } + if _, err := d.Token(); err != nil { + return nil, err + } + return object, nil + case json.Delim('['): + array := make([]any, 0) + if !d.More() && depth > 0 { + if err := registerLeaf(key, path, offset, seen); err != nil { + return nil, err + } + } + for d.More() { + index := strconv.Itoa(len(array)) + child, err := decodeValue(d, key+"."+index, path+"/"+index, depth+1, seen) + if err != nil { + return nil, err + } + array = append(array, child) + } + if _, err := d.Token(); err != nil { + return nil, err + } + return array, nil + default: + if depth > 0 { + if err := registerLeaf(key, path, offset, seen); err != nil { + return nil, err + } + } + return token, nil + } +} + +func registerLeaf(key, path string, offset int64, seen map[string]location) error { + if previous, ok := seen[key]; ok { + return &Error{Code: "json_flattened_key_collision", Key: key, Path: path, OtherPath: previous.path, Offset: offset, OtherOffset: previous.offset} + } + seen[key] = location{path, offset} + return nil +} diff --git a/internal/jsonintegrity/decode_test.go b/internal/jsonintegrity/decode_test.go new file mode 100644 index 0000000..bb781ed --- /dev/null +++ b/internal/jsonintegrity/decode_test.go @@ -0,0 +1,100 @@ +package jsonintegrity + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" +) + +func TestDecodeIntegrityLocations(t *testing.T) { + cases := []struct{ input, code, key, first, second string }{ + {`{"a.b":"one","a":{"b":"two"}}`, "json_flattened_key_collision", "a.b", "/a.b", "/a/b"}, + {`{"a":{"b":"two"},"a.b":"one"}`, "json_flattened_key_collision", "a.b", "/a/b", "/a.b"}, + {`{"a.0":{},"a":[{}]}`, "json_flattened_key_collision", "a.0", "/a.0", "/a/0"}, + {`{"a.b":null,"a":{"b":false}}`, "json_flattened_key_collision", "a.b", "/a.b", "/a/b"}, + {`{"a/b~":{"x":"one","\u0078":"two"}}`, "json_duplicate_member", "a/b~.x", "/a~1b~0/x", "/a~1b~0/x"}, + {`{"section":{"hello":"one","hello":"two"}}`, "json_duplicate_member", "section.hello", "/section/hello", "/section/hello"}, + {`{"items":[{"hello":"one","hello":"two"}]}`, "json_duplicate_member", "items.0.hello", "/items/0/hello", "/items/0/hello"}, + {`{"":"one","":"one"}`, "json_duplicate_member", "", "/", "/"}, + } + for _, tc := range cases { + t.Run(tc.input, func(t *testing.T) { + _, err := Decode([]byte(tc.input)) + var got *Error + if !errors.As(err, &got) { + t.Fatalf("got %v, want integrity error", err) + } + if got.Code != tc.code || got.Key != tc.key || got.OtherPath != tc.first || got.Path != tc.second { + t.Fatalf("got %#v, want %s %q %q %q", got, tc.code, tc.key, tc.first, tc.second) + } + if got.Offset <= got.OtherOffset { + t.Fatalf("offsets do not distinguish source locations: %#v", got) + } + if strings.Contains(err.Error(), "one") || strings.Contains(err.Error(), "two") { + t.Fatal("error leaked a value") + } + }) + } +} + +func TestDecodeValidNeighboringIdentities(t *testing.T) { + for _, input := range []string{ + `{"a.b":"dotted","a":{"c":"nested"}}`, + `{"a":"flat","a.b":"also flat"}`, + `{"":"empty",".x":"leading dot"}`, + `{"":{"x":"under empty key"}}`, + `{"numeric":{"0":"object"},"array":["array"],"metadata":{"count":9007199254740993,"ok":true,"absent":null}}`, + `["root array",{"nested":"value"}]`, `"root string"`, `null`, + } { + if _, err := Decode([]byte(input)); err != nil { + t.Fatalf("rejected valid input %s: %v", input, err) + } + } +} + +func TestDecodeMalformedAndDepthBoundary(t *testing.T) { + for _, input := range []string{"", `{`, `[`, `{"x":}`, `{"x":1,}`, `[1,]`, `true false`, `{} garbage`} { + if _, err := Decode([]byte(input)); err == nil { + t.Fatalf("accepted %q", input) + } + } + input := strings.Repeat("[", MaxDepth) + "0" + strings.Repeat("]", MaxDepth) + if _, err := Decode([]byte(input)); err != nil { + t.Fatalf("rejected depth limit: %v", err) + } + if _, err := Decode([]byte("[" + input + "]")); err == nil { + t.Fatal("accepted depth beyond limit") + } +} + +func FuzzDecodeDeterministicRoundTrip(f *testing.F) { + for _, input := range []string{`{"a":"x"}`, `{"a":"x","a":"y"}`, `{"a.b":"x","a":{"b":"y"}}`, `{"a":9007199254740993,"items":["x",null]}`, `{"":"x"}`, `"root"`, `{`} { + f.Add(input) + } + f.Fuzz(func(t *testing.T, input string) { + if len(input) > 65536 { + t.Skip() + } + value, err := Decode([]byte(input)) + second, secondErr := Decode([]byte(input)) + if err != nil { + if secondErr == nil || secondErr.Error() != err.Error() { + t.Fatalf("nondeterministic rejection: %v / %v", err, secondErr) + } + return + } + if secondErr != nil || !reflect.DeepEqual(value, second) { + t.Fatalf("nondeterministic successful parse") + } + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + roundTrip, err := Decode(encoded) + if err != nil || !reflect.DeepEqual(value, roundTrip) { + t.Fatalf("roundtrip changed decoded content: %v", err) + } + }) +} diff --git a/internal/llm/parse.go b/internal/llm/parse.go index 9628d62..9ab721d 100644 --- a/internal/llm/parse.go +++ b/internal/llm/parse.go @@ -1,9 +1,12 @@ package llm import ( - "encoding/json" + "errors" "fmt" + "sort" "strings" + + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" ) // ParseTranslationResponse extracts a map[string]string from LLM response text. @@ -14,6 +17,8 @@ func ParseTranslationResponse(text string) (map[string]string, error) { // Try raw JSON first. if result, err := tryParseJSON(text); err == nil { return result, nil + } else if isTerminalParseError(err) { + return nil, err } // Try extracting from markdown code block. @@ -24,6 +29,8 @@ func ParseTranslationResponse(text string) (map[string]string, error) { if end := strings.Index(rest, "```"); end >= 0 { if result, err := tryParseJSON(strings.TrimSpace(rest[:end])); err == nil { return result, nil + } else if isTerminalParseError(err) { + return nil, err } } } @@ -35,29 +42,51 @@ func ParseTranslationResponse(text string) (map[string]string, error) { if first >= 0 && last > first { if result, err := tryParseJSON(text[first : last+1]); err == nil { return result, nil + } else if isTerminalParseError(err) { + return nil, err } } - return nil, fmt.Errorf("could not parse translation response as JSON: %.200s", text) + return nil, fmt.Errorf("could not parse translation response as a JSON object of strings") } +func isTerminalParseError(err error) bool { + var integrity *jsonintegrity.Error + var shape *responseShapeError + return errors.As(err, &integrity) || errors.As(err, &shape) +} + +type responseShapeError struct{ message string } + +func (e *responseShapeError) Error() string { return e.message } + func tryParseJSON(text string) (map[string]string, error) { // Decode through interface values so null and other non-string leaves cannot // silently become empty strings during unmarshalling. - var nested map[string]interface{} - if err := json.Unmarshal([]byte(text), &nested); err != nil { + raw, err := jsonintegrity.Decode([]byte(text)) + if err != nil { return nil, err } + nested, ok := raw.(map[string]interface{}) + if !ok { + return nil, &responseShapeError{message: "translation response must be a JSON object"} + } result := make(map[string]string) if err := flattenResponse("", nested, result); err != nil { - return nil, err + return nil, &responseShapeError{message: err.Error()} } return result, nil } func flattenResponse(prefix string, val map[string]interface{}, out map[string]string) error { - for key, v := range val { + keys := make([]string, 0, len(val)) + for key := range val { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + v := val[key] p := key if prefix != "" { p = prefix + "." + key diff --git a/internal/llm/parse_test.go b/internal/llm/parse_test.go index 4abd2aa..5d5e2ae 100644 --- a/internal/llm/parse_test.go +++ b/internal/llm/parse_test.go @@ -2,10 +2,26 @@ package llm import ( "encoding/json" + "errors" "reflect" "testing" ) +func TestParseTranslationResponseRejectsIntegrityLoss(t *testing.T) { + for _, input := range []string{ + `{"a":"{{name}}","a":"lost"}`, + `{"a.b":"one","a":{"b":"two"}}`, + "```json\n{\"a.b\":\"one\",\"a\":{\"b\":\"two\"}}\n```", + `{"a":"one","a":"two","extra":"` + "```json\n{\"safe\":\"value\"}\n```" + `"}`, + } { + _, err := ParseTranslationResponse(input) + var coded interface{ JSONCode() string } + if !errors.As(err, &coded) { + t.Fatalf("input accepted or lost integrity error: %v", err) + } + } +} + func TestParseTranslationResponse_RawJSON(t *testing.T) { input := `{"common.save": "Enregistrer", "common.cancel": "Annuler"}` result, err := ParseTranslationResponse(input) @@ -62,10 +78,15 @@ func TestParseTranslationResponse_Invalid(t *testing.T) { func TestParseTranslationResponse_RejectsNonStringLeaves(t *testing.T) { tests := map[string]string{ - "boolean": `{"label":true}`, - "number": `{"label":42}`, - "array": `{"label":["Save"]}`, - "null": `{"label":null}`, + "boolean": `{"label":true}`, + "number": `{"label":42}`, + "array": `{"label":["Save"]}`, + "null": `{"label":null}`, + "array of objects": `{"label":[{"hidden":"value"}]}`, + "root array": `[{"hidden":"value"}]`, + "root null": `null`, + "trailing document": `{"label":"one"} {"label":"two"}`, + "trailing scalar": `{"label":"one"} true`, } for name, input := range tests { t.Run(name, func(t *testing.T) { @@ -83,6 +104,10 @@ func FuzzParseTranslationResponseRoundTrip(f *testing.F) { "```json\n{\"save\":\"Save\"}\n```", `not JSON`, `{"value":null}`, + `{"a.b":{"c":"X"},"a":{"b":{"d":"Y"}}}`, + `{"":{"":"X"}}`, + `{"a":"x","a":"y"}`, + `{"a.b":"x","a":{"b":"y"}}`, } { f.Add(seed) } From 59b2b4f43e2a79d155e847698d0e4a9cfe3a9d18 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 17:50:05 -0400 Subject: [PATCH 2/3] fix: expose catalog integrity failures across CLI workflows Retain malformed discovery candidates and structured validation and translation errors. Protect forced pseudolocale writes and cover repeated validation and side-effect-free failures with synthetic acceptance tests. Signed-off-by: Tom Main --- CHANGELOG.md | 6 + cmd/internationalizer/json.go | 19 +- cmd/internationalizer/json_integrity_test.go | 42 +++ docs/cli-onboarding.md | 29 ++ internal/onboarding/discovery.go | 35 ++- internal/onboarding/json_integrity_test.go | 49 ++++ internal/pseudo/generate.go | 7 + internal/translate/translate.go | 56 ++-- internal/validate/findings.go | 16 +- internal/validate/validate.go | 10 +- test/acceptance/json_integrity_test.go | 267 ++++++++++++++++++ .../json_provider_integrity_test.go | 89 ++++++ 12 files changed, 584 insertions(+), 41 deletions(-) create mode 100644 cmd/internationalizer/json_integrity_test.go create mode 100644 internal/onboarding/json_integrity_test.go create mode 100644 test/acceptance/json_integrity_test.go create mode 100644 test/acceptance/json_provider_integrity_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 1700e27..380ade6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Reject duplicate JSON object members and flattened-key collisions before catalog content can be lost. These integrity checks apply with or without `--strict`, including LLM translation responses and catalog rewrite operations. +- Report malformed catalogs during discovery and configuration checks. JSON errors identify conflicting member paths without printing translation values. +- Refuse ambiguous existing JSON pseudolocale targets even with `--force`; repair the catalog before replacing it. + ## 0.2.0 - 2026-09-04 ### Added diff --git a/cmd/internationalizer/json.go b/cmd/internationalizer/json.go index ae654e7..cce24c5 100644 --- a/cmd/internationalizer/json.go +++ b/cmd/internationalizer/json.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/Tom-R-Main/Internationalizer/internal/config" + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" localeid "github.com/Tom-R-Main/Internationalizer/internal/locale" "github.com/spf13/cobra" ) @@ -18,9 +19,10 @@ type recoveryAction struct { } type jsonFailure struct { - Code string `json:"code"` - Message string `json:"message"` - Recovery []recoveryAction `json:"recovery"` + Code string `json:"code"` + Message string `json:"message"` + Recovery []recoveryAction `json:"recovery"` + Details map[string]string `json:"details,omitempty"` } type jsonEnvelope struct { @@ -75,7 +77,14 @@ func emitJSON(cmd *cobra.Command, status string, data any, err error) error { if errors.Is(err, errValidationFailed) { code = "validation_failed" } - envelope.Errors = append(envelope.Errors, jsonFailure{Code: code, Message: safeErrorMessage(err), Recovery: []recoveryAction{errorRecovery(cmd, code)}}) + failure := jsonFailure{Code: code, Message: safeErrorMessage(err), Recovery: []recoveryAction{errorRecovery(cmd, code)}} + var integrity *jsonintegrity.Error + if errors.As(err, &integrity) { + failure.Code = integrity.JSONCode() + failure.Recovery = []recoveryAction{errorRecovery(cmd, failure.Code)} + failure.Details = map[string]string{"key": integrity.Key, "path": integrity.Path, "other_path": integrity.OtherPath} + } + envelope.Errors = append(envelope.Errors, failure) } enc := json.NewEncoder(cmd.OutOrStdout()) enc.SetIndent("", " ") @@ -92,6 +101,8 @@ func errorRecovery(cmd *cobra.Command, code string) recoveryAction { action := recoveryAction{Argv: []string{"internationalizer", "config", "check", "--json"}, SideEffects: []string{}, RequiredDecisions: []string{}} withConfig := true switch strings.ToLower(code) { + case "json_duplicate_member", "json_flattened_key_collision": + action.RequiredDecisions = []string{"repair_catalog_structure_without_discarding_values"} case "invalid_arguments", "invalid_plan": action.Argv = []string{"internationalizer", "commands", "--json"} withConfig = false diff --git a/cmd/internationalizer/json_integrity_test.go b/cmd/internationalizer/json_integrity_test.go new file mode 100644 index 0000000..c01cdfc --- /dev/null +++ b/cmd/internationalizer/json_integrity_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" +) + +func TestIntegrityErrorJSONIncludesLocations(t *testing.T) { + _, err := jsonintegrity.Decode([]byte(`{"a.b":"PRIVATE","a":{"b":"OTHER PRIVATE"}}`)) + if err == nil { + t.Fatal("ambiguous input accepted") + } + cmd := newValidateCmd() + var out bytes.Buffer + cmd.SetOut(&out) + if emitJSON(cmd, "error", nil, fmt.Errorf("parsing source catalog.json: %w", err)) == nil { + t.Fatal("failure swallowed") + } + var envelope struct { + Errors []struct { + Code string + Details map[string]string + } + } + if err := json.Unmarshal(out.Bytes(), &envelope); err != nil { + t.Fatal(err) + } + if len(envelope.Errors) != 1 || envelope.Errors[0].Code != "json_flattened_key_collision" { + t.Fatalf("output = %s", out.String()) + } + details := envelope.Errors[0].Details + if details["path"] != "/a/b" || details["other_path"] != "/a.b" || details["key"] != "a.b" { + t.Fatalf("details = %+v", details) + } + if bytes.Contains(out.Bytes(), []byte("PRIVATE")) { + t.Fatal("catalog value leaked") + } +} diff --git a/docs/cli-onboarding.md b/docs/cli-onboarding.md index bb03854..ce61dd5 100644 --- a/docs/cli-onboarding.md +++ b/docs/cli-onboarding.md @@ -122,6 +122,35 @@ silently reinterpreted as plain text. ## JSON, filtering, and failures +### Catalog integrity + +JSON catalogs must have unique object members and unambiguous flattened keys. +For example, `{"a.b":"First","a":{"b":"Second"}}` gives two values the +same catalog identity, `a.b`, and is rejected. Duplicate members are rejected +even when their values match or their names use different JSON escapes. +These checks apply without `--strict`; sorting keys or choosing the last value +would discard content, not repair it. + +Errors use `json_duplicate_member` or `json_flattened_key_collision`. Source +errors include member locations under `errors[].details`; target validation +findings include `path` and `other_path`. Locations are JSON pointers into the +catalog; the containing error or report identifies the catalog file. Resolve +the conflicting members explicitly, preserving the intended messages and +placeholders. Internationalizer does not choose a surviving value for you. + +Discovery keeps malformed catalog candidates visible with `parse_error_code`. +`detect` remains advisory; `config check` fails when an error diagnostic exists, +even if presentation filters hide it. The same integrity checks protect LLM +response parsing and JSON catalog rewrites. `pseudo --force` bypasses ownership +checks, not JSON integrity checks. + +Translation jobs expose a structured `input_error` when a target catalog or +provider response fails JSON integrity checks. The run can still report +`translation_failed` because other jobs may have completed; inspect the job's +error code and persistence flags before retrying. + +### Output envelope + The onboarding commands, `commands`, `translate`, and `validate` use this envelope when `--json` is supplied: diff --git a/internal/onboarding/discovery.go b/internal/onboarding/discovery.go index b10f4f3..504b990 100644 --- a/internal/onboarding/discovery.go +++ b/internal/onboarding/discovery.go @@ -43,6 +43,7 @@ type Diagnostic struct { } type Candidate struct { + ParseErrorCode string `json:"parse_error_code,omitempty"` ID string `json:"id"` Source string `json:"source"` Target string `json:"target"` @@ -159,11 +160,13 @@ func Scan(root, configPath string) (*Inspection, error) { result.Truncated = true continue } - if _, parseErr := formats.ParseUnits(format, content); parseErr != nil { - continue - } candidate := Candidate{ID: path, Source: path, Target: target, Format: format.Name(), ConfiguredBundles: []string{}, RequiresConfirmation: temporaryPath(path)} candidate.Evidence = []Evidence{{Path: path, Kind: "catalog_path", Detail: "Source-locale filename or directory; storage format does not establish message grammar."}} + if _, parseErr := formats.ParseUnits(format, content); parseErr != nil { + diagnostic := catalogParseDiagnostic(path, "", "CATALOG_PARSE_FAILED", parseErr) + candidate.ParseErrorCode = strings.ToLower(diagnostic.Code) + result.Diagnostics = append(result.Diagnostics, diagnostic) + } applyRuntime(&candidate, nearestRuntime(filepath.Dir(path), runtimes)) result.Candidates = append(result.Candidates, candidate) } @@ -186,6 +189,9 @@ func Scan(root, configPath string) (*Inspection, error) { } for _, candidate := range result.Candidates { if len(candidate.ConfiguredBundles) == 0 { + if candidate.ParseErrorCode != "" { + continue + } result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "UNCOVERED_CATALOG", Severity: "warning", Message: "Detected catalog is not covered by the current configuration: " + candidate.Source, Evidence: candidate.Evidence, RequiredDecisions: []string{"select_authoritative_source", "select_message_syntax"}, Recovery: []Recovery{{Argv: []string{"internationalizer", "config", "plan", "--help"}, SideEffects: []string{}}}}) } } @@ -528,9 +534,17 @@ func inspectBundle(root string, bundle config.Bundle, cfg, raw config.Config, in } units, err := formats.ParseSourceUnits(format, data, bundle.MessageSyntax) if err != nil { - result.Diagnostics = append(result.Diagnostics, Diagnostic{Code: "SOURCE_PARSE_FAILED", Severity: "error", Bundle: bundle.ID, Message: "Configured source could not be parsed using its storage format."}) + result.Diagnostics = append(result.Diagnostics, catalogParseDiagnostic(relativePath(root, source), bundle.ID, "SOURCE_PARSE_FAILED", err)) return resolved } + for _, locale := range cfg.TargetLocales { + path := resolved.Targets[locale] + if data, readErr := safeRead(path); readErr == nil { + if _, parseErr := formats.ParseUnits(format, data); parseErr != nil { + result.Diagnostics = append(result.Diagnostics, catalogParseDiagnostic(relativePath(root, path), bundle.ID, "TARGET_PARSE_FAILED", parseErr)) + } + } + } for _, unit := range units { findings := validate.SyntaxSourceFindings(unit.ID, unit.Value, cfg.SourceLocale, unit.Syntax) codeBraces := false @@ -562,3 +576,16 @@ func inspectBundle(root string, bundle config.Bundle, cfg, raw config.Config, in } return resolved } + +func catalogParseDiagnostic(path, bundle, fallback string, err error) Diagnostic { + d := Diagnostic{Code: fallback, Severity: "error", Bundle: bundle, + Message: "Catalog " + path + " could not be parsed using its storage format.", + Evidence: []Evidence{{Path: path, Kind: "catalog_parse", Detail: "Catalog input failed integrity validation."}}, + Recovery: []Recovery{{Argv: []string{"internationalizer", "config", "check", "--json"}, SideEffects: []string{}, RequiredDecisions: []string{"repair_catalog_structure"}}}} + var coded interface{ JSONCode() string } + if errors.As(err, &coded) { + d.Code = strings.ToUpper(coded.JSONCode()) + d.Message = "Catalog " + path + ": " + err.Error() + } + return d +} diff --git a/internal/onboarding/json_integrity_test.go b/internal/onboarding/json_integrity_test.go new file mode 100644 index 0000000..ddbc90e --- /dev/null +++ b/internal/onboarding/json_integrity_test.go @@ -0,0 +1,49 @@ +package onboarding + +import ( + "strings" + "testing" +) + +func TestScanRetainsMalformedCatalogEvidence(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: configured/en.json\nmessage_syntax: plain\n") + discoveryFile(t, root, "configured/en.json", `{"hello":"Hello"}`) + discoveryFile(t, root, "app/locales/en.json", `{"hello":`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + found := false + for _, d := range inspection.Diagnostics { + if d.Code == "CATALOG_PARSE_FAILED" && d.Severity == "error" && strings.Contains(d.Message, "app/locales/en.json") { + found = true + } + } + if !found { + t.Fatalf("malformed catalog silently disappeared: %+v", inspection) + } +} + +func TestScanReportsJSONIntegrityForConfiguredTargets(t *testing.T) { + root := t.TempDir() + discoveryFile(t, root, ".internationalizer.yml", "source_locale: en\ntarget_locales: [fr]\nsource_path: locales/en.json\nmessage_syntax: plain\n") + discoveryFile(t, root, "locales/en.json", `{"hello":"Hello"}`) + discoveryFile(t, root, "locales/fr.json", `{"hello":"PRIVATE FIRST VALUE","hello":"PRIVATE LAST VALUE"}`) + inspection, err := Scan(root, "") + if err != nil { + t.Fatal(err) + } + found := false + for _, d := range inspection.Diagnostics { + if strings.Contains(d.Message, "PRIVATE") { + t.Fatal("catalog value leaked") + } + if d.Code == "JSON_DUPLICATE_MEMBER" && d.Bundle == "default" && strings.Contains(d.Message, "locales/fr.json") { + found = true + } + } + if !found { + t.Fatalf("target integrity omitted: %+v", inspection.Diagnostics) + } +} diff --git a/internal/pseudo/generate.go b/internal/pseudo/generate.go index 4445cac..0135e72 100644 --- a/internal/pseudo/generate.go +++ b/internal/pseudo/generate.go @@ -88,6 +88,13 @@ func Generate(cfg *config.Config, opts GenerateOptions) ([]GenerateResult, error return nil, err } if existing, readErr := os.ReadFile(targetPath); readErr == nil { + // Force bypasses ownership, not JSON integrity: ambiguous members + // must be repaired explicitly before any catalog replacement. + if format.Name() == "json" { + if _, parseErr := parseTargetValues(format, sourceData, existing); parseErr != nil { + return nil, fmt.Errorf("parsing pseudo target %s: %w", targetPath, parseErr) + } + } if !opts.Force && !pseudoOwnsArtifact(manifest, bundle.ID, canonicalLocale, format, sourceData, existing) { return nil, fmt.Errorf("refusing to overwrite %s without --force because it is not a tracked pseudo artifact", targetPath) } diff --git a/internal/translate/translate.go b/internal/translate/translate.go index 067bdfc..5d1aa66 100644 --- a/internal/translate/translate.go +++ b/internal/translate/translate.go @@ -2,6 +2,7 @@ package translate import ( "context" + "errors" "fmt" "os" "sort" @@ -12,6 +13,7 @@ import ( "github.com/Tom-R-Main/Internationalizer/internal/config" "github.com/Tom-R-Main/Internationalizer/internal/formats" "github.com/Tom-R-Main/Internationalizer/internal/glossary" + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" "github.com/Tom-R-Main/Internationalizer/internal/llm" "github.com/Tom-R-Main/Internationalizer/internal/message" "github.com/Tom-R-Main/Internationalizer/internal/policy" @@ -37,30 +39,31 @@ type Options struct { // KeysTranslated counts validated provider output, even when a later failure // prevents its commit. Persistence flags attest only completed local writes. type Result struct { - BlockedBySource bool `json:"blocked_by_source"` - SourcePath string `json:"source_path,omitempty"` - BlockedLocales []string `json:"blocked_locales,omitempty"` - DryRun bool `json:"dry_run"` - Bundle string `json:"bundle"` - Locale string `json:"locale"` - TargetPath string `json:"target_path"` - KeysTotal int `json:"keys_total"` - KeysMissing int `json:"keys_missing"` - KeysSourceStale int `json:"keys_source_stale"` - KeysPolicyStale int `json:"keys_policy_stale"` - KeysManualEdit int `json:"keys_manual_edit"` - KeysUntracked int `json:"keys_untracked"` - KeysCurrent int `json:"keys_current"` - KeysCached int `json:"keys_cached"` - KeysTranslated int `json:"keys_translated"` - KeysSkipped int `json:"keys_skipped"` - Batches int `json:"batches"` - ProviderCalls int `json:"provider_calls"` - CatalogWritten bool `json:"catalog_written"` - ManifestUpdated bool `json:"manifest_updated"` - TokensIn int `json:"tokens_in"` - TokensOut int `json:"tokens_out"` - Errors []string `json:"errors,omitempty"` + InputError *jsonintegrity.Error `json:"input_error,omitempty"` + BlockedBySource bool `json:"blocked_by_source"` + SourcePath string `json:"source_path,omitempty"` + BlockedLocales []string `json:"blocked_locales,omitempty"` + DryRun bool `json:"dry_run"` + Bundle string `json:"bundle"` + Locale string `json:"locale"` + TargetPath string `json:"target_path"` + KeysTotal int `json:"keys_total"` + KeysMissing int `json:"keys_missing"` + KeysSourceStale int `json:"keys_source_stale"` + KeysPolicyStale int `json:"keys_policy_stale"` + KeysManualEdit int `json:"keys_manual_edit"` + KeysUntracked int `json:"keys_untracked"` + KeysCurrent int `json:"keys_current"` + KeysCached int `json:"keys_cached"` + KeysTranslated int `json:"keys_translated"` + KeysSkipped int `json:"keys_skipped"` + Batches int `json:"batches"` + ProviderCalls int `json:"provider_calls"` + CatalogWritten bool `json:"catalog_written"` + ManifestUpdated bool `json:"manifest_updated"` + TokensIn int `json:"tokens_in"` + TokensOut int `json:"tokens_out"` + Errors []string `json:"errors,omitempty"` } // RunError reports that one or more locale jobs failed. Results remain @@ -260,7 +263,7 @@ func prepareBundles(bundles []config.Bundle) ([]preparedBundle, error) { } units, err := formats.ParseSourceUnits(format, data, bundle.MessageSyntax) if err != nil { - return nil, fmt.Errorf("parsing bundle %q source: %w", bundle.ID, err) + return nil, fmt.Errorf("parsing bundle %q source %s: %w", bundle.ID, bundle.Source, err) } prepared = append(prepared, preparedBundle{bundle: bundle, format: format, sourceUnits: units, sourceKeys: formats.UnitValues(units), sourceData: data}) } @@ -351,6 +354,7 @@ func translateLocale( } if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("parsing target %s: %v", targetPath, err)) + errors.As(err, &result.InputError) return jobOutput{result: result} } case os.IsNotExist(err): @@ -447,6 +451,7 @@ func translateLocale( }) if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("batch %d: %v", i/batchSize+1, err)) + errors.As(err, &result.InputError) return jobOutput{result: result} } if response == nil { @@ -514,6 +519,7 @@ func translateLocale( } if err != nil { result.Errors = append(result.Errors, fmt.Sprintf("serializing target %s: %v", targetPath, err)) + errors.As(err, &result.InputError) return jobOutput{result: result} } output = appendOneNewline(output) diff --git a/internal/validate/findings.go b/internal/validate/findings.go index 1be3c32..4ebfe38 100644 --- a/internal/validate/findings.go +++ b/internal/validate/findings.go @@ -21,6 +21,8 @@ const ( CodePolicyStale FindingCode = "policy_stale" CodeTargetModified FindingCode = "target_modified" CodeNeedsReview FindingCode = "needs_review" + CodeJSONDuplicateMember FindingCode = "json_duplicate_member" + CodeJSONFlattenedKeyCollision FindingCode = "json_flattened_key_collision" ) // Severity determines whether a finding fails validation. @@ -33,12 +35,14 @@ const ( // Finding describes one stable validation outcome for automation and humans. type Finding struct { - Code FindingCode `json:"code"` - Severity Severity `json:"severity"` - Key string `json:"key,omitempty"` - Message string `json:"message"` - Expected []string `json:"expected,omitempty"` - Actual []string `json:"actual,omitempty"` + Path string `json:"path,omitempty"` + OtherPath string `json:"other_path,omitempty"` + Code FindingCode `json:"code"` + Severity Severity `json:"severity"` + Key string `json:"key,omitempty"` + Message string `json:"message"` + Expected []string `json:"expected,omitempty"` + Actual []string `json:"actual,omitempty"` } func sortFindings(findings []Finding) { diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 8f5b784..c38312f 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -1,6 +1,7 @@ package validate import ( + "errors" "fmt" "os" "regexp" @@ -11,6 +12,7 @@ import ( "github.com/Tom-R-Main/Internationalizer/internal/config" "github.com/Tom-R-Main/Internationalizer/internal/formats" "github.com/Tom-R-Main/Internationalizer/internal/glossary" + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" "github.com/Tom-R-Main/Internationalizer/internal/message" "github.com/Tom-R-Main/Internationalizer/internal/policy" "github.com/Tom-R-Main/Internationalizer/internal/state" @@ -91,7 +93,7 @@ func ValidateWithOptions(cfg *config.Config, opts Options) ([]Report, error) { } sourceUnits, err := formats.ParseSourceUnits(format, sourceData, bundle.MessageSyntax) if err != nil { - return nil, fmt.Errorf("parsing bundle %q source: %w", bundle.ID, err) + return nil, fmt.Errorf("parsing bundle %q source %s: %w", bundle.ID, bundle.Source, err) } sourceKeys := formats.UnitValues(sourceUnits) sourceFindings := make(map[string][]Finding) @@ -197,7 +199,11 @@ func validateLocale(bundle, sourceLocale, locale, sourcePath string, sourceData targetKeys, err := parseTarget(format, sourceData, targetData) if err != nil { report.Missing = allKeys(validationKeys) - report.Errors = append(report.Errors, fmt.Sprintf("parsing target: %v", err)) + report.Errors = append(report.Errors, fmt.Sprintf("parsing target %s: %v", targetPath, err)) + var integrity *jsonintegrity.Error + if errors.As(err, &integrity) { + report.Findings = append(report.Findings, Finding{Code: FindingCode(integrity.JSONCode()), Severity: SeverityError, Message: err.Error(), Key: integrity.Key, Path: integrity.Path, OtherPath: integrity.OtherPath}) + } sortFindings(report.Findings) return report } diff --git a/test/acceptance/json_integrity_test.go b/test/acceptance/json_integrity_test.go new file mode 100644 index 0000000..6d5fb2c --- /dev/null +++ b/test/acceptance/json_integrity_test.go @@ -0,0 +1,267 @@ +package acceptance_test + +import ( + "encoding/json" + "fmt" + "io/fs" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" +) + +func jsonIntegrityFixture(t *testing.T, providerURL string) string { + t.Helper() + root := t.TempDir() + mustWriteFile(t, filepath.Join(root, ".internationalizer.yml"), fmt.Sprintf(`source_locale: en +target_locales: [fr] +bundles: + - id: app + source: locales/en.json + target: locales/{locale}.json + message_syntax: i18next +llm: + provider: openai + model: gpt-5.6-luna + api_key_env: JSON_INTEGRITY_TEST_KEY + base_url: %s/api.openai.com +manifest_path: .internationalizer.lock +tm_path: .internationalizer/tm.jsonl +`, providerURL)) + mustWriteFile(t, filepath.Join(root, "locales/en.json"), `{"hello":"Hello {{name}}"}`) + mustWriteFile(t, filepath.Join(root, "locales/fr.json"), `{"hello":"Bonjour {{name}}"}`) + return root +} + +func requireJSONIntegrityFailure(t *testing.T, result cliResult, code string) { + t.Helper() + if result.exitCode == 0 { + t.Fatalf("ambiguous JSON passed: stdout=%s stderr=%s", result.stdout, result.stderr) + } + decodeOnboardingJSON(t, result) + if !strings.Contains(result.stdout, `"`+code+`"`) { + t.Fatalf("missing integrity code %q: stdout=%s stderr=%s", code, result.stdout, result.stderr) + } +} + +func TestJSONIntegrityCollisionAlwaysFailsValidation(t *testing.T) { + root := jsonIntegrityFixture(t, "http://127.0.0.1:1") + mustWriteFile(t, filepath.Join(root, "locales/en.json"), `{"a.b":"Hello {{name}}","a":{"b":"Hello"}}`) + mustWriteFile(t, filepath.Join(root, "locales/fr.json"), `{"a.b":"Bonjour"}`) + for _, strict := range []bool{false, true} { + t.Run(fmt.Sprintf("strict=%t", strict), func(t *testing.T) { + args := []string{"validate", "--json"} + if strict { + args = append(args, "--strict") + } + var first string + for run := range 100 { + result := runCLI(t, root, nil, args...) + requireJSONIntegrityFailure(t, result, "json_flattened_key_collision") + if run == 0 { + first = result.stdout + } else if result.stdout != first { + t.Fatalf("diagnostic changed on run %d:\nfirst=%s\nnow=%s", run+1, first, result.stdout) + } + } + }) + } +} + +func TestJSONIntegrityValidationRejectsSourceAndTargetAmbiguity(t *testing.T) { + cases := []struct{ name, content, code string }{ + {"duplicate", `{"hello":"Hello {{name}}","hello":"Hello"}`, "json_duplicate_member"}, + {"equal-duplicate", `{"hello":"Hello","hello":"Hello"}`, "json_duplicate_member"}, + {"escaped-duplicate", `{"hello":"Hello {{name}}","\u0068ello":"Hello"}`, "json_duplicate_member"}, + {"nested-duplicate", `{"section":{"hello":"Hello {{name}}","hello":"Hello"}}`, "json_duplicate_member"}, + {"nonstring-duplicate", `{"hello":"Hello {{name}}","hello":false}`, "json_duplicate_member"}, + {"flattened", `{"a.b":"Hello {{name}}","a":{"b":"Hello"}}`, "json_flattened_key_collision"}, + {"array-alias", `{"a.0":"Hello {{name}}","a":["Hello"]}`, "json_flattened_key_collision"}, + {"nonstring-alias", `{"a.b":"Hello {{name}}","a":{"b":false}}`, "json_flattened_key_collision"}, + } + for _, tc := range cases { + for _, locale := range []string{"en", "fr"} { + t.Run(tc.name+"/"+locale, func(t *testing.T) { + root := jsonIntegrityFixture(t, "http://127.0.0.1:1") + mustWriteFile(t, filepath.Join(root, "locales", locale+".json"), tc.content) + for _, strict := range []bool{false, true} { + args := []string{"validate", "--json"} + if strict { + args = append(args, "--strict") + } + requireJSONIntegrityFailure(t, runCLI(t, root, nil, args...), tc.code) + } + filtered := runCLI(t, root, nil, "validate", "--json", "--limit", "1", "--finding-code", "missing") + requireJSONIntegrityFailure(t, filtered, tc.code) + }) + } + } +} + +func TestJSONIntegrityDiscoveryRetainsMalformedCatalog(t *testing.T) { + for _, configured := range []bool{false, true} { + t.Run(fmt.Sprintf("configured=%t", configured), func(t *testing.T) { + root := jsonIntegrityFixture(t, "http://127.0.0.1:1") + catalog := "web/locales/en.json" + if configured { + catalog = "locales/en.json" + } + mustWriteFile(t, filepath.Join(root, catalog), `{"hello":"Hello {{name}}","hello":"Hello"}`) + before := jsonIntegritySnapshot(t, root) + for _, args := range [][]string{{"detect", "--json"}, {"config", "check", "--json"}} { + result := runCLI(t, root, nil, args...) + decodeOnboardingJSON(t, result) + if args[0] == "config" && result.exitCode == 0 { + t.Fatalf("config check passed malformed catalog: %s", result.stdout) + } + if !strings.Contains(result.stdout, "JSON_DUPLICATE_MEMBER") { + t.Fatalf("malformed catalog silently disappeared: %s", result.stdout) + } + var decoded struct { + Data struct { + Inspection struct { + Candidates []struct { + Source string `json:"source"` + ParseErrorCode string `json:"parse_error_code"` + } `json:"candidates"` + } `json:"inspection"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(result.stdout), &decoded); err != nil { + t.Fatal(err) + } + found := false + for _, candidate := range decoded.Data.Inspection.Candidates { + if filepath.ToSlash(candidate.Source) == catalog && candidate.ParseErrorCode == "json_duplicate_member" { + found = true + } + } + if !found { + t.Fatalf("malformed candidate %s was not retained with its parse code: %s", catalog, result.stdout) + } + } + if after := jsonIntegritySnapshot(t, root); !reflect.DeepEqual(before, after) { + t.Fatal("discovery changed project files") + } + }) + } +} + +func TestJSONIntegrityMutatingCommandsRejectWithoutSideEffects(t *testing.T) { + for _, command := range []struct { + name string + args []string + }{ + {"translate", []string{"translate", "--json"}}, + {"dry-run", []string{"translate", "--dry-run", "--json"}}, + {"adopt", []string{"translate", "--adopt-existing", "--json"}}, + {"pseudo", []string{"pseudo", "--locale", "en-XA", "--force"}}, + {"approve", []string{"review", "approve", "--locale", "fr", "--all"}}, + } { + for _, input := range []struct{ name, content, diagnostic string }{ + {"duplicate", `{"hello":"Hello {{name}}","hello":"Hello"}`, "duplicate"}, + {"collision", `{"a.b":"Hello {{name}}","a":{"b":"Hello"}}`, "collid"}, + } { + for _, source := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/source=%t", command.name, input.name, source), func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"output":[{"type":"message","content":[{"type":"output_text","text":"{\"hello\":\"Bonjour\"}"}]}]}`)) + })) + defer server.Close() + root := jsonIntegrityFixture(t, server.URL) + if command.name == "approve" { + runCLI(t, root, nil, "translate", "--adopt-existing").requireSuccess(t) + } + locale := "fr" + if command.name == "pseudo" { + locale = "en-XA" + } + if source { + locale = "en" + } + mustWriteFile(t, filepath.Join(root, "locales", locale+".json"), input.content) + before := jsonIntegritySnapshot(t, root) + result := runCLI(t, root, []string{"JSON_INTEGRITY_TEST_KEY=synthetic-only"}, command.args...) + if result.exitCode == 0 { + t.Errorf("command accepted malformed catalog: %s", result.stdout) + } + if !strings.Contains(strings.ToLower(result.stdout+result.stderr), input.diagnostic) { + t.Errorf("failure did not explain %s: stdout=%s stderr=%s", input.diagnostic, result.stdout, result.stderr) + } + if command.args[0] == "translate" { + var output struct { + Errors []struct { + Code string `json:"code"` + } `json:"errors"` + Data struct { + Jobs []struct { + InputError *struct { + Code string `json:"code"` + Path string `json:"path"` + OtherPath string `json:"other_path"` + } `json:"input_error"` + } `json:"jobs"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(result.stdout), &output); err != nil { + t.Fatal(err) + } + code := "json_duplicate_member" + if input.name == "collision" { + code = "json_flattened_key_collision" + } + if source { + if len(output.Errors) != 1 || output.Errors[0].Code != code { + t.Errorf("source integrity code was hidden by generic translation error: %s", result.stdout) + } + } else if len(output.Data.Jobs) != 1 || output.Data.Jobs[0].InputError == nil { + t.Errorf("target job omitted structured input_error: %s", result.stdout) + } else if detail := output.Data.Jobs[0].InputError; detail.Code != code || detail.Path == "" || detail.OtherPath == "" { + t.Errorf("target input_error omitted integrity code or original paths: %s", result.stdout) + } + } + if input.name == "collision" { + for _, pointer := range []string{"/a.b", "/a/b"} { + if !strings.Contains(result.stdout+result.stderr, pointer) { + t.Errorf("collision diagnostic omitted original path %q: stdout=%s stderr=%s", pointer, result.stdout, result.stderr) + } + } + } + if calls.Load() != 0 { + t.Errorf("malformed catalog triggered %d provider requests", calls.Load()) + } + if after := jsonIntegritySnapshot(t, root); !reflect.DeepEqual(before, after) { + t.Errorf("command changed catalog, config, or state after malformed input: before=%v after=%v", before, after) + } + }) + } + } + } +} + +func jsonIntegritySnapshot(t *testing.T, root string) map[string]string { + t.Helper() + files := map[string]string{} + if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() { + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + files[relative] = string(mustReadFile(t, path)) + } + return nil + }); err != nil { + t.Fatal(err) + } + return files +} diff --git a/test/acceptance/json_provider_integrity_test.go b/test/acceptance/json_provider_integrity_test.go new file mode 100644 index 0000000..ad3690b --- /dev/null +++ b/test/acceptance/json_provider_integrity_test.go @@ -0,0 +1,89 @@ +package acceptance_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" + + "github.com/Tom-R-Main/Internationalizer/internal/jsonintegrity" +) + +func TestJSONIntegrityProviderResponseFailsWithoutPersistence(t *testing.T) { + for _, tc := range []struct { + name, response, code, path, otherPath string + }{ + {"duplicate", `{"a.b":"PRIVATE FIRST VALUE","a.b":"PRIVATE LAST VALUE"}`, "json_duplicate_member", "/a.b", "/a.b"}, + {"collision", `{"a.b":"PRIVATE FIRST VALUE","a":{"b":"PRIVATE LAST VALUE"}}`, "json_flattened_key_collision", "/a/b", "/a.b"}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.URL.Path != "/api.openai.com/v1/responses" { + t.Errorf("unexpected provider path: %q", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "output": []any{map[string]any{ + "type": "message", + "content": []any{map[string]any{"type": "output_text", "text": tc.response}}, + }}, + }); err != nil { + t.Errorf("writing mock response: %v", err) + } + })) + defer server.Close() + root := jsonIntegrityFixture(t, server.URL) + // Seed real provenance without a provider, then leave a single new + // source key pending. Existing target, manifest, and TM must survive. + runCLI(t, root, nil, "translate", "--adopt-existing").requireSuccess(t) + mustReadFile(t, filepath.Join(root, ".internationalizer.lock")) + mustWriteFile(t, filepath.Join(root, ".internationalizer", "tm.jsonl"), "{\"bundle\":\"unrelated\",\"key\":\"retained\",\"source\":\"Seed\",\"target\":\"Graine\",\"locale\":\"fr\",\"hash\":\"existing\",\"policy_hash\":\"existing\"}\n") + mustWriteFile(t, filepath.Join(root, "locales", "en.json"), `{"hello":"Hello {{name}}","a.b":"New message"}`) + before := jsonIntegritySnapshot(t, root) + result := runCLI(t, root, []string{"JSON_INTEGRITY_TEST_KEY=synthetic-only"}, "translate", "--json") + requireJSONIntegrityFailure(t, result, tc.code) + var output struct { + Status string `json:"status"` + Data struct { + ProviderCalled bool `json:"provider_called"` + Summary struct { + BlockedJobs int `json:"blocked_jobs"` + GeneratedKeys int `json:"generated_keys"` + PersistedJobs int `json:"persisted_jobs"` + } `json:"summary"` + Jobs []struct { + InputError *jsonintegrity.Error `json:"input_error"` + ProviderCalls int `json:"provider_calls"` + CatalogWritten bool `json:"catalog_written"` + ManifestUpdated bool `json:"manifest_updated"` + } `json:"jobs"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(result.stdout), &output); err != nil { + t.Fatal(err) + } + if calls.Load() != 1 || !output.Data.ProviderCalled || len(output.Data.Jobs) != 1 { + t.Fatalf("expected exactly one attempted provider job, calls=%d: %s", calls.Load(), result.stdout) + } + job := output.Data.Jobs[0] + if job.InputError == nil || job.InputError.Code != tc.code || job.InputError.Key != "a.b" || job.InputError.Path != tc.path || job.InputError.OtherPath != tc.otherPath { + t.Fatalf("provider integrity details missing: %s", result.stdout) + } + if job.ProviderCalls != 1 || job.CatalogWritten || job.ManifestUpdated || output.Status != "blocked" || output.Data.Summary.BlockedJobs != 1 || output.Data.Summary.GeneratedKeys != 0 || output.Data.Summary.PersistedJobs != 0 { + t.Fatalf("failure reported generation or persistence: %s", result.stdout) + } + if strings.Contains(result.stdout+result.stderr, "PRIVATE") { + t.Fatal("provider values leaked through integrity diagnostics") + } + if after := jsonIntegritySnapshot(t, root); !reflect.DeepEqual(before, after) { + t.Fatal("failed provider response changed target, manifest, TM, or other project files") + } + }) + } +} From eaa3b862d025319d78a9eefd2a72c4630d3cfa29 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Fri, 4 Sep 2026 18:06:44 -0400 Subject: [PATCH 3/3] fix: append missing JSON array entries in numeric order Signed-off-by: Tom Main --- internal/formats/json.go | 47 +++++++++++++++++++- internal/formats/json_test.go | 83 +++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 2 deletions(-) diff --git a/internal/formats/json.go b/internal/formats/json.go index a8e8afb..611736b 100644 --- a/internal/formats/json.go +++ b/internal/formats/json.go @@ -120,7 +120,7 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt for key := range entries { keys = append(keys, key) } - sort.Strings(keys) + sort.Slice(keys, func(i, j int) bool { return jsonPathLess(keys[i], keys[j]) }) for _, key := range keys { if _, ok := replaced[key]; ok { continue @@ -128,7 +128,12 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt if strings.Count(key, ".") >= jsonintegrity.MaxDepth { return nil, &jsonintegrity.Error{Code: "json_nesting_limit"} } - if err := setPath(&raw, strings.Split(key, "."), entries[key]); err != nil { + parts := strings.Split(key, ".") + // Root arrays use the same leading-dot identities emitted by flatten. + if _, rootArray := raw.([]interface{}); rootArray && parts[0] == "" { + parts = parts[1:] + } + if err := setPath(&raw, parts, entries[key]); err != nil { return nil, fmt.Errorf("json set path %q: %w", key, err) } } @@ -146,6 +151,44 @@ func serializePreservingOrder(entries map[string]string, original []byte) ([]byt return bytes.TrimRight(buf.Bytes(), "\n"), nil } +// jsonPathLess orders canonical array-index segments numerically so contiguous +// appends do not encounter index 10 before index 2. Numeric and other segments +// have separate ranks; mixing numeric pair comparisons with lexical fallback +// would violate transitivity (for example, "2", "10", and "1x"). +// This changes insertion order, not how existing objects or dotted keys resolve. +func jsonPathLess(left, right string) bool { + lparts, rparts := strings.Split(left, "."), strings.Split(right, ".") + for i := 0; i < len(lparts) && i < len(rparts); i++ { + lpart, rpart := lparts[i], rparts[i] + if lpart == rpart { + continue + } + lnumeric, rnumeric := canonicalJSONIndex(lpart), canonicalJSONIndex(rpart) + if lnumeric != rnumeric { + return lnumeric + } + // Decimal length comparison avoids machine-integer overflow. Equal + // length canonical decimal indices have lexical numeric ordering. + if lnumeric && len(lpart) != len(rpart) { + return len(lpart) < len(rpart) + } + return lpart < rpart + } + return len(lparts) < len(rparts) +} + +func canonicalJSONIndex(segment string) bool { + if segment == "" || (len(segment) > 1 && segment[0] == '0') { + return false + } + for _, c := range segment { + if c < '0' || c > '9' { + return false + } + } + return true +} + func replaceLeaves(prefix string, val interface{}, entries map[string]string, replaced map[string]struct{}) { switch v := val.(type) { case map[string]interface{}: diff --git a/internal/formats/json_test.go b/internal/formats/json_test.go index fa01499..0d4079d 100644 --- a/internal/formats/json_test.go +++ b/internal/formats/json_test.go @@ -3,11 +3,94 @@ package formats import ( "encoding/json" "errors" + "fmt" "reflect" "strings" "testing" ) +func TestJSONSerializeExtendsArraysInIndexOrder(t *testing.T) { + for _, tc := range []struct{ name, original, prefix string }{ + {"array", `{"items":["old0","old1"]}`, "items."}, + {"nested array", `{"groups":[{"items":["old0","old1"]}]}`, "groups.0.items."}, + {"array of arrays", `{"matrix":[["old0","old1"]]}`, "matrix.0."}, + {"root array", `["old0","old1"]`, "."}, + {"numeric object keys", `{"items":{"0":"old0","1":"old1"},"items.note":"dotted"}`, "items."}, + } { + t.Run(tc.name, func(t *testing.T) { + entries := make(map[string]string) + for i := range 13 { + entries[fmt.Sprintf("%s%d", tc.prefix, i)] = fmt.Sprintf("translated%d", i) + } + f := &JSONFormat{} + output, err := f.Serialize(entries, []byte(tc.original)) + if err != nil { + t.Fatal(err) + } + parsed, err := f.Parse(output) + if err != nil { + t.Fatal(err) + } + for key, value := range entries { + if parsed[key] != value { + t.Fatalf("lost %s: %s", key, output) + } + } + var decoded any + if err := json.Unmarshal(output, &decoded); err != nil { + t.Fatal(err) + } + switch tc.name { + case "array": + if _, ok := decoded.(map[string]any)["items"].([]any); !ok { + t.Fatal("array shape changed") + } + case "nested array": + if _, ok := decoded.(map[string]any)["groups"].([]any)[0].(map[string]any)["items"].([]any); !ok { + t.Fatal("nested array shape changed") + } + case "root array": + if _, ok := decoded.([]any); !ok { + t.Fatal("root array shape changed") + } + case "array of arrays": + if _, ok := decoded.(map[string]any)["matrix"].([]any)[0].([]any); !ok { + t.Fatal("inner array shape changed") + } + case "numeric object keys": + if _, ok := decoded.(map[string]any)["items"].(map[string]any); !ok { + t.Fatal("numeric object shape changed") + } + if parsed["items.note"] != "dotted" { + t.Fatal("dotted key lost") + } + } + }) + } +} + +func TestJSONPathOrderingIsStrictAndTransitive(t *testing.T) { + paths := []string{"", ".0", ".2", ".10", "items", "items.0", "items.2", "items.10", "items.1x", "items.02", "items.-1", "items.+1", "items.99999999999999999999999999", "items.2.a", "items.2.3", "items.2.10", "items..x", "x"} + for _, a := range paths { + if jsonPathLess(a, a) { + t.Fatalf("reflexive comparator: %q", a) + } + for _, b := range paths { + if a != b && jsonPathLess(a, b) == jsonPathLess(b, a) { + t.Fatalf("not a total order: %q, %q", a, b) + } + for _, c := range paths { + if jsonPathLess(a, b) && jsonPathLess(b, c) && !jsonPathLess(a, c) { + t.Fatalf("not transitive: %q < %q < %q", a, b, c) + } + } + } + } + if !jsonPathLess("items.2", "items.10") || !jsonPathLess("rows.0.items.2", "rows.0.items.10") { + t.Fatal("array indices not in numeric order") + } +} + func TestJSONRejectsIntegrityLoss(t *testing.T) { cases := []struct{ name, input, code string }{ {"duplicate", `{"key":"{{name}}","key":"Hello"}`, "json_duplicate_member"},