diff --git a/eval/README.md b/eval/README.md index 2009d1f..aa72838 100644 --- a/eval/README.md +++ b/eval/README.md @@ -119,3 +119,69 @@ profile, not on a 0.004 difference in F1. In rough order of value: add cases from a real taxonomy with labels assigned before any model output is seen; split into tune and report halves; raise n past about thirty so confidence intervals mean something. + +## Zero-shot TF-IDF and compression on JevBench + +`jevbench_hybrid.py` measures arbitrary typed decisions using only the current +state, instructions, labels, and option descriptions. There is no labeled +training, support set, learned combination weight, or probability calibration. +TF-IDF vocabulary and IDF are computed from the current request alone; compressor +dictionaries are seeded with that request’s text at inference. + +```sh +git clone https://github.com/fstandhartinger/jevbench.git /tmp/compression-jevbench +git -C /tmp/compression-jevbench checkout 2fa63fa3226cb369795525ed011800f57dcbd894 +OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 uv run --python 3.12 eval/jevbench_hybrid.py \ + --jevbench /tmp/compression-jevbench --out eval/data/jevbench-zero-shot-reproduction +``` + +Run from the repository root. The output directory must be new. Dependencies +are pinned in the script; no API keys, model downloads, or paid calls are needed. +The benchmark repository supplies the 231 public questions and published Jev +outcomes for comparison. Private, sealed, and imported questions are absent. + +The fixed sweep contains 54 methods: five TF-IDF scores (word, window, character, +label-only, and a window/character combination), three compression scores +(gzip normalized compression distance, DEFLATE conditional gain, and Zstd +conditional gain), all 45 pairwise blends at 25/50/75% TF-IDF weight, and one +equal four-way blend. Scores are standardized within each question before +combining them. Negligible numerical variation is collapsed and final scores +rounded before ties are resolved by label name, matching JevBench’s scorer. + +### Public results + +Generated measurements are in [jevbench-zero-shot-results.json](jevbench-zero-shot-results.json). +No training examples were used for any row. Jev is a published reference on +the exact same item IDs, not a fresh API run. + +| Method | Overall (231) | Easy (48) | Original (72) | Hard (111) | +|---|---:|---:|---:|---:| +| Gzip | 42.0% | 52.1% | 34.7% | 42.3% | +| Character TF-IDF | 49.4% | 79.2% | 38.9% | 43.2% | +| Window + character TF-IDF | 48.9% | 79.2% | 40.3% | 41.4% | +| 75% TF-IDF + 25% Zstd (best observed blend) | 50.6% | 81.2% | 41.7% | 43.2% | +| Jev 1.13.0, published | 86.6% | 100.0% | 98.6% | 73.0% | + +The best blend is the high end of an exploratory sweep, not an independently +selected winner. It adds only three correct answers over character TF-IDF +(+1.30 percentage points; paired scenario-bootstrap 95% interval −3.07 to +5.73 +points). This is not convincing evidence of an improvement. Uniform random +choice averages 31.8% on these variable-sized label sets. + +The methods are fast but miss many reasoning decisions: the highlighted blend +gets 5/19 long-policy questions and 5/18 multi-hop questions correct. Running +all 54 scoring rules together took a median 1.53 ms per request on an Apple M5 +Max, including per-request TF-IDF fitting and feature extraction. This excludes +loading, verification, and network/serving overhead. It is not an API latency. + +Every prediction function receives only inference fields. The harness scores +answers afterward and checks every prediction again with reversed option order. +All 12,474 predictions were independently checked with the upstream label scorer. +No probabilities are invented from similarity scores; these are label accuracies, +not official composite scores or a leaderboard submission. + +Each run writes per-item predictions and scores, the full per-family sweep, +source/script hashes, software versions, timing, and paired bootstrap intervals +to `eval/data/`. The checked-in summary retains the full accuracy sweep and +selected family breakdowns; raw predictions stay in the ignored output folder. +The Worker and deployed service are unchanged. diff --git a/eval/jevbench-zero-shot-results.json b/eval/jevbench-zero-shot-results.json new file mode 100644 index 0000000..212ca1e --- /dev/null +++ b/eval/jevbench-zero-shot-results.json @@ -0,0 +1,2093 @@ +{ + "manifest": { + "source_sha256": { + "easy": "231df3c2c8e88a1a8c137ebe85de96ba70fabd330849098ac7b3c52c70b7172b", + "original": "5c2414edb3006b8bfcb70fda433f0f9ca015759433849f8d3104328a1f7c4180", + "hard": "89e9e6becb33ed88c1de7d42dcc87531b2fb64cfaef4e1986faf7c37b3f80ebb" + }, + "script_sha256": "7162b92992aa7ac4706b03e6ee957c5c987059c219810464e75aeb9bdcb02eed", + "platform": "macOS-26.5.1-arm64-arm-64bit", + "python": "3.12.14", + "features": [ + "word_cosine", + "word_window_cosine", + "char_cosine", + "gzip_ncd", + "deflate_gain", + "zstd_gain", + "label_word_cosine" + ], + "versions": { + "numpy": "2.5.3", + "scipy": "1.18.1", + "scikit-learn": "1.9.1", + "zstandard": "0.25.0" + }, + "all_methods_p50_ms": 1.533333, + "all_methods_p95_ms": 11.855979000000001, + "option_reversal_predictions_checked": 12474, + "note": "Public zero-shot diagnostic only; no gold answers or labeled examples enter prediction. IDF is computed per request. All methods are fixed; multiple exploratory comparisons, no official sealed evaluation. Timing includes every method together, excluding verification and loading." + }, + "uniform_chance": 0.3176046176046176, + "versus_tfidf": { + "tfidf-word": { + "accuracy_difference": -0.03463203463203463, + "scenario_bootstrap_95": [ + -0.07860262008733625, + 0.008928571428571428 + ] + }, + "tfidf-window": { + "accuracy_difference": -0.04329004329004329, + "scenario_bootstrap_95": [ + -0.08334205020920502, + -0.004273504273504274 + ] + }, + "tfidf-char": { + "accuracy_difference": 0.004329004329004329, + "scenario_bootstrap_95": [ + -0.047619047619047616, + 0.05803571428571429 + ] + }, + "tfidf-label": { + "accuracy_difference": -0.0735930735930736, + "scenario_bootstrap_95": [ + -0.15625679347826088, + 0.00847457627118644 + ] + }, + "gzip": { + "accuracy_difference": -0.06926406926406926, + "scenario_bootstrap_95": [ + -0.14979060709761585, + 0.013215859030837005 + ] + }, + "deflate": { + "accuracy_difference": -0.09956709956709957, + "scenario_bootstrap_95": [ + -0.17467787416531222, + -0.021929824561403508 + ] + }, + "zstd": { + "accuracy_difference": -0.09090909090909091, + "scenario_bootstrap_95": [ + -0.1630901287553648, + -0.017167381974248927 + ] + }, + "tfidf-word+gzip:0.25": { + "accuracy_difference": -0.05194805194805195, + "scenario_bootstrap_95": [ + -0.12664784522872072, + 0.025423728813559324 + ] + }, + "tfidf-word+gzip:0.5": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.09333333333333334, + 0.013217320962145705 + ] + }, + "tfidf-word+gzip:0.75": { + "accuracy_difference": -0.008658008658008658, + "scenario_bootstrap_95": [ + -0.05194805194805195, + 0.035398230088495575 + ] + }, + "tfidf-word+deflate:0.25": { + "accuracy_difference": -0.08658008658008658, + "scenario_bootstrap_95": [ + -0.16240822320117473, + -0.009048754636989946 + ] + }, + "tfidf-word+deflate:0.5": { + "accuracy_difference": -0.06060606060606061, + "scenario_bootstrap_95": [ + -0.11555555555555555, + -0.00851063829787234 + ] + }, + "tfidf-word+deflate:0.75": { + "accuracy_difference": -0.03463203463203463, + "scenario_bootstrap_95": [ + -0.07488986784140969, + 0.004566733693603109 + ] + }, + "tfidf-word+zstd:0.25": { + "accuracy_difference": -0.09523809523809523, + "scenario_bootstrap_95": [ + -0.1630901287553648, + -0.02643171806167401 + ] + }, + "tfidf-word+zstd:0.5": { + "accuracy_difference": -0.05627705627705628, + "scenario_bootstrap_95": [ + -0.11013215859030837, + 0.0 + ] + }, + "tfidf-word+zstd:0.75": { + "accuracy_difference": -0.030303030303030304, + "scenario_bootstrap_95": [ + -0.07017543859649122, + 0.008849557522123894 + ] + }, + "tfidf-window+gzip:0.25": { + "accuracy_difference": -0.04329004329004329, + "scenario_bootstrap_95": [ + -0.11666666666666667, + 0.03389830508474576 + ] + }, + "tfidf-window+gzip:0.5": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.09210526315789473, + 0.013452914798206279 + ] + }, + "tfidf-window+gzip:0.75": { + "accuracy_difference": -0.021645021645021644, + "scenario_bootstrap_95": [ + -0.06222222222222222, + 0.01832262247026242 + ] + }, + "tfidf-window+deflate:0.25": { + "accuracy_difference": -0.08658008658008658, + "scenario_bootstrap_95": [ + -0.16033755274261605, + -0.01293103448275862 + ] + }, + "tfidf-window+deflate:0.5": { + "accuracy_difference": -0.04329004329004329, + "scenario_bootstrap_95": [ + -0.09649122807017543, + 0.012450736991961628 + ] + }, + "tfidf-window+deflate:0.75": { + "accuracy_difference": -0.04329004329004329, + "scenario_bootstrap_95": [ + -0.08299619640387275, + -0.004328537841468883 + ] + }, + "tfidf-window+zstd:0.25": { + "accuracy_difference": -0.08658008658008658, + "scenario_bootstrap_95": [ + -0.15418502202643172, + -0.021092632805465875 + ] + }, + "tfidf-window+zstd:0.5": { + "accuracy_difference": -0.05627705627705628, + "scenario_bootstrap_95": [ + -0.1091703056768559, + -0.004366812227074236 + ] + }, + "tfidf-window+zstd:0.75": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.07555555555555556, + -0.00423728813559322 + ] + }, + "tfidf-char+gzip:0.25": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.11688311688311688, + 0.039473684210526314 + ] + }, + "tfidf-char+gzip:0.5": { + "accuracy_difference": -0.030303030303030304, + "scenario_bootstrap_95": [ + -0.09583333333333334, + 0.035398230088495575 + ] + }, + "tfidf-char+gzip:0.75": { + "accuracy_difference": -0.008658008658008658, + "scenario_bootstrap_95": [ + -0.06809238043280597, + 0.04911264814221645 + ] + }, + "tfidf-char+deflate:0.25": { + "accuracy_difference": -0.09523809523809523, + "scenario_bootstrap_95": [ + -0.16956521739130434, + -0.020918322873082318 + ] + }, + "tfidf-char+deflate:0.5": { + "accuracy_difference": -0.05627705627705628, + "scenario_bootstrap_95": [ + -0.1227292663476874, + 0.008438818565400843 + ] + }, + "tfidf-char+deflate:0.75": { + "accuracy_difference": -0.012987012987012988, + "scenario_bootstrap_95": [ + -0.06722783283717074, + 0.04310344827586207 + ] + }, + "tfidf-char+zstd:0.25": { + "accuracy_difference": -0.09090909090909091, + "scenario_bootstrap_95": [ + -0.15450643776824036, + -0.02631291657090328 + ] + }, + "tfidf-char+zstd:0.5": { + "accuracy_difference": -0.05627705627705628, + "scenario_bootstrap_95": [ + -0.11063829787234042, + -0.004166666666666667 + ] + }, + "tfidf-char+zstd:0.75": { + "accuracy_difference": 0.0, + "scenario_bootstrap_95": [ + -0.05240174672489083, + 0.05454736440030555 + ] + }, + "tfidf-label+gzip:0.25": { + "accuracy_difference": -0.06926406926406926, + "scenario_bootstrap_95": [ + -0.14977973568281938, + 0.01276595744680851 + ] + }, + "tfidf-label+gzip:0.5": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.11392405063291139, + 0.03508771929824561 + ] + }, + "tfidf-label+gzip:0.75": { + "accuracy_difference": -0.030303030303030304, + "scenario_bootstrap_95": [ + -0.10460251046025104, + 0.043478260869565216 + ] + }, + "tfidf-label+deflate:0.25": { + "accuracy_difference": -0.09956709956709957, + "scenario_bootstrap_95": [ + -0.17672413793103448, + -0.02127659574468085 + ] + }, + "tfidf-label+deflate:0.5": { + "accuracy_difference": -0.06493506493506493, + "scenario_bootstrap_95": [ + -0.13393533549783548, + 0.004291845493562232 + ] + }, + "tfidf-label+deflate:0.75": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.10666666666666667, + 0.030042918454935622 + ] + }, + "tfidf-label+zstd:0.25": { + "accuracy_difference": -0.06926406926406926, + "scenario_bootstrap_95": [ + -0.13964209843030803, + 0.0 + ] + }, + "tfidf-label+zstd:0.5": { + "accuracy_difference": -0.05194805194805195, + "scenario_bootstrap_95": [ + -0.11914893617021277, + 0.01694915254237288 + ] + }, + "tfidf-label+zstd:0.75": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.10300765988567909, + 0.02575107296137339 + ] + }, + "tfidf+gzip:0.25": { + "accuracy_difference": -0.047619047619047616, + "scenario_bootstrap_95": [ + -0.12196151075434804, + 0.02643464192429141 + ] + }, + "tfidf+gzip:0.5": { + "accuracy_difference": -0.03896103896103896, + "scenario_bootstrap_95": [ + -0.10177885520617587, + 0.022323930973734748 + ] + }, + "tfidf+gzip:0.75": { + "accuracy_difference": 0.0, + "scenario_bootstrap_95": [ + -0.047619047619047616, + 0.04782608695652174 + ] + }, + "tfidf+deflate:0.25": { + "accuracy_difference": -0.08658008658008658, + "scenario_bootstrap_95": [ + -0.16017316017316016, + -0.012875536480686695 + ] + }, + "tfidf+deflate:0.5": { + "accuracy_difference": -0.07792207792207792, + "scenario_bootstrap_95": [ + -0.14102564102564102, + -0.013391369047619069 + ] + }, + "tfidf+deflate:0.75": { + "accuracy_difference": -0.012987012987012988, + "scenario_bootstrap_95": [ + -0.05932834475297512, + 0.034782608695652174 + ] + }, + "tfidf+zstd:0.25": { + "accuracy_difference": -0.08225108225108226, + "scenario_bootstrap_95": [ + -0.14530102790014685, + -0.021274341868013014 + ] + }, + "tfidf+zstd:0.5": { + "accuracy_difference": -0.06060606060606061, + "scenario_bootstrap_95": [ + -0.11441004629283955, + -0.008657075682937766 + ] + }, + "tfidf+zstd:0.75": { + "accuracy_difference": 0.017316017316017316, + "scenario_bootstrap_95": [ + -0.013215859030837005, + 0.04954954954954955 + ] + }, + "all-four": { + "accuracy_difference": -0.06493506493506493, + "scenario_bootstrap_95": [ + -0.12334801762114538, + -0.004347826086956522 + ] + } + }, + "published_jev_reference": { + "source_sha256": "c772b3b85809f483449ec2a6ae0272347371ed5f40f7faff8394ecf4ee22ae7c", + "note": "Published Jev 1.13.0 outcomes on these exact public IDs; not rerun.", + "n": 231, + "correct": 200, + "accuracy": 0.8658008658008658, + "by_tier": { + "easy": { + "n": 48, + "correct": 48, + "accuracy": 1.0 + }, + "hard": { + "n": 111, + "correct": 81, + "accuracy": 0.7297297297297297 + }, + "original": { + "n": 72, + "correct": 71, + "accuracy": 0.9861111111111112 + } + }, + "by_family": { + "adequacy": { + "n": 12, + "correct": 12, + "accuracy": 1.0 + }, + "adversarial": { + "n": 6, + "correct": 6, + "accuracy": 1.0 + }, + "ambiguous": { + "n": 7, + "correct": 6, + "accuracy": 0.8571428571428571 + }, + "extraction": { + "n": 24, + "correct": 24, + "accuracy": 1.0 + }, + "fact": { + "n": 12, + "correct": 12, + "accuracy": 1.0 + }, + "intent": { + "n": 24, + "correct": 24, + "accuracy": 1.0 + }, + "judge_hard": { + "n": 17, + "correct": 13, + "accuracy": 0.7647058823529411 + }, + "long_policy": { + "n": 19, + "correct": 12, + "accuracy": 0.631578947368421 + }, + "multi_hop": { + "n": 18, + "correct": 15, + "accuracy": 0.8333333333333334 + }, + "ordinal": { + "n": 12, + "correct": 12, + "accuracy": 1.0 + }, + "policy": { + "n": 12, + "correct": 11, + "accuracy": 0.9166666666666666 + }, + "probability": { + "n": 10, + "correct": 7, + "accuracy": 0.7 + }, + "routing": { + "n": 12, + "correct": 12, + "accuracy": 1.0 + }, + "routing_hard": { + "n": 5, + "correct": 5, + "accuracy": 1.0 + }, + "temporal_numeric": { + "n": 15, + "correct": 4, + "accuracy": 0.26666666666666666 + }, + "tool_selection": { + "n": 12, + "correct": 12, + "accuracy": 1.0 + }, + "tradeoff": { + "n": 6, + "correct": 5, + "accuracy": 0.8333333333333334 + }, + "trap": { + "n": 8, + "correct": 8, + "accuracy": 1.0 + } + } + }, + "methods": { + "tfidf-word": { + "n": 231, + "correct": 105, + "accuracy": 0.45454545454545453, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 41, + "accuracy": 0.36936936936936937 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-window": { + "n": 231, + "correct": 103, + "accuracy": 0.4458874458874459, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 39, + "accuracy": 0.35135135135135137 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-char": { + "n": 231, + "correct": 114, + "accuracy": 0.4935064935064935, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 48, + "accuracy": 0.43243243243243246 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-label": { + "n": 231, + "correct": 96, + "accuracy": 0.4155844155844156, + "by_tier": { + "easy": { + "n": 48, + "correct": 35, + "accuracy": 0.7291666666666666 + }, + "hard": { + "n": 111, + "correct": 35, + "accuracy": 0.3153153153153153 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf": { + "n": 231, + "correct": 113, + "accuracy": 0.48917748917748916, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 46, + "accuracy": 0.4144144144144144 + }, + "original": { + "n": 72, + "correct": 29, + "accuracy": 0.4027777777777778 + } + } + }, + "gzip": { + "n": 231, + "correct": 97, + "accuracy": 0.4199134199134199, + "by_tier": { + "easy": { + "n": 48, + "correct": 25, + "accuracy": 0.5208333333333334 + }, + "hard": { + "n": 111, + "correct": 47, + "accuracy": 0.42342342342342343 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "deflate": { + "n": 231, + "correct": 90, + "accuracy": 0.38961038961038963, + "by_tier": { + "easy": { + "n": 48, + "correct": 32, + "accuracy": 0.6666666666666666 + }, + "hard": { + "n": 111, + "correct": 34, + "accuracy": 0.3063063063063063 + }, + "original": { + "n": 72, + "correct": 24, + "accuracy": 0.3333333333333333 + } + } + }, + "zstd": { + "n": 231, + "correct": 92, + "accuracy": 0.39826839826839827, + "by_tier": { + "easy": { + "n": 48, + "correct": 25, + "accuracy": 0.5208333333333334 + }, + "hard": { + "n": 111, + "correct": 37, + "accuracy": 0.3333333333333333 + }, + "original": { + "n": 72, + "correct": 30, + "accuracy": 0.4166666666666667 + } + } + }, + "tfidf-word+gzip:0.25": { + "n": 231, + "correct": 101, + "accuracy": 0.43722943722943725, + "by_tier": { + "easy": { + "n": 48, + "correct": 30, + "accuracy": 0.625 + }, + "hard": { + "n": 111, + "correct": 44, + "accuracy": 0.3963963963963964 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-word+gzip:0.5": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 44, + "accuracy": 0.3963963963963964 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-word+gzip:0.75": { + "n": 231, + "correct": 111, + "accuracy": 0.4805194805194805, + "by_tier": { + "easy": { + "n": 48, + "correct": 40, + "accuracy": 0.8333333333333334 + }, + "hard": { + "n": 111, + "correct": 44, + "accuracy": 0.3963963963963964 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-word+deflate:0.25": { + "n": 231, + "correct": 93, + "accuracy": 0.4025974025974026, + "by_tier": { + "easy": { + "n": 48, + "correct": 33, + "accuracy": 0.6875 + }, + "hard": { + "n": 111, + "correct": 35, + "accuracy": 0.3153153153153153 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-word+deflate:0.5": { + "n": 231, + "correct": 99, + "accuracy": 0.42857142857142855, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 34, + "accuracy": 0.3063063063063063 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-word+deflate:0.75": { + "n": 231, + "correct": 105, + "accuracy": 0.45454545454545453, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 40, + "accuracy": 0.36036036036036034 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-word+zstd:0.25": { + "n": 231, + "correct": 91, + "accuracy": 0.3939393939393939, + "by_tier": { + "easy": { + "n": 48, + "correct": 29, + "accuracy": 0.6041666666666666 + }, + "hard": { + "n": 111, + "correct": 38, + "accuracy": 0.34234234234234234 + }, + "original": { + "n": 72, + "correct": 24, + "accuracy": 0.3333333333333333 + } + } + }, + "tfidf-word+zstd:0.5": { + "n": 231, + "correct": 100, + "accuracy": 0.4329004329004329, + "by_tier": { + "easy": { + "n": 48, + "correct": 35, + "accuracy": 0.7291666666666666 + }, + "hard": { + "n": 111, + "correct": 40, + "accuracy": 0.36036036036036034 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-word+zstd:0.75": { + "n": 231, + "correct": 106, + "accuracy": 0.4588744588744589, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 41, + "accuracy": 0.36936936936936937 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-window+gzip:0.25": { + "n": 231, + "correct": 103, + "accuracy": 0.4458874458874459, + "by_tier": { + "easy": { + "n": 48, + "correct": 30, + "accuracy": 0.625 + }, + "hard": { + "n": 111, + "correct": 46, + "accuracy": 0.4144144144144144 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-window+gzip:0.5": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 44, + "accuracy": 0.3963963963963964 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-window+gzip:0.75": { + "n": 231, + "correct": 108, + "accuracy": 0.4675324675324675, + "by_tier": { + "easy": { + "n": 48, + "correct": 40, + "accuracy": 0.8333333333333334 + }, + "hard": { + "n": 111, + "correct": 41, + "accuracy": 0.36936936936936937 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-window+deflate:0.25": { + "n": 231, + "correct": 93, + "accuracy": 0.4025974025974026, + "by_tier": { + "easy": { + "n": 48, + "correct": 33, + "accuracy": 0.6875 + }, + "hard": { + "n": 111, + "correct": 35, + "accuracy": 0.3153153153153153 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-window+deflate:0.5": { + "n": 231, + "correct": 103, + "accuracy": 0.4458874458874459, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 38, + "accuracy": 0.34234234234234234 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-window+deflate:0.75": { + "n": 231, + "correct": 103, + "accuracy": 0.4458874458874459, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 38, + "accuracy": 0.34234234234234234 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-window+zstd:0.25": { + "n": 231, + "correct": 93, + "accuracy": 0.4025974025974026, + "by_tier": { + "easy": { + "n": 48, + "correct": 29, + "accuracy": 0.6041666666666666 + }, + "hard": { + "n": 111, + "correct": 40, + "accuracy": 0.36036036036036034 + }, + "original": { + "n": 72, + "correct": 24, + "accuracy": 0.3333333333333333 + } + } + }, + "tfidf-window+zstd:0.5": { + "n": 231, + "correct": 100, + "accuracy": 0.4329004329004329, + "by_tier": { + "easy": { + "n": 48, + "correct": 35, + "accuracy": 0.7291666666666666 + }, + "hard": { + "n": 111, + "correct": 40, + "accuracy": 0.36036036036036034 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-window+zstd:0.75": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 39, + "accuracy": 0.35135135135135137 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-char+gzip:0.25": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 27, + "accuracy": 0.5625 + }, + "hard": { + "n": 111, + "correct": 48, + "accuracy": 0.43243243243243246 + }, + "original": { + "n": 72, + "correct": 29, + "accuracy": 0.4027777777777778 + } + } + }, + "tfidf-char+gzip:0.5": { + "n": 231, + "correct": 106, + "accuracy": 0.4588744588744589, + "by_tier": { + "easy": { + "n": 48, + "correct": 31, + "accuracy": 0.6458333333333334 + }, + "hard": { + "n": 111, + "correct": 48, + "accuracy": 0.43243243243243246 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf-char+gzip:0.75": { + "n": 231, + "correct": 111, + "accuracy": 0.4805194805194805, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 48, + "accuracy": 0.43243243243243246 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-char+deflate:0.25": { + "n": 231, + "correct": 91, + "accuracy": 0.3939393939393939, + "by_tier": { + "easy": { + "n": 48, + "correct": 32, + "accuracy": 0.6666666666666666 + }, + "hard": { + "n": 111, + "correct": 34, + "accuracy": 0.3063063063063063 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-char+deflate:0.5": { + "n": 231, + "correct": 100, + "accuracy": 0.4329004329004329, + "by_tier": { + "easy": { + "n": 48, + "correct": 36, + "accuracy": 0.75 + }, + "hard": { + "n": 111, + "correct": 36, + "accuracy": 0.32432432432432434 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-char+deflate:0.75": { + "n": 231, + "correct": 110, + "accuracy": 0.47619047619047616, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 44, + "accuracy": 0.3963963963963964 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-char+zstd:0.25": { + "n": 231, + "correct": 92, + "accuracy": 0.39826839826839827, + "by_tier": { + "easy": { + "n": 48, + "correct": 28, + "accuracy": 0.5833333333333334 + }, + "hard": { + "n": 111, + "correct": 38, + "accuracy": 0.34234234234234234 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-char+zstd:0.5": { + "n": 231, + "correct": 100, + "accuracy": 0.4329004329004329, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 41, + "accuracy": 0.36936936936936937 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-char+zstd:0.75": { + "n": 231, + "correct": 113, + "accuracy": 0.48917748917748916, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 47, + "accuracy": 0.42342342342342343 + }, + "original": { + "n": 72, + "correct": 28, + "accuracy": 0.3888888888888889 + } + } + }, + "tfidf-label+gzip:0.25": { + "n": 231, + "correct": 97, + "accuracy": 0.4199134199134199, + "by_tier": { + "easy": { + "n": 48, + "correct": 27, + "accuracy": 0.5625 + }, + "hard": { + "n": 111, + "correct": 45, + "accuracy": 0.40540540540540543 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-label+gzip:0.5": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 45, + "accuracy": 0.40540540540540543 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-label+gzip:0.75": { + "n": 231, + "correct": 106, + "accuracy": 0.4588744588744589, + "by_tier": { + "easy": { + "n": 48, + "correct": 39, + "accuracy": 0.8125 + }, + "hard": { + "n": 111, + "correct": 42, + "accuracy": 0.3783783783783784 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf-label+deflate:0.25": { + "n": 231, + "correct": 90, + "accuracy": 0.38961038961038963, + "by_tier": { + "easy": { + "n": 48, + "correct": 33, + "accuracy": 0.6875 + }, + "hard": { + "n": 111, + "correct": 33, + "accuracy": 0.2972972972972973 + }, + "original": { + "n": 72, + "correct": 24, + "accuracy": 0.3333333333333333 + } + } + }, + "tfidf-label+deflate:0.5": { + "n": 231, + "correct": 98, + "accuracy": 0.42424242424242425, + "by_tier": { + "easy": { + "n": 48, + "correct": 38, + "accuracy": 0.7916666666666666 + }, + "hard": { + "n": 111, + "correct": 34, + "accuracy": 0.3063063063063063 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-label+deflate:0.75": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 39, + "accuracy": 0.8125 + }, + "hard": { + "n": 111, + "correct": 39, + "accuracy": 0.35135135135135137 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf-label+zstd:0.25": { + "n": 231, + "correct": 97, + "accuracy": 0.4199134199134199, + "by_tier": { + "easy": { + "n": 48, + "correct": 30, + "accuracy": 0.625 + }, + "hard": { + "n": 111, + "correct": 38, + "accuracy": 0.34234234234234234 + }, + "original": { + "n": 72, + "correct": 29, + "accuracy": 0.4027777777777778 + } + } + }, + "tfidf-label+zstd:0.5": { + "n": 231, + "correct": 101, + "accuracy": 0.43722943722943725, + "by_tier": { + "easy": { + "n": 48, + "correct": 35, + "accuracy": 0.7291666666666666 + }, + "hard": { + "n": 111, + "correct": 35, + "accuracy": 0.3153153153153153 + }, + "original": { + "n": 72, + "correct": 31, + "accuracy": 0.4305555555555556 + } + } + }, + "tfidf-label+zstd:0.75": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 37, + "accuracy": 0.3333333333333333 + }, + "original": { + "n": 72, + "correct": 30, + "accuracy": 0.4166666666666667 + } + } + }, + "tfidf+gzip:0.25": { + "n": 231, + "correct": 102, + "accuracy": 0.44155844155844154, + "by_tier": { + "easy": { + "n": 48, + "correct": 29, + "accuracy": 0.6041666666666666 + }, + "hard": { + "n": 111, + "correct": 46, + "accuracy": 0.4144144144144144 + }, + "original": { + "n": 72, + "correct": 27, + "accuracy": 0.375 + } + } + }, + "tfidf+gzip:0.5": { + "n": 231, + "correct": 104, + "accuracy": 0.45021645021645024, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 46, + "accuracy": 0.4144144144144144 + }, + "original": { + "n": 72, + "correct": 24, + "accuracy": 0.3333333333333333 + } + } + }, + "tfidf+gzip:0.75": { + "n": 231, + "correct": 113, + "accuracy": 0.48917748917748916, + "by_tier": { + "easy": { + "n": 48, + "correct": 40, + "accuracy": 0.8333333333333334 + }, + "hard": { + "n": 111, + "correct": 47, + "accuracy": 0.42342342342342343 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf+deflate:0.25": { + "n": 231, + "correct": 93, + "accuracy": 0.4025974025974026, + "by_tier": { + "easy": { + "n": 48, + "correct": 34, + "accuracy": 0.7083333333333334 + }, + "hard": { + "n": 111, + "correct": 34, + "accuracy": 0.3063063063063063 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf+deflate:0.5": { + "n": 231, + "correct": 95, + "accuracy": 0.41125541125541126, + "by_tier": { + "easy": { + "n": 48, + "correct": 36, + "accuracy": 0.75 + }, + "hard": { + "n": 111, + "correct": 33, + "accuracy": 0.2972972972972973 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf+deflate:0.75": { + "n": 231, + "correct": 110, + "accuracy": 0.47619047619047616, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 47, + "accuracy": 0.42342342342342343 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf+zstd:0.25": { + "n": 231, + "correct": 94, + "accuracy": 0.4069264069264069, + "by_tier": { + "easy": { + "n": 48, + "correct": 29, + "accuracy": 0.6041666666666666 + }, + "hard": { + "n": 111, + "correct": 39, + "accuracy": 0.35135135135135137 + }, + "original": { + "n": 72, + "correct": 26, + "accuracy": 0.3611111111111111 + } + } + }, + "tfidf+zstd:0.5": { + "n": 231, + "correct": 99, + "accuracy": 0.42857142857142855, + "by_tier": { + "easy": { + "n": 48, + "correct": 35, + "accuracy": 0.7291666666666666 + }, + "hard": { + "n": 111, + "correct": 39, + "accuracy": 0.35135135135135137 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + }, + "tfidf+zstd:0.75": { + "n": 231, + "correct": 117, + "accuracy": 0.5064935064935064, + "by_tier": { + "easy": { + "n": 48, + "correct": 39, + "accuracy": 0.8125 + }, + "hard": { + "n": 111, + "correct": 48, + "accuracy": 0.43243243243243246 + }, + "original": { + "n": 72, + "correct": 30, + "accuracy": 0.4166666666666667 + } + } + }, + "all-four": { + "n": 231, + "correct": 98, + "accuracy": 0.42424242424242425, + "by_tier": { + "easy": { + "n": 48, + "correct": 37, + "accuracy": 0.7708333333333334 + }, + "hard": { + "n": 111, + "correct": 36, + "accuracy": 0.32432432432432434 + }, + "original": { + "n": 72, + "correct": 25, + "accuracy": 0.3472222222222222 + } + } + } + }, + "family_detail": { + "tfidf-char": { + "adequacy": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "adversarial": { + "n": 6, + "correct": 4, + "accuracy": 0.6666666666666666 + }, + "ambiguous": { + "n": 7, + "correct": 0, + "accuracy": 0.0 + }, + "extraction": { + "n": 24, + "correct": 15, + "accuracy": 0.625 + }, + "fact": { + "n": 12, + "correct": 8, + "accuracy": 0.6666666666666666 + }, + "intent": { + "n": 24, + "correct": 10, + "accuracy": 0.4166666666666667 + }, + "judge_hard": { + "n": 17, + "correct": 7, + "accuracy": 0.4117647058823529 + }, + "long_policy": { + "n": 19, + "correct": 6, + "accuracy": 0.3157894736842105 + }, + "multi_hop": { + "n": 18, + "correct": 5, + "accuracy": 0.2777777777777778 + }, + "ordinal": { + "n": 12, + "correct": 8, + "accuracy": 0.6666666666666666 + }, + "policy": { + "n": 12, + "correct": 6, + "accuracy": 0.5 + }, + "probability": { + "n": 10, + "correct": 4, + "accuracy": 0.4 + }, + "routing": { + "n": 12, + "correct": 3, + "accuracy": 0.25 + }, + "routing_hard": { + "n": 5, + "correct": 5, + "accuracy": 1.0 + }, + "temporal_numeric": { + "n": 15, + "correct": 8, + "accuracy": 0.5333333333333333 + }, + "tool_selection": { + "n": 12, + "correct": 11, + "accuracy": 0.9166666666666666 + }, + "tradeoff": { + "n": 6, + "correct": 5, + "accuracy": 0.8333333333333334 + }, + "trap": { + "n": 8, + "correct": 4, + "accuracy": 0.5 + } + }, + "tfidf": { + "adequacy": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "adversarial": { + "n": 6, + "correct": 3, + "accuracy": 0.5 + }, + "ambiguous": { + "n": 7, + "correct": 2, + "accuracy": 0.2857142857142857 + }, + "extraction": { + "n": 24, + "correct": 16, + "accuracy": 0.6666666666666666 + }, + "fact": { + "n": 12, + "correct": 10, + "accuracy": 0.8333333333333334 + }, + "intent": { + "n": 24, + "correct": 9, + "accuracy": 0.375 + }, + "judge_hard": { + "n": 17, + "correct": 7, + "accuracy": 0.4117647058823529 + }, + "long_policy": { + "n": 19, + "correct": 6, + "accuracy": 0.3157894736842105 + }, + "multi_hop": { + "n": 18, + "correct": 5, + "accuracy": 0.2777777777777778 + }, + "ordinal": { + "n": 12, + "correct": 8, + "accuracy": 0.6666666666666666 + }, + "policy": { + "n": 12, + "correct": 7, + "accuracy": 0.5833333333333334 + }, + "probability": { + "n": 10, + "correct": 4, + "accuracy": 0.4 + }, + "routing": { + "n": 12, + "correct": 2, + "accuracy": 0.16666666666666666 + }, + "routing_hard": { + "n": 5, + "correct": 5, + "accuracy": 1.0 + }, + "temporal_numeric": { + "n": 15, + "correct": 6, + "accuracy": 0.4 + }, + "tool_selection": { + "n": 12, + "correct": 10, + "accuracy": 0.8333333333333334 + }, + "tradeoff": { + "n": 6, + "correct": 4, + "accuracy": 0.6666666666666666 + }, + "trap": { + "n": 8, + "correct": 4, + "accuracy": 0.5 + } + }, + "gzip": { + "adequacy": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "adversarial": { + "n": 6, + "correct": 3, + "accuracy": 0.5 + }, + "ambiguous": { + "n": 7, + "correct": 3, + "accuracy": 0.42857142857142855 + }, + "extraction": { + "n": 24, + "correct": 11, + "accuracy": 0.4583333333333333 + }, + "fact": { + "n": 12, + "correct": 8, + "accuracy": 0.6666666666666666 + }, + "intent": { + "n": 24, + "correct": 7, + "accuracy": 0.2916666666666667 + }, + "judge_hard": { + "n": 17, + "correct": 7, + "accuracy": 0.4117647058823529 + }, + "long_policy": { + "n": 19, + "correct": 4, + "accuracy": 0.21052631578947367 + }, + "multi_hop": { + "n": 18, + "correct": 4, + "accuracy": 0.2222222222222222 + }, + "ordinal": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "policy": { + "n": 12, + "correct": 6, + "accuracy": 0.5 + }, + "probability": { + "n": 10, + "correct": 5, + "accuracy": 0.5 + }, + "routing": { + "n": 12, + "correct": 3, + "accuracy": 0.25 + }, + "routing_hard": { + "n": 5, + "correct": 5, + "accuracy": 1.0 + }, + "temporal_numeric": { + "n": 15, + "correct": 5, + "accuracy": 0.3333333333333333 + }, + "tool_selection": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "tradeoff": { + "n": 6, + "correct": 6, + "accuracy": 1.0 + }, + "trap": { + "n": 8, + "correct": 5, + "accuracy": 0.625 + } + }, + "tfidf+zstd:0.75": { + "adequacy": { + "n": 12, + "correct": 5, + "accuracy": 0.4166666666666667 + }, + "adversarial": { + "n": 6, + "correct": 3, + "accuracy": 0.5 + }, + "ambiguous": { + "n": 7, + "correct": 1, + "accuracy": 0.14285714285714285 + }, + "extraction": { + "n": 24, + "correct": 16, + "accuracy": 0.6666666666666666 + }, + "fact": { + "n": 12, + "correct": 10, + "accuracy": 0.8333333333333334 + }, + "intent": { + "n": 24, + "correct": 10, + "accuracy": 0.4166666666666667 + }, + "judge_hard": { + "n": 17, + "correct": 7, + "accuracy": 0.4117647058823529 + }, + "long_policy": { + "n": 19, + "correct": 8, + "accuracy": 0.42105263157894735 + }, + "multi_hop": { + "n": 18, + "correct": 4, + "accuracy": 0.2222222222222222 + }, + "ordinal": { + "n": 12, + "correct": 9, + "accuracy": 0.75 + }, + "policy": { + "n": 12, + "correct": 7, + "accuracy": 0.5833333333333334 + }, + "probability": { + "n": 10, + "correct": 5, + "accuracy": 0.5 + }, + "routing": { + "n": 12, + "correct": 2, + "accuracy": 0.16666666666666666 + }, + "routing_hard": { + "n": 5, + "correct": 5, + "accuracy": 1.0 + }, + "temporal_numeric": { + "n": 15, + "correct": 6, + "accuracy": 0.4 + }, + "tool_selection": { + "n": 12, + "correct": 10, + "accuracy": 0.8333333333333334 + }, + "tradeoff": { + "n": 6, + "correct": 5, + "accuracy": 0.8333333333333334 + }, + "trap": { + "n": 8, + "correct": 4, + "accuracy": 0.5 + } + } + }, + "verification": { + "official_scorer_matches": 12474, + "best_hybrid_vs_char_tfidf": { + "accuracy_difference": 0.012987012987012988, + "scenario_bootstrap_95": [ + -0.03070175438596491, + 0.05726872246696035 + ] + }, + "exact_repeatability_predictions_and_scores": 12474 + }, + "selection_note": "The highlighted hybrid is the highest accuracy of 45 fixed blends in an exploratory sweep. It was not independently selected or validated. No gold answers or labeled examples are inputs to any prediction method." +} diff --git a/eval/jevbench_hybrid.py b/eval/jevbench_hybrid.py new file mode 100644 index 0000000..f11308e --- /dev/null +++ b/eval/jevbench_hybrid.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# dependencies = ["numpy==2.5.3", "scipy==1.18.1", "scikit-learn==1.9.1", "zstandard==0.25.0"] +# /// +"""TF-IDF + compression on JevBench's public typed decisions. + +uv run --python 3.12 eval/jevbench_hybrid.py --jevbench /path/to/jevbench + +Zero-shot only: no labeled examples, learned weights, retrieval corpus, or +calibration. Each TF-IDF vocabulary/IDF uses only the current request. Fixed +scoring rules are evaluated side by side; no winner is trained on the answers. +""" +import argparse +import gzip +import hashlib +import importlib.metadata +import json +import platform +import time +import zlib +from pathlib import Path + +import numpy as np +import zstandard as zstd +from sklearn.feature_extraction.text import TfidfVectorizer + +FEATURES = ["word_cosine", "word_window_cosine", "char_cosine", "gzip_ncd", + "deflate_gain", "zstd_gain", "label_word_cosine"] + + +def normalized(text): + return " ".join(text.lower().split()) + + +def centered(values): + v = np.asarray(values, dtype=float) + std = v.std(axis=0) + return np.where(std < 1e-10, 0, (v - v.mean(axis=0)) / np.maximum(std, 1e-10)) + + +def inference_text(task): + state = task["state"] + if not isinstance(state, str): + state = json.dumps(state, ensure_ascii=False, sort_keys=True) + return normalized(state + "\n" + task["question"]["instructions"]) + + +def option_texts(task): + q = task["question"] + criteria = q.get("criteria") or {} + if isinstance(criteria, list): + criteria = {str(i): value for i, value in enumerate(criteria)} + options = [] + for label in task["labels"]: + key = {"yes": "true", "no": "false"}.get(label, label) if q["type"] == "noul" else label + options.append(normalized(label.replace("_", " ") + ": " + str(criteria.get(key) or ""))) + return options + + +def features(task): + text, options = inference_text(task), option_texts(task) + words = text.split() + chunks = [" ".join(words[i:i + 160]) for i in range(0, len(words), 80)] or [""] + docs = [text] + options + chunks + [x.replace("_", " ") for x in task["labels"]] + vectors = TfidfVectorizer(ngram_range=(1, 2), token_pattern=r"(?u)\b\w+\b").fit_transform(docs) + n = len(options) + word = (vectors[1:n+1] @ vectors[0].T).toarray().ravel() + window = (vectors[1:n+1] @ vectors[n+1:n+1+len(chunks)].T).toarray().max(axis=1) + label = (vectors[-n:] @ vectors[0].T).toarray().ravel() + chars = TfidfVectorizer(analyzer="char", ngram_range=(3, 5), max_features=30000).fit_transform([text] + options) + char = (chars[1:] @ chars[0].T).toarray().ravel() + x = text.encode() + cx = len(gzip.compress(x, mtime=0)) + raw = zlib.compressobj(level=6, wbits=-15, zdict=x[-32768:]) + zd = zstd.ZstdCompressor(level=3, dict_data=zstd.ZstdCompressionDict(x, dict_type=zstd.DICT_TYPE_RAWCONTENT)) + zplain = zstd.ZstdCompressor(level=3) + values = [] + for option in options: + y = option.encode() + cy = len(gzip.compress(y, mtime=0)) + joint = min(len(gzip.compress(x + b" " + y, mtime=0)), len(gzip.compress(y + b" " + x, mtime=0))) + c = raw.copy() + conditioned = len(c.compress(y) + c.flush()) + plain = zlib.compressobj(level=6, wbits=-15) + unconditioned = len(plain.compress(y) + plain.flush()) + values.append([-(joint - min(cx, cy)) / max(cx, cy), + (unconditioned - conditioned) / max(len(y), 1), + (len(zplain.compress(y)) - len(zd.compress(y))) / max(len(y), 1)]) + a = np.asarray(values) + return np.column_stack([word, window, char, a, label]) + + +def pick(scores, task): + scores = np.asarray(scores) + ties = np.flatnonzero(scores == scores.max()) + # JevBench's distribution scorer resolves ties by label name. + return int(min(ties, key=lambda i: task["labels"][i])) + + +def methods(f): + word, window, char, gz, df, zs, label = f.T + tfidf = centered((centered(window) + centered(char)) / 2) + out = { + "tfidf-word": centered(word), "tfidf-window": centered(window), + "tfidf-char": centered(char), "tfidf-label": centered(label), + "tfidf": tfidf, "gzip": centered(gz), + "deflate": centered(df), "zstd": centered(zs), + } + for lexical in ["tfidf-word", "tfidf-window", "tfidf-char", "tfidf-label", "tfidf"]: + for compressor in ["gzip", "deflate", "zstd"]: + for weight in [.25, .5, .75]: + out[f"{lexical}+{compressor}:{weight:g}"] = weight * out[lexical] + (1-weight) * out[compressor] + out["all-four"] = (tfidf + out["gzip"] + out["deflate"] + out["zstd"]) / 4 + return {name: np.round(scores, 10) for name, scores in out.items()} + + +def summarize(rows): + def score(group): + return {"n": len(group), "correct": sum(r["correct"] for r in group), + "accuracy": sum(r["correct"] for r in group) / len(group)} + return {**score(rows), + "by_tier": {t: score([r for r in rows if r["tier"] == t]) for t in sorted({r["tier"] for r in rows})}, + "by_family": {f: score([r for r in rows if r["family"] == f]) for f in sorted({r["family"] for r in rows})}} + + +def paired_interval(a, b, seed=0): + groups = sorted({r["group"] for r in a}) + delta = {g: [int(x["correct"]) - int(y["correct"]) for x, y in zip(a, b) if x["group"] == g] for g in groups} + rng = np.random.default_rng(seed) + samples = [] + for _ in range(5000): + chosen = rng.choice(groups, len(groups), replace=True) + differences = [v for g in chosen for v in delta[g]] + samples.append(np.mean(differences)) + return {"accuracy_difference": float(np.mean([int(x["correct"]) - int(y["correct"]) for x, y in zip(a, b)])), + "scenario_bootstrap_95": np.percentile(samples, [2.5, 97.5]).tolist()} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--jevbench", required=True, type=Path) + ap.add_argument("--out", type=Path, default=Path(__file__).parent / "data" / "jevbench-zero-shot") + args = ap.parse_args() + args.out.mkdir(parents=True, exist_ok=False) + tasks, source_hashes = [], {} + for tier in ["easy", "original", "hard"]: + path = args.jevbench / "datasets" / "public" / f"{tier}.jsonl" + source_hashes[tier] = hashlib.sha256(path.read_bytes()).hexdigest() + for line in path.read_text().splitlines(): + row = json.loads(line) + assert row["expected"] is not None and not row.get("provenance", {}).get("exclude_reason") + row["tier"] = tier + tasks.append(row) + outputs, elapsed = {}, [] + for task in tasks: + view = {k: task[k] for k in ["state", "question", "labels"]} + start = time.perf_counter_ns() + f = features(view) + scores = methods(f) + predicted = {name: view["labels"][pick(values, view)] for name, values in scores.items()} + elapsed.append((time.perf_counter_ns() - start) / 1e6) + assert f.shape == (len(view["labels"]), len(FEATURES)) and np.isfinite(f).all() + reversed_view = {**view, "labels": list(reversed(view["labels"]))} + reversed_features = features(reversed_view) + assert np.allclose(f, reversed_features[::-1], atol=1e-12) + reversed_scores = methods(reversed_features) + for name, prediction in predicted.items(): + assert prediction == reversed_view["labels"][pick(reversed_scores[name], reversed_view)], (task["id"], name) + outputs.setdefault(name, []).append({"id": task["id"], "group": task.get("group") or task["id"], + "tier": task["tier"], "family": task["family"], "gold": str(task["expected"]), + "prediction": prediction, "correct": prediction == str(task["expected"]), + "labels": view["labels"], "scores": scores[name].tolist()}) + published_path = args.jevbench / "results/v1.2/jevbench-v1.2-per-task.json" + published = json.loads(published_path.read_text())["systems"]["jev-1.13.0"]["public_tasks"] + reference = [{**r, "correct": published[r["id"]][0] == "c"} for r in outputs["tfidf"]] + report = {"manifest": {"source_sha256": source_hashes, + "script_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "platform": platform.platform(), "python": platform.python_version(), "features": FEATURES, + "versions": {p: importlib.metadata.version(p) for p in ["numpy", "scipy", "scikit-learn", "zstandard"]}, + "all_methods_p50_ms": float(np.median(elapsed)), "all_methods_p95_ms": float(np.percentile(elapsed, 95)), + "option_reversal_predictions_checked": sum(len(v) for v in outputs.values()), + "note": "Public zero-shot diagnostic only; no gold answers or labeled examples enter prediction. IDF is computed per request. All methods are fixed; multiple exploratory comparisons, no official sealed evaluation. Timing includes every method together, excluding verification and loading."}, + "uniform_chance": float(np.mean([1/len(t["labels"]) for t in tasks])), + "methods": {name: summarize(rows) for name, rows in outputs.items()}, + "versus_tfidf": {name: paired_interval(rows, outputs["tfidf"]) for name, rows in outputs.items() if name != "tfidf"}, + "published_jev_reference": {"source_sha256": hashlib.sha256(published_path.read_bytes()).hexdigest(), + "note": "Published Jev 1.13.0 outcomes on these exact public IDs; not rerun.", **summarize(reference)}, + "oracle_choose_tfidf_or_gzip": {"note": "Diagnostic only: uses gold answers to pick the correct method. Not a deployable model or a bound on other hybrids.", + "accuracy": float(np.mean([a["correct"] or b["correct"] for a,b in zip(outputs["tfidf"],outputs["gzip"])]))}} + for name, rows in outputs.items(): + (args.out / f"{name}-predictions.json").write_text(json.dumps(rows, indent=2)) + print(name, json.dumps(report["methods"][name]), flush=True) + (args.out / "summary.json").write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main()