Skip to content

Commit 42cd3cd

Browse files
azretazret
committed
Accept attention best winner from gpt-5.4-2026-03-05 at 1982.30us
1 parent 59029dd commit 42cd3cd

3 files changed

Lines changed: 137 additions & 478 deletions

File tree

‎attention/SCORES.json‎

Lines changed: 35 additions & 187 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
"overall": [
33
{
44
"model": "seed",
5-
"candidate": 7684.1,
6-
"baseline": 7684.1,
5+
"candidate": 8234.2,
6+
"baseline": 8234.2,
77
"ratio_text": "x1.000",
88
"notes": "seeded baseline winner",
99
"direction": "lower_is_better",
@@ -12,101 +12,21 @@
1212
},
1313
{
1414
"model": "gpt-5.4-2026-03-05",
15-
"candidate": 242.1,
16-
"baseline": 7684.1,
17-
"ratio_text": "x31.739",
15+
"candidate": 1982.3,
16+
"baseline": 8234.2,
17+
"ratio_text": "x4.154",
1818
"notes": "best winner: used SIMD intrinsics, handles grouped-query attention",
1919
"direction": "lower_is_better",
20-
"timestamp_utc": "2026-03-13 05:56:00",
21-
"commit": "50210af"
22-
},
23-
{
24-
"model": "gpt-5.4-2026-03-05",
25-
"candidate": 2571.6,
26-
"baseline": 7684.1,
27-
"ratio_text": "x2.988",
28-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
29-
"direction": "lower_is_better",
30-
"timestamp_utc": "2026-03-13 05:58:51",
31-
"commit": "eda5596"
32-
},
33-
{
34-
"model": "gpt-5.4-2026-03-05",
35-
"candidate": 1768.0,
36-
"baseline": 7684.1,
37-
"ratio_text": "x4.346",
38-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
39-
"direction": "lower_is_better",
40-
"timestamp_utc": "2026-03-13 05:59:28",
41-
"commit": "dc473b2"
42-
},
43-
{
44-
"model": "gpt-5.4-2026-03-05",
45-
"candidate": 1783.6,
46-
"baseline": 7684.1,
47-
"ratio_text": "x4.308",
48-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
49-
"direction": "lower_is_better",
50-
"timestamp_utc": "2026-03-13 06:00:06",
51-
"commit": "4c45855"
52-
},
53-
{
54-
"model": "gpt-5.4-2026-03-05",
55-
"candidate": 1826.6,
56-
"baseline": 7684.1,
57-
"ratio_text": "x4.207",
58-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
59-
"direction": "lower_is_better",
60-
"timestamp_utc": "2026-03-13 06:00:46",
61-
"commit": "dec591c"
62-
},
63-
{
64-
"model": "gpt-5.4-2026-03-05",
65-
"candidate": 1861.0,
66-
"baseline": 7684.1,
67-
"ratio_text": "x4.129",
68-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
69-
"direction": "lower_is_better",
70-
"timestamp_utc": "2026-03-13 06:01:22",
71-
"commit": "348e884"
72-
},
73-
{
74-
"model": "gpt-5.4-2026-03-05",
75-
"candidate": 1808.9,
76-
"baseline": 7684.1,
77-
"ratio_text": "x4.248",
78-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
79-
"direction": "lower_is_better",
80-
"timestamp_utc": "2026-03-13 06:03:05",
81-
"commit": "28c37da"
82-
},
83-
{
84-
"model": "gpt-5.4-2026-03-05",
85-
"candidate": 1765.4,
86-
"baseline": 7684.1,
87-
"ratio_text": "x4.353",
88-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
89-
"direction": "lower_is_better",
90-
"timestamp_utc": "2026-03-13 06:06:00",
91-
"commit": "822a676"
92-
},
93-
{
94-
"model": "gpt-5.4-2026-03-05",
95-
"candidate": 1814.0,
96-
"baseline": 7684.1,
97-
"ratio_text": "x4.236",
98-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
99-
"direction": "lower_is_better",
100-
"timestamp_utc": "2026-03-13 06:08:02",
101-
"commit": "bde3367"
20+
"timestamp_utc": "2026-03-13 06:29:34",
21+
"commit": "pending"
10222
}
10323
],
10424
"cases": [
10525
{
10626
"case_name": "gpt2_bench_ctx32",
10727
"model": "seed",
108-
"candidate": 569.2,
109-
"baseline": 569.2,
28+
"candidate": 622.3,
29+
"baseline": 622.3,
11030
"ratio_text": "x1.000",
11131
"notes": "seeded case baseline",
11232
"direction": "lower_is_better",
@@ -117,8 +37,8 @@
11737
{
11838
"case_name": "gpt2_bench_ctx128",
11939
"model": "seed",
120-
"candidate": 7684.1,
121-
"baseline": 7684.1,
40+
"candidate": 8234.2,
41+
"baseline": 8234.2,
12242
"ratio_text": "x1.000",
12343
"notes": "seeded case baseline",
12444
"direction": "lower_is_better",
@@ -129,8 +49,8 @@
12949
{
13050
"case_name": "gpt2_bench_ctx256",
13151
"model": "seed",
132-
"candidate": 34925.1,
133-
"baseline": 34925.1,
52+
"candidate": 35690.3,
53+
"baseline": 35690.3,
13454
"ratio_text": "x1.000",
13555
"notes": "seeded case baseline",
13656
"direction": "lower_is_better",
@@ -141,117 +61,45 @@
14161
{
14262
"case_name": "gpt2_bench_ctx32",
14363
"model": "gpt-5.4-2026-03-05",
144-
"candidate": 242.1,
145-
"baseline": 569.2,
146-
"ratio_text": "x2.351",
64+
"candidate": 244.0,
65+
"baseline": 622.3,
66+
"ratio_text": "x2.550",
14767
"notes": "best winner: used SIMD intrinsics, handles grouped-query attention",
14868
"direction": "lower_is_better",
14969
"unit": "us",
150-
"timestamp_utc": "2026-03-13 05:56:00",
151-
"commit": "50210af"
152-
},
153-
{
154-
"case_name": "gpt2_bench_ctx128",
155-
"model": "gpt-5.4-2026-03-05",
156-
"candidate": 2571.6,
157-
"baseline": 7684.1,
158-
"ratio_text": "x2.988",
159-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
160-
"direction": "lower_is_better",
161-
"unit": "us",
162-
"timestamp_utc": "2026-03-13 05:58:51",
163-
"commit": "eda5596"
164-
},
165-
{
166-
"case_name": "gpt2_bench_ctx256",
167-
"model": "gpt-5.4-2026-03-05",
168-
"candidate": 10878.1,
169-
"baseline": 34925.1,
170-
"ratio_text": "x3.211",
171-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
172-
"direction": "lower_is_better",
173-
"unit": "us",
174-
"timestamp_utc": "2026-03-13 05:58:51",
175-
"commit": "eda5596"
176-
},
177-
{
178-
"case_name": "gpt2_bench_ctx32",
179-
"model": "gpt-5.4-2026-03-05",
180-
"candidate": 236.3,
181-
"baseline": 242.1,
182-
"ratio_text": "x2.409",
183-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
184-
"direction": "lower_is_better",
185-
"unit": "us",
186-
"timestamp_utc": "2026-03-13 05:59:28",
187-
"commit": "dc473b2"
70+
"timestamp_utc": "2026-03-13 06:29:34",
71+
"commit": "pending"
18872
},
18973
{
19074
"case_name": "gpt2_bench_ctx128",
19175
"model": "gpt-5.4-2026-03-05",
192-
"candidate": 1768.0,
193-
"baseline": 2571.6,
194-
"ratio_text": "x4.346",
195-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
196-
"direction": "lower_is_better",
197-
"unit": "us",
198-
"timestamp_utc": "2026-03-13 05:59:28",
199-
"commit": "dc473b2"
200-
},
201-
{
202-
"case_name": "gpt2_bench_ctx256",
203-
"model": "gpt-5.4-2026-03-05",
204-
"candidate": 6785.0,
205-
"baseline": 10878.1,
206-
"ratio_text": "x5.147",
207-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
208-
"direction": "lower_is_better",
209-
"unit": "us",
210-
"timestamp_utc": "2026-03-13 05:59:28",
211-
"commit": "dc473b2"
212-
},
213-
{
214-
"case_name": "gpt2_bench_ctx32",
215-
"model": "gpt-5.4-2026-03-05",
216-
"candidate": 224.1,
217-
"baseline": 236.3,
218-
"ratio_text": "x2.540",
219-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
220-
"direction": "lower_is_better",
221-
"unit": "us",
222-
"timestamp_utc": "2026-03-13 06:00:06",
223-
"commit": "4c45855"
224-
},
225-
{
226-
"case_name": "gpt2_bench_ctx256",
227-
"model": "gpt-5.4-2026-03-05",
228-
"candidate": 6242.4,
229-
"baseline": 6785.0,
230-
"ratio_text": "x5.595",
231-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
76+
"candidate": 1982.3,
77+
"baseline": 8234.2,
78+
"ratio_text": "x4.154",
79+
"notes": "best winner: used SIMD intrinsics, handles grouped-query attention",
23280
"direction": "lower_is_better",
23381
"unit": "us",
234-
"timestamp_utc": "2026-03-13 06:00:06",
235-
"commit": "4c45855"
82+
"timestamp_utc": "2026-03-13 06:29:34",
83+
"commit": "pending"
23684
},
23785
{
23886
"case_name": "gpt2_bench_ctx256",
23987
"model": "gpt-5.4-2026-03-05",
240-
"candidate": 5882.5,
241-
"baseline": 6242.4,
242-
"ratio_text": "x5.937",
243-
"notes": "accepted nonbest: used SIMD intrinsics, handles grouped-query attention",
88+
"candidate": 6471.5,
89+
"baseline": 35690.3,
90+
"ratio_text": "x5.515",
91+
"notes": "best winner: used SIMD intrinsics, handles grouped-query attention",
24492
"direction": "lower_is_better",
24593
"unit": "us",
246-
"timestamp_utc": "2026-03-13 06:01:22",
247-
"commit": "348e884"
94+
"timestamp_utc": "2026-03-13 06:29:34",
95+
"commit": "pending"
24896
}
24997
],
25098
"usage": {
251-
"prompt_tokens": 108298,
252-
"completion_tokens": 56839,
253-
"total_tokens": 165137,
254-
"cached_prompt_tokens": 10752,
255-
"estimated_cost_usd": 1.0991379999999997
99+
"prompt_tokens": 3147,
100+
"completion_tokens": 1456,
101+
"total_tokens": 4603,
102+
"cached_prompt_tokens": 0,
103+
"estimated_cost_usd": 0.0297075
256104
}
257105
}

‎attention/SCORES.md‎

Lines changed: 13 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,25 @@
11
# Attention Scores
22

3-
- Total prompt tokens used: 108298
4-
- Total completion tokens used: 56839
5-
- Total cached prompt tokens used: 10752
6-
- Total tokens Used: 165137
7-
- Total cost: $1.09913800
3+
- Total prompt tokens used: 3147
4+
- Total completion tokens used: 1456
5+
- Total cached prompt tokens used: 0
6+
- Total tokens Used: 4603
7+
- Total cost: $0.02970750
88

99
## Overall Winners
1010

1111
| Timestamp | Model | Median | Seed | Win | Commit | Notes |
1212
| --- | --- | ---: | ---: | ---: | --- | --- |
13-
| 2026-03-13 05:56:00 | gpt-5.4-2026-03-05 | 242.10 (us) | 7684.10 (us) | x31.739 | [50210af](D:/loop.py/attention/winners/50210af.c) | best winner: used SIMD intrinsics, handles grouped-query attention |
14-
| 2026-03-13 06:06:00 | gpt-5.4-2026-03-05 | 1765.40 (us) | 7684.10 (us) | x4.353 | [822a676](D:/loop.py/attention/winners/822a676.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
15-
| 2026-03-13 05:59:28 | gpt-5.4-2026-03-05 | 1768.00 (us) | 7684.10 (us) | x4.346 | [dc473b2](D:/loop.py/attention/winners/dc473b2.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
16-
| 2026-03-13 06:00:06 | gpt-5.4-2026-03-05 | 1783.60 (us) | 7684.10 (us) | x4.308 | [4c45855](D:/loop.py/attention/winners/4c45855.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
17-
| 2026-03-13 06:03:05 | gpt-5.4-2026-03-05 | 1808.90 (us) | 7684.10 (us) | x4.248 | [28c37da](D:/loop.py/attention/winners/28c37da.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
18-
| 2026-03-13 06:08:02 | gpt-5.4-2026-03-05 | 1814.00 (us) | 7684.10 (us) | x4.236 | [bde3367](D:/loop.py/attention/winners/bde3367.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
19-
| 2026-03-13 06:00:46 | gpt-5.4-2026-03-05 | 1826.60 (us) | 7684.10 (us) | x4.207 | [dec591c](D:/loop.py/attention/winners/dec591c.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
20-
| 2026-03-13 06:01:22 | gpt-5.4-2026-03-05 | 1861.00 (us) | 7684.10 (us) | x4.129 | [348e884](D:/loop.py/attention/winners/348e884.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
21-
| 2026-03-13 05:58:51 | gpt-5.4-2026-03-05 | 2571.60 (us) | 7684.10 (us) | x2.988 | [eda5596](D:/loop.py/attention/winners/eda5596.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
22-
| seed | seed | 7684.10 (us) | 7684.10 (us) | x1.000 | seed | seeded baseline winner |
13+
| 2026-03-13 06:29:34 | gpt-5.4-2026-03-05 | 1982.30 (us) | 8234.20 (us) | x4.154 | pending | best winner: used SIMD intrinsics, handles grouped-query attention |
14+
| seed | seed | 8234.20 (us) | 8234.20 (us) | x1.000 | seed | seeded baseline winner |
2315

2416
## Per Case Winners
2517

2618
| Case | Timestamp | Model | Median | Case Best | Win | Commit | Notes |
2719
| --- | --- | --- | ---: | ---: | ---: | --- | --- |
28-
| gpt2_bench_ctx128 | 2026-03-13 05:59:28 | gpt-5.4-2026-03-05 | 1768.00 (us) | 2571.60 (us) | x4.346 | [dc473b2](D:/loop.py/attention/winners/dc473b2.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
29-
| gpt2_bench_ctx128 | 2026-03-13 05:58:51 | gpt-5.4-2026-03-05 | 2571.60 (us) | 7684.10 (us) | x2.988 | [eda5596](D:/loop.py/attention/winners/eda5596.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
30-
| gpt2_bench_ctx128 | seed | seed | 7684.10 (us) | 7684.10 (us) | x1.000 | seed | seeded case baseline |
31-
| gpt2_bench_ctx256 | 2026-03-13 06:01:22 | gpt-5.4-2026-03-05 | 5882.50 (us) | 6242.40 (us) | x5.937 | [348e884](D:/loop.py/attention/winners/348e884.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
32-
| gpt2_bench_ctx256 | 2026-03-13 06:00:06 | gpt-5.4-2026-03-05 | 6242.40 (us) | 6785.00 (us) | x5.595 | [4c45855](D:/loop.py/attention/winners/4c45855.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
33-
| gpt2_bench_ctx256 | 2026-03-13 05:59:28 | gpt-5.4-2026-03-05 | 6785.00 (us) | 10878.10 (us) | x5.147 | [dc473b2](D:/loop.py/attention/winners/dc473b2.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
34-
| gpt2_bench_ctx256 | 2026-03-13 05:58:51 | gpt-5.4-2026-03-05 | 10878.10 (us) | 34925.10 (us) | x3.211 | [eda5596](D:/loop.py/attention/winners/eda5596.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
35-
| gpt2_bench_ctx256 | seed | seed | 34925.10 (us) | 34925.10 (us) | x1.000 | seed | seeded case baseline |
36-
| gpt2_bench_ctx32 | 2026-03-13 06:00:06 | gpt-5.4-2026-03-05 | 224.10 (us) | 236.30 (us) | x2.540 | [4c45855](D:/loop.py/attention/winners/4c45855.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
37-
| gpt2_bench_ctx32 | 2026-03-13 05:59:28 | gpt-5.4-2026-03-05 | 236.30 (us) | 242.10 (us) | x2.409 | [dc473b2](D:/loop.py/attention/winners/dc473b2.c) | accepted nonbest: used SIMD intrinsics, handles grouped-query attention |
38-
| gpt2_bench_ctx32 | 2026-03-13 05:56:00 | gpt-5.4-2026-03-05 | 242.10 (us) | 569.20 (us) | x2.351 | [50210af](D:/loop.py/attention/winners/50210af.c) | best winner: used SIMD intrinsics, handles grouped-query attention |
39-
| gpt2_bench_ctx32 | seed | seed | 569.20 (us) | 569.20 (us) | x1.000 | seed | seeded case baseline |
20+
| gpt2_bench_ctx128 | 2026-03-13 06:29:34 | gpt-5.4-2026-03-05 | 1982.30 (us) | 8234.20 (us) | x4.154 | pending | best winner: used SIMD intrinsics, handles grouped-query attention |
21+
| gpt2_bench_ctx128 | seed | seed | 8234.20 (us) | 8234.20 (us) | x1.000 | seed | seeded case baseline |
22+
| gpt2_bench_ctx256 | 2026-03-13 06:29:34 | gpt-5.4-2026-03-05 | 6471.50 (us) | 35690.30 (us) | x5.515 | pending | best winner: used SIMD intrinsics, handles grouped-query attention |
23+
| gpt2_bench_ctx256 | seed | seed | 35690.30 (us) | 35690.30 (us) | x1.000 | seed | seeded case baseline |
24+
| gpt2_bench_ctx32 | 2026-03-13 06:29:34 | gpt-5.4-2026-03-05 | 244.00 (us) | 622.30 (us) | x2.550 | pending | best winner: used SIMD intrinsics, handles grouped-query attention |
25+
| gpt2_bench_ctx32 | seed | seed | 622.30 (us) | 622.30 (us) | x1.000 | seed | seeded case baseline |

0 commit comments

Comments
 (0)