Seth/gemma4-research
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

docs: remove V100 from GPU bakeoff

Browse Source 
V100 data was degraded by SDXL co-residence on CT 167 (31/32 GB VRAM
occupied, Gemma 4 models forced 95% onto CPU). Rather than ship a
prominent caveat, drop the V100 column entirely so the doc reports
only apples-to-apples measurements. V100 can be added back once an
isolated run is possible.

Removed: V100 column from TL;DR and per-model tables, hardware row,
caveat section, and associated raw JSONs under runs/pve197/. Harness
config keeps pve197 in HOSTS for future re-runs.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mortdecai
2026-04-20 05:47:41 -04:00
parent b6190357ba
commit 22af59756f
6 changed files with 31 additions and 420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gpu-bakeoff/runs/pve197/gemma4-26b/long.json
-81
View File
@@ -1,81 +0,0 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 118.0,
"prefill_tok_per_s": 2695.59,
"output_tokens": 256,
"eval_ms": 32720.5,
"decode_tok_per_s": 7.82,
"load_ms": 475.8,
"total_ms": 33548.1,
"harness_wall_s": 33.555,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 118.3,
"prefill_tok_per_s": 2689.01,
"output_tokens": 256,
"eval_ms": 31273.0,
"decode_tok_per_s": 8.19,
"load_ms": 492.5,
"total_ms": 32116.6,
"harness_wall_s": 32.123,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 117.3,
"prefill_tok_per_s": 2711.41,
"output_tokens": 256,
"eval_ms": 33434.9,
"decode_tok_per_s": 7.66,
"load_ms": 496.0,
"total_ms": 34298.7,
"harness_wall_s": 34.305,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 3562.7,
"prefill_tok_per_s": 89.26,
"output_tokens": 256,
"eval_ms": 32215.7,
"decode_tok_per_s": 7.95,
"load_ms": 491.7,
"total_ms": 36521.3,
"harness_wall_s": 36.529,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 2689.01,
"median": 2695.59,
"max": 2711.41,
"n": 3
},
"decode_tok_per_s": {
"min": 7.66,
"median": 7.82,
"max": 8.19,
"n": 3
},
"total_ms": {
"min": 32116.6,
"median": 33548.1,
"max": 34298.7,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-26b/short.json
-81
View File
@@ -1,81 +0,0 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 112.5,
"prefill_tok_per_s": 240.05,
"output_tokens": 256,
"eval_ms": 30919.5,
"decode_tok_per_s": 8.28,
"load_ms": 531.1,
"total_ms": 31828.4,
"harness_wall_s": 31.832,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 113.6,
"prefill_tok_per_s": 237.6,
"output_tokens": 256,
"eval_ms": 30399.9,
"decode_tok_per_s": 8.42,
"load_ms": 479.4,
"total_ms": 31242.1,
"harness_wall_s": 31.246,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 111.0,
"prefill_tok_per_s": 243.16,
"output_tokens": 256,
"eval_ms": 30712.9,
"decode_tok_per_s": 8.34,
"load_ms": 483.2,
"total_ms": 31552.8,
"harness_wall_s": 31.557,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 843.7,
"prefill_tok_per_s": 32.0,
"output_tokens": 256,
"eval_ms": 30499.4,
"decode_tok_per_s": 8.39,
"load_ms": 5877.7,
"total_ms": 37664.4,
"harness_wall_s": 37.668,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 237.6,
"median": 240.05,
"max": 243.16,
"n": 3
},
"decode_tok_per_s": {
"min": 8.28,
"median": 8.34,
"max": 8.42,
"n": 3
},
"total_ms": {
"min": 31242.1,
"median": 31552.8,
"max": 31828.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-31b/long.json
-81
View File
@@ -1,81 +0,0 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 728.7,
"prefill_tok_per_s": 436.37,
"output_tokens": 256,
"eval_ms": 163511.0,
"decode_tok_per_s": 1.57,
"load_ms": 495.0,
"total_ms": 164970.4,
"harness_wall_s": 164.977,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 682.8,
"prefill_tok_per_s": 465.71,
"output_tokens": 256,
"eval_ms": 168727.1,
"decode_tok_per_s": 1.52,
"load_ms": 545.3,
"total_ms": 170207.4,
"harness_wall_s": 170.214,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 950.0,
"prefill_tok_per_s": 334.75,
"output_tokens": 256,
"eval_ms": 163102.9,
"decode_tok_per_s": 1.57,
"load_ms": 507.9,
"total_ms": 164801.8,
"harness_wall_s": 164.809,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 3883.3,
"prefill_tok_per_s": 81.89,
"output_tokens": 256,
"eval_ms": 172199.4,
"decode_tok_per_s": 1.49,
"load_ms": 528.0,
"total_ms": 176864.8,
"harness_wall_s": 176.871,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 334.75,
"median": 436.37,
"max": 465.71,
"n": 3
},
"decode_tok_per_s": {
"min": 1.52,
"median": 1.57,
"max": 1.57,
"n": 3
},
"total_ms": {
"min": 164801.8,
"median": 164970.4,
"max": 170207.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-31b/short.json
-81
View File
@@ -1,81 +0,0 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 665.6,
"prefill_tok_per_s": 40.56,
"output_tokens": 256,
"eval_ms": 164631.1,
"decode_tok_per_s": 1.55,
"load_ms": 512.6,
"total_ms": 166062.7,
"harness_wall_s": 166.067,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 660.3,
"prefill_tok_per_s": 40.89,
"output_tokens": 256,
"eval_ms": 159594.3,
"decode_tok_per_s": 1.6,
"load_ms": 523.6,
"total_ms": 161012.3,
"harness_wall_s": 161.016,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 887.8,
"prefill_tok_per_s": 30.41,
"output_tokens": 256,
"eval_ms": 167584.3,
"decode_tok_per_s": 1.53,
"load_ms": 486.8,
"total_ms": 169188.9,
"harness_wall_s": 169.194,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 6642.4,
"prefill_tok_per_s": 4.06,
"output_tokens": 256,
"eval_ms": 173530.1,
"decode_tok_per_s": 1.48,
"load_ms": 20142.1,
"total_ms": 200836.5,
"harness_wall_s": 200.841,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 30.41,
"median": 40.56,
"max": 40.89,
"n": 3
},
"decode_tok_per_s": {
"min": 1.53,
"median": 1.55,
"max": 1.6,
"n": 3
},
"total_ms": {
"min": 161012.3,
"median": 166062.7,
"max": 169188.9,
"n": 3
}
}
}