Seth/gemma4-research
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: GPU bakeoff — 3090 Ti vs V100 vs Strix Halo

Browse Source 
Cross-host Gemma 4 throughput comparison across three architectures.
Harness at scripts/gpu-bakeoff/; writeup at
docs/reference/gpu-bakeoff-2026-04-20.md.

Key findings:
- RTX 3090 Ti wins decode decisively (128 tok/s on gemma4:26b MoE Q4,
  ~4.7× faster than gemma4:31b dense on the same card).
- AMD Strix Halo iGPU lands at ~42% of 3090 Ti decode on ~25% of the
  memory bandwidth — good SIMD utilization, especially for MoE.
- V100 numbers are DEGRADED: CT 167 ai-visualizer SDXL consumes 31/32
  GB of its VRAM, forcing Gemma 4 models 95% onto CPU. Isolated V100
  run requires SDXL eviction — left as follow-up.
- MoE vs dense is the dominant latency factor across all GPUs: ~4 B
  active params of gemma4:26b beats 31.3 B active of gemma4:31b by
  the same ratio (~4.7×) on every card tested.

Methodology: 1 warmup + 3 measurement runs per (host × model ×
prompt-length), Ollama's canonical timing fields, temp=0 greedy,
num_predict=256. All three Ollama servers accessed via HTTP (Strix
via Tailscale).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mortdecai
2026-04-20 05:45:26 -04:00
parent df5542f7d6
commit b6190357ba
20 changed files with 1483 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gpu-bakeoff/runs/steel141/gemma4-26b-q8/long.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "steel141",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b-q8/short.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "steel141",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 12.5,
"prefill_tok_per_s": 25397.83,
"output_tokens": 256,
"eval_ms": 1988.3,
"decode_tok_per_s": 128.76,
"load_ms": 319.7,
"total_ms": 2461.3,
"harness_wall_s": 2.465,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 13.3,
"prefill_tok_per_s": 23848.87,
"output_tokens": 256,
"eval_ms": 1999.2,
"decode_tok_per_s": 128.05,
"load_ms": 343.2,
"total_ms": 2500.0,
"harness_wall_s": 2.503,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 14.2,
"prefill_tok_per_s": 22372.04,
"output_tokens": 256,
"eval_ms": 1998.9,
"decode_tok_per_s": 128.07,
"load_ms": 326.0,
"total_ms": 2479.5,
"harness_wall_s": 2.483,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 96.4,
"prefill_tok_per_s": 3298.64,
"output_tokens": 256,
"eval_ms": 2018.1,
"decode_tok_per_s": 126.85,
"load_ms": 328.7,
"total_ms": 2578.7,
"harness_wall_s": 2.582,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 22372.04,
"median": 23848.87,
"max": 25397.83,
"n": 3
},
"decode_tok_per_s": {
"min": 128.05,
"median": 128.07,
"max": 128.76,
"n": 3
},
"total_ms": {
"min": 2461.3,
"median": 2479.5,
"max": 2500.0,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 19.8,
"prefill_tok_per_s": 1366.3,
"output_tokens": 256,
"eval_ms": 2089.1,
"decode_tok_per_s": 122.54,
"load_ms": 361.8,
"total_ms": 2614.4,
"harness_wall_s": 2.617,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 12.9,
"prefill_tok_per_s": 2088.79,
"output_tokens": 256,
"eval_ms": 1996.8,
"decode_tok_per_s": 128.2,
"load_ms": 341.7,
"total_ms": 2489.2,
"harness_wall_s": 2.491,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 13.1,
"prefill_tok_per_s": 2062.75,
"output_tokens": 256,
"eval_ms": 1995.2,
"decode_tok_per_s": 128.31,
"load_ms": 330.7,
"total_ms": 2473.9,
"harness_wall_s": 2.476,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 47.7,
"prefill_tok_per_s": 566.39,
"output_tokens": 256,
"eval_ms": 2014.5,
"decode_tok_per_s": 127.08,
"load_ms": 4346.8,
"total_ms": 6739.3,
"harness_wall_s": 6.752,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 1366.3,
"median": 2062.75,
"max": 2088.79,
"n": 3
},
"decode_tok_per_s": {
"min": 122.54,
"median": 128.2,
"max": 128.31,
"n": 3
},
"total_ms": {
"min": 2473.9,
"median": 2489.2,
"max": 2614.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-31b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 38.2,
"prefill_tok_per_s": 8318.85,
"output_tokens": 256,
"eval_ms": 9390.5,
"decode_tok_per_s": 27.26,
"load_ms": 317.4,
"total_ms": 9886.3,
"harness_wall_s": 9.89,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 42.7,
"prefill_tok_per_s": 7454.7,
"output_tokens": 256,
"eval_ms": 9429.0,
"decode_tok_per_s": 27.15,
"load_ms": 316.0,
"total_ms": 9929.8,
"harness_wall_s": 9.933,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 41.2,
"prefill_tok_per_s": 7716.07,
"output_tokens": 256,
"eval_ms": 9477.4,
"decode_tok_per_s": 27.01,
"load_ms": 334.3,
"total_ms": 9996.2,
"harness_wall_s": 10.0,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 967.7,
"prefill_tok_per_s": 328.62,
"output_tokens": 256,
"eval_ms": 9339.6,
"decode_tok_per_s": 27.41,
"load_ms": 324.2,
"total_ms": 10774.3,
"harness_wall_s": 10.778,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 7454.7,
"median": 7716.07,
"max": 8318.85,
"n": 3
},
"decode_tok_per_s": {
"min": 27.01,
"median": 27.15,
"max": 27.26,
"n": 3
},
"total_ms": {
"min": 9886.3,
"median": 9929.8,
"max": 9996.2,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-31b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 44.1,
"prefill_tok_per_s": 611.75,
"output_tokens": 256,
"eval_ms": 9189.5,
"decode_tok_per_s": 27.86,
"load_ms": 373.7,
"total_ms": 9759.8,
"harness_wall_s": 9.762,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 40.4,
"prefill_tok_per_s": 668.59,
"output_tokens": 256,
"eval_ms": 9115.3,
"decode_tok_per_s": 28.08,
"load_ms": 340.5,
"total_ms": 9635.7,
"harness_wall_s": 9.638,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 40.9,
"prefill_tok_per_s": 660.95,
"output_tokens": 256,
"eval_ms": 9123.7,
"decode_tok_per_s": 28.06,
"load_ms": 325.8,
"total_ms": 9626.6,
"harness_wall_s": 9.629,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 139.6,
"prefill_tok_per_s": 193.44,
"output_tokens": 256,
"eval_ms": 9190.0,
"decode_tok_per_s": 27.86,
"load_ms": 13817.9,
"total_ms": 23488.4,
"harness_wall_s": 23.491,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 611.75,
"median": 660.95,
"max": 668.59,
"n": 3
},
"decode_tok_per_s": {
"min": 27.86,
"median": 28.06,
"max": 28.08,
"n": 3
},
"total_ms": {
"min": 9626.6,
"median": 9635.7,
"max": 9759.8,
"n": 3
}
}
}