Seth/gemma4-research
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: GPU bakeoff — 3090 Ti vs V100 vs Strix Halo

Browse Source 
Cross-host Gemma 4 throughput comparison across three architectures.
Harness at scripts/gpu-bakeoff/; writeup at
docs/reference/gpu-bakeoff-2026-04-20.md.

Key findings:
- RTX 3090 Ti wins decode decisively (128 tok/s on gemma4:26b MoE Q4,
  ~4.7× faster than gemma4:31b dense on the same card).
- AMD Strix Halo iGPU lands at ~42% of 3090 Ti decode on ~25% of the
  memory bandwidth — good SIMD utilization, especially for MoE.
- V100 numbers are DEGRADED: CT 167 ai-visualizer SDXL consumes 31/32
  GB of its VRAM, forcing Gemma 4 models 95% onto CPU. Isolated V100
  run requires SDXL eviction — left as follow-up.
- MoE vs dense is the dominant latency factor across all GPUs: ~4 B
  active params of gemma4:26b beats 31.3 B active of gemma4:31b by
  the same ratio (~4.7×) on every card tested.

Methodology: 1 warmup + 3 measurement runs per (host × model ×
prompt-length), Ollama's canonical timing fields, temp=0 greedy,
num_predict=256. All three Ollama servers accessed via HTTP (Strix
via Tailscale).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mortdecai
2026-04-20 05:45:26 -04:00
parent df5542f7d6
commit b6190357ba
20 changed files with 1483 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/gpu-bakeoff/harness.py
+242
View File
@@ -0,0 +1,242 @@
"""GPU bakeoff harness — Gemma 4 throughput across heterogeneous GPUs.
Measures prefill rate, decode rate, load time, and wall-clock across
three hosts:
- steel141 : RTX 3090 Ti (24 GB GDDR6X, compute 8.6, ~1008 GB/s)
- pve197 : Tesla V100-PCIE-32GB (32 GB HBM2, compute 7.0, ~900 GB/s)
- matt-strix: AMD Strix Halo iGPU (shared LPDDR5X, ~256 GB/s)
Per (host, model, prompt_length), runs 1 warmup + N measurement runs,
records Ollama's canonical timing fields, and writes one JSON trace to
`runs/<host>/<model>/<prompt_len>.json`.
All three Ollama servers are polled via HTTP; no SSH required. All
timings come from Ollama's own /api/generate response fields so wall-
clock jitter between the harness and the server is excluded.
Invocation:
python3 harness.py --host steel141 --model gemma4:26b --prompt short
python3 harness.py all # runs the full planned matrix
"""
from __future__ import annotations
import argparse
import json
import sys
import time
import urllib.request
from pathlib import Path
HOSTS = {
"steel141": {"url": "http://127.0.0.1:11434", "gpu": "RTX 3090 Ti", "vram_gb": 24},
"pve197": {"url": "http://192.168.0.179:11434", "gpu": "Tesla V100-PCIE-32GB", "vram_gb": 32},
"matt-strix": {"url": "http://100.117.155.64:11434", "gpu": "AMD Strix Halo iGPU", "vram_gb": None},
}
# Per-host model tag mapping. matt-strix uses gemma4:31b, the others
# use gemma4:31b-it-q4_K_M — identical weights, different tags.
MODEL_ALIASES = {
"gemma4:26b": {"steel141": "gemma4:26b", "pve197": "gemma4:26b", "matt-strix": "gemma4:26b"},
"gemma4:31b": {"steel141": "gemma4:31b-it-q4_K_M", "pve197": "gemma4:31b-it-q4_K_M", "matt-strix": "gemma4:31b"},
# V100-only edge case — only 32 GB host has headroom for the Q8 MoE.
"gemma4:26b-q8": {"pve197": "gemma4:26b-a4b-it-q8_0"},
}
PROMPTS = {
"short": "Write exactly one sentence summarizing how a transformer language model works.",
"long": (
"You are reviewing a short technical passage and must produce a concise summary.\n\n"
"Passage:\n"
"Modern large language models are trained using a combination of self-supervised "
"pretraining on vast text corpora and subsequent instruction-tuning on curated "
"prompt-response pairs. The pretraining stage exposes the model to diverse writing "
"styles, factual information, and reasoning patterns, but leaves it largely unaware "
"of how to follow user instructions. Instruction-tuning, typically via supervised "
"fine-tuning (SFT) followed by a preference-optimization stage such as Direct "
"Preference Optimization (DPO) or Reinforcement Learning from Human Feedback (RLHF), "
"aligns the model's behavior with human expectations. This two-stage recipe — "
"massive pretraining plus alignment — has become the dominant paradigm for open "
"and closed foundation models alike. Variants exist: some models add a midtraining "
"stage between the two for curriculum or skill rebalancing; others use constitutional "
"methods or reinforcement learning with verifiable rewards. For specialized domains "
"like code or math, domain-specific SFT datasets and reward models are commonly "
"layered on top of a general-purpose base. Throughout the process, the model's "
"parameters remain fixed in architecture but shift substantially in value, with "
"alignment stages typically touching a small fraction of the parameter space "
"compared to the changes induced by pretraining.\n\n"
"Task: Summarize the passage above in exactly three sentences, covering (1) what "
"pretraining does, (2) what instruction-tuning does, and (3) why both stages are "
"necessary in modern LLM recipes."
),
}
def _gen(url: str, model: str, prompt: str, num_predict: int, num_ctx: int, keep_alive: str) -> dict:
"""Single /api/generate call, stream=False, greedy decoding."""
payload = {
"model": model,
"prompt": prompt,
"stream": False,
"options": {
"num_ctx": num_ctx,
"num_predict": num_predict,
"temperature": 0.0,
"top_k": 1,
},
"keep_alive": keep_alive,
}
req = urllib.request.Request(
f"{url}/api/generate",
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
)
t0 = time.time()
with urllib.request.urlopen(req, timeout=600) as r:
d = json.loads(r.read())
d["_harness_wall_s"] = round(time.time() - t0, 3)
return d
def _metrics(d: dict) -> dict:
"""Extract canonical rates from Ollama's response.
Fields (all nanoseconds unless noted):
total_duration — end-to-end, including load
load_duration — time to load model into memory
prompt_eval_count — input tokens
prompt_eval_duration — time to prefill
eval_count — output tokens
eval_duration — time to decode
"""
pec = d.get("prompt_eval_count") or 0
ped = d.get("prompt_eval_duration") or 0
ec = d.get("eval_count") or 0
ed = d.get("eval_duration") or 0
total = d.get("total_duration") or 0
load = d.get("load_duration") or 0
prefill_rate = (pec / (ped / 1e9)) if ped else None
decode_rate = (ec / (ed / 1e9)) if ed else None
return {
"prompt_tokens": pec,
"prompt_eval_ms": round(ped / 1e6, 1) if ped else None,
"prefill_tok_per_s": round(prefill_rate, 2) if prefill_rate else None,
"output_tokens": ec,
"eval_ms": round(ed / 1e6, 1) if ed else None,
"decode_tok_per_s": round(decode_rate, 2) if decode_rate else None,
"load_ms": round(load / 1e6, 1) if load else None,
"total_ms": round(total / 1e6, 1) if total else None,
"harness_wall_s": d.get("_harness_wall_s"),
"done_reason": d.get("done_reason"),
}
def run_matrix(
host: str,
model_alias: str,
prompt_key: str,
num_predict: int = 256,
num_ctx: int = 4096,
runs: int = 3,
) -> dict:
host_cfg = HOSTS[host]
model_tag = MODEL_ALIASES[model_alias].get(host)
if not model_tag:
return {"host": host, "model_alias": model_alias, "skipped": "model not available on host"}
prompt = PROMPTS[prompt_key]
url = host_cfg["url"]
trace = {
"host": host,
"gpu": host_cfg["gpu"],
"vram_gb": host_cfg["vram_gb"],
"model_alias": model_alias,
"model_tag": model_tag,
"prompt_key": prompt_key,
"prompt_chars": len(prompt),
"num_predict": num_predict,
"num_ctx": num_ctx,
"runs": [],
"warmup": None,
}
# Warmup — discarded. First call absorbs model load time.
try:
w = _gen(url, model_tag, prompt, num_predict=num_predict, num_ctx=num_ctx, keep_alive="10m")
trace["warmup"] = _metrics(w)
except Exception as e:
trace["error"] = f"warmup failed: {e}"
return trace
# Measurement runs.
for i in range(runs):
try:
r = _gen(url, model_tag, prompt, num_predict=num_predict, num_ctx=num_ctx, keep_alive="10m")
trace["runs"].append(_metrics(r))
except Exception as e:
trace["runs"].append({"error": str(e)})
# Aggregate.
valid = [r for r in trace["runs"] if r.get("decode_tok_per_s") is not None]
if valid:
def _vals(k): return [r[k] for r in valid if r.get(k) is not None]
def _stats(xs):
if not xs: return None
s = sorted(xs)
return {"min": s[0], "median": s[len(s)//2], "max": s[-1], "n": len(s)}
trace["summary"] = {
"prefill_tok_per_s": _stats(_vals("prefill_tok_per_s")),
"decode_tok_per_s": _stats(_vals("decode_tok_per_s")),
"total_ms": _stats(_vals("total_ms")),
}
return trace
def _run_one(host: str, model: str, prompt: str, out_dir: Path, runs: int) -> None:
t = run_matrix(host, model, prompt, runs=runs)
safe_model = model.replace(":", "-").replace("/", "-")
path = out_dir / host / safe_model / f"{prompt}.json"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(t, indent=2))
s = t.get("summary") or {}
dec = s.get("decode_tok_per_s") or {}
pre = s.get("prefill_tok_per_s") or {}
skipped = t.get("skipped") or t.get("error")
if skipped:
print(f"[{host:10s}] {model:16s} {prompt:6s} — {skipped}")
else:
print(f"[{host:10s}] {model:16s} {prompt:6s} — "
f"prefill={pre.get('median','?'):>7} tok/s "
f"decode={dec.get('median','?'):>6} tok/s")
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--host", choices=list(HOSTS) + ["all"], default="all")
ap.add_argument("--model", choices=list(MODEL_ALIASES) + ["all"], default="all")
ap.add_argument("--prompt", choices=list(PROMPTS) + ["all"], default="all")
ap.add_argument("--runs", type=int, default=3)
ap.add_argument("--out-dir", type=Path, default=Path(__file__).parent / "runs")
args = ap.parse_args()
hosts = list(HOSTS) if args.host == "all" else [args.host]
models = list(MODEL_ALIASES) if args.model == "all" else [args.model]
prompts = list(PROMPTS) if args.prompt == "all" else [args.prompt]
for host in hosts:
for model in models:
for prompt in prompts:
_run_one(host, model, prompt, args.out_dir, args.runs)
return 0
if __name__ == "__main__":
sys.exit(main())
scripts/gpu-bakeoff/runs/matt-strix-rerun.log
+6
View File
@@ -0,0 +1,6 @@
[matt-strix] gemma4:26b short — prefill=1275.71 tok/s decode= 53.83 tok/s
[matt-strix] gemma4:26b long — prefill=14326.07 tok/s decode= 52.42 tok/s
[matt-strix] gemma4:31b short — prefill= 291.74 tok/s decode= 10.64 tok/s
[matt-strix] gemma4:31b long — prefill= 3277.8 tok/s decode= 10.42 tok/s
[matt-strix] gemma4:26b-q8 short — model not available on host
[matt-strix] gemma4:26b-q8 long — model not available on host
scripts/gpu-bakeoff/runs/matt-strix/gemma4-26b-q8/long.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "matt-strix",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/matt-strix/gemma4-26b-q8/short.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "matt-strix",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/matt-strix/gemma4-26b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "matt-strix",
"gpu": "AMD Strix Halo iGPU",
"vram_gb": null,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 319,
"prompt_eval_ms": 22.5,
"prefill_tok_per_s": 14201.36,
"output_tokens": 256,
"eval_ms": 4883.4,
"decode_tok_per_s": 52.42,
"load_ms": 151.1,
"total_ms": 5120.3,
"harness_wall_s": 5.186,
"done_reason": "length"
},
{
"prompt_tokens": 319,
"prompt_eval_ms": 22.1,
"prefill_tok_per_s": 14448.45,
"output_tokens": 256,
"eval_ms": 4881.1,
"decode_tok_per_s": 52.45,
"load_ms": 159.1,
"total_ms": 5124.5,
"harness_wall_s": 5.18,
"done_reason": "length"
},
{
"prompt_tokens": 319,
"prompt_eval_ms": 22.3,
"prefill_tok_per_s": 14326.07,
"output_tokens": 256,
"eval_ms": 4885.3,
"decode_tok_per_s": 52.4,
"load_ms": 155.4,
"total_ms": 5128.9,
"harness_wall_s": 5.192,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 319,
"prompt_eval_ms": 265.0,
"prefill_tok_per_s": 1203.86,
"output_tokens": 256,
"eval_ms": 4880.6,
"decode_tok_per_s": 52.45,
"load_ms": 159.8,
"total_ms": 5368.3,
"harness_wall_s": 5.429,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 14201.36,
"median": 14326.07,
"max": 14448.45,
"n": 3
},
"decode_tok_per_s": {
"min": 52.4,
"median": 52.42,
"max": 52.45,
"n": 3
},
"total_ms": {
"min": 5120.3,
"median": 5124.5,
"max": 5128.9,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/matt-strix/gemma4-26b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "matt-strix",
"gpu": "AMD Strix Halo iGPU",
"vram_gb": null,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 28,
"prompt_eval_ms": 21.9,
"prefill_tok_per_s": 1278.99,
"output_tokens": 256,
"eval_ms": 4754.7,
"decode_tok_per_s": 53.84,
"load_ms": 172.3,
"total_ms": 5008.5,
"harness_wall_s": 5.057,
"done_reason": "length"
},
{
"prompt_tokens": 28,
"prompt_eval_ms": 21.9,
"prefill_tok_per_s": 1275.71,
"output_tokens": 256,
"eval_ms": 4755.7,
"decode_tok_per_s": 53.83,
"load_ms": 151.6,
"total_ms": 4988.3,
"harness_wall_s": 5.043,
"done_reason": "length"
},
{
"prompt_tokens": 28,
"prompt_eval_ms": 22.0,
"prefill_tok_per_s": 1271.11,
"output_tokens": 256,
"eval_ms": 4757.6,
"decode_tok_per_s": 53.81,
"load_ms": 154.4,
"total_ms": 4993.2,
"harness_wall_s": 5.048,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 28,
"prompt_eval_ms": 93.1,
"prefill_tok_per_s": 300.9,
"output_tokens": 256,
"eval_ms": 4756.6,
"decode_tok_per_s": 53.82,
"load_ms": 2272.4,
"total_ms": 7250.0,
"harness_wall_s": 7.341,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 1271.11,
"median": 1275.71,
"max": 1278.99,
"n": 3
},
"decode_tok_per_s": {
"min": 53.81,
"median": 53.83,
"max": 53.84,
"n": 3
},
"total_ms": {
"min": 4988.3,
"median": 4993.2,
"max": 5008.5,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/matt-strix/gemma4-31b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "matt-strix",
"gpu": "AMD Strix Halo iGPU",
"vram_gb": null,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 319,
"prompt_eval_ms": 97.2,
"prefill_tok_per_s": 3282.33,
"output_tokens": 256,
"eval_ms": 24566.1,
"decode_tok_per_s": 10.42,
"load_ms": 157.2,
"total_ms": 24879.5,
"harness_wall_s": 24.945,
"done_reason": "length"
},
{
"prompt_tokens": 319,
"prompt_eval_ms": 97.3,
"prefill_tok_per_s": 3277.8,
"output_tokens": 256,
"eval_ms": 24560.7,
"decode_tok_per_s": 10.42,
"load_ms": 162.3,
"total_ms": 24880.5,
"harness_wall_s": 24.943,
"done_reason": "length"
},
{
"prompt_tokens": 319,
"prompt_eval_ms": 97.4,
"prefill_tok_per_s": 3274.93,
"output_tokens": 256,
"eval_ms": 24559.5,
"decode_tok_per_s": 10.42,
"load_ms": 157.1,
"total_ms": 24876.8,
"harness_wall_s": 24.94,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 319,
"prompt_eval_ms": 1052.0,
"prefill_tok_per_s": 303.23,
"output_tokens": 256,
"eval_ms": 24563.0,
"decode_tok_per_s": 10.42,
"load_ms": 167.6,
"total_ms": 25843.0,
"harness_wall_s": 25.896,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 3274.93,
"median": 3277.8,
"max": 3282.33,
"n": 3
},
"decode_tok_per_s": {
"min": 10.42,
"median": 10.42,
"max": 10.42,
"n": 3
},
"total_ms": {
"min": 24876.8,
"median": 24879.5,
"max": 24880.5,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/matt-strix/gemma4-31b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "matt-strix",
"gpu": "AMD Strix Halo iGPU",
"vram_gb": null,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 28,
"prompt_eval_ms": 96.4,
"prefill_tok_per_s": 290.33,
"output_tokens": 256,
"eval_ms": 24049.7,
"decode_tok_per_s": 10.64,
"load_ms": 169.4,
"total_ms": 24372.6,
"harness_wall_s": 24.428,
"done_reason": "length"
},
{
"prompt_tokens": 28,
"prompt_eval_ms": 96.0,
"prefill_tok_per_s": 291.74,
"output_tokens": 256,
"eval_ms": 24046.4,
"decode_tok_per_s": 10.65,
"load_ms": 165.7,
"total_ms": 24365.4,
"harness_wall_s": 24.429,
"done_reason": "length"
},
{
"prompt_tokens": 28,
"prompt_eval_ms": 95.6,
"prefill_tok_per_s": 292.74,
"output_tokens": 256,
"eval_ms": 24065.8,
"decode_tok_per_s": 10.64,
"load_ms": 164.3,
"total_ms": 24385.6,
"harness_wall_s": 24.432,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 28,
"prompt_eval_ms": 207.0,
"prefill_tok_per_s": 135.28,
"output_tokens": 256,
"eval_ms": 24181.8,
"decode_tok_per_s": 10.59,
"load_ms": 5509.8,
"total_ms": 30028.6,
"harness_wall_s": 30.082,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 290.33,
"median": 291.74,
"max": 292.74,
"n": 3
},
"decode_tok_per_s": {
"min": 10.64,
"median": 10.64,
"max": 10.65,
"n": 3
},
"total_ms": {
"min": 24365.4,
"median": 24372.6,
"max": 24385.6,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-26b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 118.0,
"prefill_tok_per_s": 2695.59,
"output_tokens": 256,
"eval_ms": 32720.5,
"decode_tok_per_s": 7.82,
"load_ms": 475.8,
"total_ms": 33548.1,
"harness_wall_s": 33.555,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 118.3,
"prefill_tok_per_s": 2689.01,
"output_tokens": 256,
"eval_ms": 31273.0,
"decode_tok_per_s": 8.19,
"load_ms": 492.5,
"total_ms": 32116.6,
"harness_wall_s": 32.123,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 117.3,
"prefill_tok_per_s": 2711.41,
"output_tokens": 256,
"eval_ms": 33434.9,
"decode_tok_per_s": 7.66,
"load_ms": 496.0,
"total_ms": 34298.7,
"harness_wall_s": 34.305,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 3562.7,
"prefill_tok_per_s": 89.26,
"output_tokens": 256,
"eval_ms": 32215.7,
"decode_tok_per_s": 7.95,
"load_ms": 491.7,
"total_ms": 36521.3,
"harness_wall_s": 36.529,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 2689.01,
"median": 2695.59,
"max": 2711.41,
"n": 3
},
"decode_tok_per_s": {
"min": 7.66,
"median": 7.82,
"max": 8.19,
"n": 3
},
"total_ms": {
"min": 32116.6,
"median": 33548.1,
"max": 34298.7,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-26b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 112.5,
"prefill_tok_per_s": 240.05,
"output_tokens": 256,
"eval_ms": 30919.5,
"decode_tok_per_s": 8.28,
"load_ms": 531.1,
"total_ms": 31828.4,
"harness_wall_s": 31.832,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 113.6,
"prefill_tok_per_s": 237.6,
"output_tokens": 256,
"eval_ms": 30399.9,
"decode_tok_per_s": 8.42,
"load_ms": 479.4,
"total_ms": 31242.1,
"harness_wall_s": 31.246,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 111.0,
"prefill_tok_per_s": 243.16,
"output_tokens": 256,
"eval_ms": 30712.9,
"decode_tok_per_s": 8.34,
"load_ms": 483.2,
"total_ms": 31552.8,
"harness_wall_s": 31.557,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 843.7,
"prefill_tok_per_s": 32.0,
"output_tokens": 256,
"eval_ms": 30499.4,
"decode_tok_per_s": 8.39,
"load_ms": 5877.7,
"total_ms": 37664.4,
"harness_wall_s": 37.668,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 237.6,
"median": 240.05,
"max": 243.16,
"n": 3
},
"decode_tok_per_s": {
"min": 8.28,
"median": 8.34,
"max": 8.42,
"n": 3
},
"total_ms": {
"min": 31242.1,
"median": 31552.8,
"max": 31828.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-31b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 728.7,
"prefill_tok_per_s": 436.37,
"output_tokens": 256,
"eval_ms": 163511.0,
"decode_tok_per_s": 1.57,
"load_ms": 495.0,
"total_ms": 164970.4,
"harness_wall_s": 164.977,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 682.8,
"prefill_tok_per_s": 465.71,
"output_tokens": 256,
"eval_ms": 168727.1,
"decode_tok_per_s": 1.52,
"load_ms": 545.3,
"total_ms": 170207.4,
"harness_wall_s": 170.214,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 950.0,
"prefill_tok_per_s": 334.75,
"output_tokens": 256,
"eval_ms": 163102.9,
"decode_tok_per_s": 1.57,
"load_ms": 507.9,
"total_ms": 164801.8,
"harness_wall_s": 164.809,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 3883.3,
"prefill_tok_per_s": 81.89,
"output_tokens": 256,
"eval_ms": 172199.4,
"decode_tok_per_s": 1.49,
"load_ms": 528.0,
"total_ms": 176864.8,
"harness_wall_s": 176.871,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 334.75,
"median": 436.37,
"max": 465.71,
"n": 3
},
"decode_tok_per_s": {
"min": 1.52,
"median": 1.57,
"max": 1.57,
"n": 3
},
"total_ms": {
"min": 164801.8,
"median": 164970.4,
"max": 170207.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/pve197/gemma4-31b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "pve197",
"gpu": "Tesla V100-PCIE-32GB",
"vram_gb": 32,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 665.6,
"prefill_tok_per_s": 40.56,
"output_tokens": 256,
"eval_ms": 164631.1,
"decode_tok_per_s": 1.55,
"load_ms": 512.6,
"total_ms": 166062.7,
"harness_wall_s": 166.067,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 660.3,
"prefill_tok_per_s": 40.89,
"output_tokens": 256,
"eval_ms": 159594.3,
"decode_tok_per_s": 1.6,
"load_ms": 523.6,
"total_ms": 161012.3,
"harness_wall_s": 161.016,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 887.8,
"prefill_tok_per_s": 30.41,
"output_tokens": 256,
"eval_ms": 167584.3,
"decode_tok_per_s": 1.53,
"load_ms": 486.8,
"total_ms": 169188.9,
"harness_wall_s": 169.194,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 6642.4,
"prefill_tok_per_s": 4.06,
"output_tokens": 256,
"eval_ms": 173530.1,
"decode_tok_per_s": 1.48,
"load_ms": 20142.1,
"total_ms": 200836.5,
"harness_wall_s": 200.841,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 30.41,
"median": 40.56,
"max": 40.89,
"n": 3
},
"decode_tok_per_s": {
"min": 1.53,
"median": 1.55,
"max": 1.6,
"n": 3
},
"total_ms": {
"min": 161012.3,
"median": 166062.7,
"max": 169188.9,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b-q8/long.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "steel141",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b-q8/short.json
+5
View File
@@ -0,0 +1,5 @@
{
"host": "steel141",
"model_alias": "gemma4:26b-q8",
"skipped": "model not available on host"
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 12.5,
"prefill_tok_per_s": 25397.83,
"output_tokens": 256,
"eval_ms": 1988.3,
"decode_tok_per_s": 128.76,
"load_ms": 319.7,
"total_ms": 2461.3,
"harness_wall_s": 2.465,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 13.3,
"prefill_tok_per_s": 23848.87,
"output_tokens": 256,
"eval_ms": 1999.2,
"decode_tok_per_s": 128.05,
"load_ms": 343.2,
"total_ms": 2500.0,
"harness_wall_s": 2.503,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 14.2,
"prefill_tok_per_s": 22372.04,
"output_tokens": 256,
"eval_ms": 1998.9,
"decode_tok_per_s": 128.07,
"load_ms": 326.0,
"total_ms": 2479.5,
"harness_wall_s": 2.483,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 96.4,
"prefill_tok_per_s": 3298.64,
"output_tokens": 256,
"eval_ms": 2018.1,
"decode_tok_per_s": 126.85,
"load_ms": 328.7,
"total_ms": 2578.7,
"harness_wall_s": 2.582,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 22372.04,
"median": 23848.87,
"max": 25397.83,
"n": 3
},
"decode_tok_per_s": {
"min": 128.05,
"median": 128.07,
"max": 128.76,
"n": 3
},
"total_ms": {
"min": 2461.3,
"median": 2479.5,
"max": 2500.0,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-26b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:26b",
"model_tag": "gemma4:26b",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 19.8,
"prefill_tok_per_s": 1366.3,
"output_tokens": 256,
"eval_ms": 2089.1,
"decode_tok_per_s": 122.54,
"load_ms": 361.8,
"total_ms": 2614.4,
"harness_wall_s": 2.617,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 12.9,
"prefill_tok_per_s": 2088.79,
"output_tokens": 256,
"eval_ms": 1996.8,
"decode_tok_per_s": 128.2,
"load_ms": 341.7,
"total_ms": 2489.2,
"harness_wall_s": 2.491,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 13.1,
"prefill_tok_per_s": 2062.75,
"output_tokens": 256,
"eval_ms": 1995.2,
"decode_tok_per_s": 128.31,
"load_ms": 330.7,
"total_ms": 2473.9,
"harness_wall_s": 2.476,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 47.7,
"prefill_tok_per_s": 566.39,
"output_tokens": 256,
"eval_ms": 2014.5,
"decode_tok_per_s": 127.08,
"load_ms": 4346.8,
"total_ms": 6739.3,
"harness_wall_s": 6.752,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 1366.3,
"median": 2062.75,
"max": 2088.79,
"n": 3
},
"decode_tok_per_s": {
"min": 122.54,
"median": 128.2,
"max": 128.31,
"n": 3
},
"total_ms": {
"min": 2473.9,
"median": 2489.2,
"max": 2614.4,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-31b/long.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "long",
"prompt_chars": 1614,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 318,
"prompt_eval_ms": 38.2,
"prefill_tok_per_s": 8318.85,
"output_tokens": 256,
"eval_ms": 9390.5,
"decode_tok_per_s": 27.26,
"load_ms": 317.4,
"total_ms": 9886.3,
"harness_wall_s": 9.89,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 42.7,
"prefill_tok_per_s": 7454.7,
"output_tokens": 256,
"eval_ms": 9429.0,
"decode_tok_per_s": 27.15,
"load_ms": 316.0,
"total_ms": 9929.8,
"harness_wall_s": 9.933,
"done_reason": "length"
},
{
"prompt_tokens": 318,
"prompt_eval_ms": 41.2,
"prefill_tok_per_s": 7716.07,
"output_tokens": 256,
"eval_ms": 9477.4,
"decode_tok_per_s": 27.01,
"load_ms": 334.3,
"total_ms": 9996.2,
"harness_wall_s": 10.0,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 318,
"prompt_eval_ms": 967.7,
"prefill_tok_per_s": 328.62,
"output_tokens": 256,
"eval_ms": 9339.6,
"decode_tok_per_s": 27.41,
"load_ms": 324.2,
"total_ms": 10774.3,
"harness_wall_s": 10.778,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 7454.7,
"median": 7716.07,
"max": 8318.85,
"n": 3
},
"decode_tok_per_s": {
"min": 27.01,
"median": 27.15,
"max": 27.26,
"n": 3
},
"total_ms": {
"min": 9886.3,
"median": 9929.8,
"max": 9996.2,
"n": 3
}
}
}
scripts/gpu-bakeoff/runs/steel141/gemma4-31b/short.json
+81
View File
@@ -0,0 +1,81 @@
{
"host": "steel141",
"gpu": "RTX 3090 Ti",
"vram_gb": 24,
"model_alias": "gemma4:31b",
"model_tag": "gemma4:31b-it-q4_K_M",
"prompt_key": "short",
"prompt_chars": 78,
"num_predict": 256,
"num_ctx": 4096,
"runs": [
{
"prompt_tokens": 27,
"prompt_eval_ms": 44.1,
"prefill_tok_per_s": 611.75,
"output_tokens": 256,
"eval_ms": 9189.5,
"decode_tok_per_s": 27.86,
"load_ms": 373.7,
"total_ms": 9759.8,
"harness_wall_s": 9.762,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 40.4,
"prefill_tok_per_s": 668.59,
"output_tokens": 256,
"eval_ms": 9115.3,
"decode_tok_per_s": 28.08,
"load_ms": 340.5,
"total_ms": 9635.7,
"harness_wall_s": 9.638,
"done_reason": "length"
},
{
"prompt_tokens": 27,
"prompt_eval_ms": 40.9,
"prefill_tok_per_s": 660.95,
"output_tokens": 256,
"eval_ms": 9123.7,
"decode_tok_per_s": 28.06,
"load_ms": 325.8,
"total_ms": 9626.6,
"harness_wall_s": 9.629,
"done_reason": "length"
}
],
"warmup": {
"prompt_tokens": 27,
"prompt_eval_ms": 139.6,
"prefill_tok_per_s": 193.44,
"output_tokens": 256,
"eval_ms": 9190.0,
"decode_tok_per_s": 27.86,
"load_ms": 13817.9,
"total_ms": 23488.4,
"harness_wall_s": 23.491,
"done_reason": "length"
},
"summary": {
"prefill_tok_per_s": {
"min": 611.75,
"median": 660.95,
"max": 668.59,
"n": 3
},
"decode_tok_per_s": {
"min": 27.86,
"median": 28.06,
"max": 28.08,
"n": 3
},
"total_ms": {
"min": 9626.6,
"median": 9635.7,
"max": 9759.8,
"n": 3
}
}
}