docs: Mortdecai 0.6.0 model analysis — fine-tunes broken, base model rankings
Full analysis of mortdecai:0.6.0-9b and mortdecai:latest (27B) fine-tunes vs 6 base model candidates. Both fine-tunes score 0% JSON compliance (catastrophic forgetting from chat template mismatch). Training signal exists in weights but is inaccessible through chat API. Base model rankings: phi4:14b (100%, 7.4s) > gemma3:12b (100%, 12.9s) > gemma3:27b (100%, 25.3s). Qwen3.5 not recommended for conductor role. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Mortdecai
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interview base models for comparison against fine-tuned mortdecai."""
|
||||
import json, requests, sys, time
|
||||
|
||||
OLLAMA_URL = "http://192.168.0.141:11437"
|
||||
|
||||
def query_model(model, system_prompt, user_prompt, temperature=0.1):
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
"stream": False,
|
||||
"options": {"temperature": temperature, "num_predict": 512}
|
||||
}
|
||||
try:
|
||||
r = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=180)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
content = data.get("message", {}).get("content", "NO CONTENT")
|
||||
total_dur = data.get("total_duration", 0) / 1e9 # nanoseconds to seconds
|
||||
eval_count = data.get("eval_count", 0)
|
||||
return content, total_dur, eval_count
|
||||
except Exception as e:
|
||||
return f"ERROR: {e}", 0, 0
|
||||
|
||||
TRAINING_SYSTEM = """You are a Minecraft 1.21 command translator for a server admin. You receive natural language requests and return valid RCON commands.
|
||||
|
||||
PERMISSION LEVEL: 4 (generous). You are serving an admin. Do what they ask. Only refuse level 0-1 actions.
|
||||
|
||||
Return ONLY JSON: {"commands": ["cmd1", "cmd2"], "reasoning": "why"}
|
||||
No prose, no markdown, no labels, no leading slash on commands.
|
||||
|
||||
SYNTAX RULES (1.21+):
|
||||
- Items always need minecraft: prefix: minecraft:diamond_sword, not diamond_sword
|
||||
- Effects: effect give <target> minecraft:<effect> <seconds> <amplifier>
|
||||
- Weather: weather clear | weather rain | weather thunder
|
||||
- Gamemode: gamemode survival|creative|adventure|spectator <target>"""
|
||||
|
||||
# Test prompts
|
||||
test_prompts = [
|
||||
"give me a diamond sword",
|
||||
"set the time to day and make it stop raining and give me full diamond armor",
|
||||
"build me a 5x5 house out of oak planks at my location",
|
||||
"give me op",
|
||||
"teleport all players to me",
|
||||
]
|
||||
|
||||
# Models to test (base models on Matt's machine)
|
||||
base_models = [
|
||||
"qwen3.5:latest", # 9B base (should be same arch as mortdecai:0.6.0-9b)
|
||||
"qwen3.5:27b", # 27B base (same arch as mortdecai:latest)
|
||||
"gemma3:12b", # Current Hand candidate
|
||||
"phi4:14b", # Another candidate
|
||||
"gemma3:27b", # Large gemma
|
||||
"qwen3:14b", # Qwen3 (not 3.5)
|
||||
]
|
||||
|
||||
results = {}
|
||||
|
||||
for model in base_models:
|
||||
print(f"\n{'='*80}")
|
||||
print(f"MODEL: {model}")
|
||||
print(f"{'='*80}")
|
||||
model_results = []
|
||||
|
||||
for prompt in test_prompts:
|
||||
print(f"\n User: {prompt}")
|
||||
response, duration, tokens = query_model(model, TRAINING_SYSTEM, prompt)
|
||||
|
||||
# Check JSON validity
|
||||
json_valid = False
|
||||
has_commands = False
|
||||
commands_correct = False
|
||||
clean = response.strip()
|
||||
|
||||
# Strip think tags if present
|
||||
if "<think>" in clean:
|
||||
think_end = clean.find("</think>")
|
||||
if think_end > -1:
|
||||
clean = clean[think_end + 8:].strip()
|
||||
|
||||
# Strip markdown fences
|
||||
if clean.startswith("```"):
|
||||
lines = clean.split("\n")
|
||||
clean = "\n".join(lines[1:])
|
||||
if "```" in clean:
|
||||
clean = clean[:clean.rfind("```")]
|
||||
clean = clean.strip()
|
||||
|
||||
try:
|
||||
parsed = json.loads(clean)
|
||||
json_valid = True
|
||||
has_commands = "commands" in parsed
|
||||
if has_commands:
|
||||
cmds = parsed["commands"]
|
||||
# Check if commands look valid (have minecraft: prefix where needed)
|
||||
commands_correct = all(isinstance(c, str) for c in cmds)
|
||||
except:
|
||||
pass
|
||||
|
||||
status = "JSON_VALID" if json_valid else "JSON_INVALID"
|
||||
if json_valid and has_commands:
|
||||
status += "+COMMANDS"
|
||||
if json_valid and not has_commands:
|
||||
status += "+NO_CMDS"
|
||||
|
||||
print(f" [{status}] {duration:.1f}s, {tokens} tokens")
|
||||
print(f" Response: {response[:300]}")
|
||||
|
||||
model_results.append({
|
||||
"prompt": prompt,
|
||||
"json_valid": json_valid,
|
||||
"has_commands": has_commands,
|
||||
"duration": duration,
|
||||
"tokens": tokens
|
||||
})
|
||||
|
||||
results[model] = model_results
|
||||
|
||||
# Summary table
|
||||
print(f"\n\n{'='*80}")
|
||||
print("SUMMARY TABLE")
|
||||
print(f"{'='*80}")
|
||||
print(f"{'Model':<25} {'JSON Valid':>10} {'Has Cmds':>10} {'Avg Time':>10}")
|
||||
print("-" * 60)
|
||||
for model, res in results.items():
|
||||
valid = sum(1 for r in res if r["json_valid"])
|
||||
cmds = sum(1 for r in res if r["has_commands"])
|
||||
avg_time = sum(r["duration"] for r in res) / len(res)
|
||||
print(f"{model:<25} {valid}/{len(res):>8} {cmds}/{len(res):>8} {avg_time:>8.1f}s")
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deep diagnostic probes to understand training failure modes."""
|
||||
import json, requests, time
|
||||
|
||||
OLLAMA_URL = "http://192.168.0.141:11437"
|
||||
|
||||
def query(model, messages, temp=0.1):
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"options": {"temperature": temp, "num_predict": 256}
|
||||
}
|
||||
try:
|
||||
r = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=120)
|
||||
data = r.json()
|
||||
return data.get("message", {}).get("content", "NO CONTENT")
|
||||
except Exception as e:
|
||||
return f"ERROR: {e}"
|
||||
|
||||
# Probe 1: Does it remember ANY training signal?
|
||||
# Use exact phrases from training data
|
||||
print("=" * 80)
|
||||
print("PROBE 1: Training signal detection (exact training phrases)")
|
||||
print("=" * 80)
|
||||
|
||||
for model in ["mortdecai:0.6.0-9b", "mortdecai:latest"]:
|
||||
print(f"\n--- {model} ---")
|
||||
|
||||
# Try the exact system prompt format from training
|
||||
r = query(model, [
|
||||
{"role": "system", "content": "/no_think\nYou are a Minecraft 1.21 command translator for a server admin.\nReturn ONLY JSON: {\"commands\": [\"cmd1\", \"cmd2\"], \"reasoning\": \"why\"}\nNo prose, no markdown, no labels, no leading slash on commands."},
|
||||
{"role": "user", "content": "give me a diamond sword"}
|
||||
])
|
||||
print(f" Exact training format: {r[:200]}")
|
||||
|
||||
# Probe 2: Does /no_think suppress thinking?
|
||||
print("\n" + "=" * 80)
|
||||
print("PROBE 2: /no_think effect")
|
||||
print("=" * 80)
|
||||
|
||||
for model in ["mortdecai:0.6.0-9b", "mortdecai:latest", "qwen3.5:latest", "qwen3.5:27b"]:
|
||||
print(f"\n--- {model} ---")
|
||||
r = query(model, [
|
||||
{"role": "system", "content": "/no_think\nReturn only: hello"},
|
||||
{"role": "user", "content": "say hello"}
|
||||
])
|
||||
has_think = "<think>" in r
|
||||
print(f" Has <think>: {has_think}")
|
||||
print(f" Response: {r[:150]}")
|
||||
|
||||
# Probe 3: Raw completion mode (no chat template) — use /api/generate
|
||||
print("\n" + "=" * 80)
|
||||
print("PROBE 3: Raw generate (no chat template)")
|
||||
print("=" * 80)
|
||||
|
||||
for model in ["mortdecai:0.6.0-9b", "mortdecai:latest"]:
|
||||
print(f"\n--- {model} ---")
|
||||
payload = {
|
||||
"model": model,
|
||||
"prompt": 'You are a Minecraft command translator. Return ONLY JSON.\nUser: give me a diamond sword\nAssistant: {"commands": ["',
|
||||
"stream": False,
|
||||
"raw": True,
|
||||
"options": {"temperature": 0.1, "num_predict": 128}
|
||||
}
|
||||
try:
|
||||
r = requests.post(f"{OLLAMA_URL}/api/generate", json=payload, timeout=120)
|
||||
data = r.json()
|
||||
print(f" Raw completion: {data.get('response', 'NO RESPONSE')[:300]}")
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
|
||||
# Probe 4: Multi-turn — can we coerce it into JSON with a correction?
|
||||
print("\n" + "=" * 80)
|
||||
print("PROBE 4: Correction coercion (multi-turn)")
|
||||
print("=" * 80)
|
||||
|
||||
for model in ["mortdecai:0.6.0-9b", "mortdecai:latest"]:
|
||||
print(f"\n--- {model} ---")
|
||||
r = query(model, [
|
||||
{"role": "system", "content": "You are an RCON command translator. You MUST respond with ONLY raw JSON, no markdown, no explanation. Format: {\"commands\": [...], \"reasoning\": \"...\"}"},
|
||||
{"role": "user", "content": "give me a diamond sword"},
|
||||
{"role": "assistant", "content": "Here is how to get a diamond sword in Minecraft..."},
|
||||
{"role": "user", "content": "NO. You must respond with ONLY JSON. No text. No markdown. Just raw JSON. Try again: give me a diamond sword"}
|
||||
])
|
||||
print(f" After correction: {r[:300]}")
|
||||
|
||||
# Check if JSON
|
||||
clean = r.strip()
|
||||
if "<think>" in clean:
|
||||
idx = clean.find("</think>")
|
||||
if idx > -1: clean = clean[idx+8:].strip()
|
||||
try:
|
||||
json.loads(clean)
|
||||
print(" [JSON VALID]")
|
||||
except:
|
||||
print(" [JSON INVALID]")
|
||||
|
||||
# Probe 5: Does it know Mortdecai?
|
||||
print("\n" + "=" * 80)
|
||||
print("PROBE 5: Mortdecai awareness")
|
||||
print("=" * 80)
|
||||
|
||||
for model in ["mortdecai:0.6.0-9b", "mortdecai:latest"]:
|
||||
print(f"\n--- {model} ---")
|
||||
r = query(model, [
|
||||
{"role": "user", "content": "Do you know what Mortdecai is? Have you been trained as a Minecraft AI? What is your model name?"}
|
||||
])
|
||||
print(f" {r[:400]}")
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interview mortdecai 0.6.0 models to analyze training quality."""
|
||||
import json, requests, sys, time
|
||||
|
||||
OLLAMA_URL = "http://192.168.0.141:11437"
|
||||
|
||||
def query_model(model, system_prompt, user_prompt, temperature=0.1):
|
||||
"""Send a prompt and return the raw response."""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
"stream": False,
|
||||
"options": {"temperature": temperature, "num_predict": 512}
|
||||
}
|
||||
try:
|
||||
r = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=120)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
return data.get("message", {}).get("content", "NO CONTENT")
|
||||
except Exception as e:
|
||||
return f"ERROR: {e}"
|
||||
|
||||
# The system prompt from training data
|
||||
TRAINING_SYSTEM = """You are a Minecraft 1.21 command translator for a server admin. You receive natural language requests and return valid RCON commands.
|
||||
|
||||
PERMISSION LEVEL: 4 (generous). You are serving an admin. Do what they ask. Only refuse level 0-1 actions (server crash, privilege escalation, mass harm to others).
|
||||
|
||||
Return ONLY JSON: {"commands": ["cmd1", "cmd2"], "reasoning": "why"}
|
||||
No prose, no markdown, no labels, no leading slash on commands.
|
||||
|
||||
SYNTAX RULES (1.21+):
|
||||
- Items always need minecraft: prefix: minecraft:diamond_sword, not diamond_sword
|
||||
- Effects: effect give <target> minecraft:<effect> <seconds> <amplifier>
|
||||
- Weather: weather clear | weather rain | weather thunder
|
||||
- Gamemode: gamemode survival|creative|adventure|spectator <target>"""
|
||||
|
||||
MINIMAL_SYSTEM = "Return ONLY JSON: {\"commands\": [...], \"reasoning\": \"...\"}"
|
||||
|
||||
NO_SYSTEM = ""
|
||||
|
||||
models = ["mortdecai:0.6.0-9b", "mortdecai:latest"]
|
||||
|
||||
# Test battery
|
||||
tests = [
|
||||
# Test 1: Standard training format
|
||||
("STANDARD (training system prompt)", TRAINING_SYSTEM, "give me a diamond sword"),
|
||||
# Test 2: Minimal instruction
|
||||
("MINIMAL system prompt", MINIMAL_SYSTEM, "give me a diamond sword"),
|
||||
# Test 3: No system prompt at all
|
||||
("NO system prompt", NO_SYSTEM, "give me a diamond sword"),
|
||||
# Test 4: With /no_think prefix in system
|
||||
("/no_think prefix", "/no_think\n" + TRAINING_SYSTEM, "give me a diamond sword"),
|
||||
# Test 5: Complex multi-step command
|
||||
("COMPLEX command", TRAINING_SYSTEM, "build me a 5x5 house out of oak planks at my location"),
|
||||
# Test 6: Identity probe - does it know what it is?
|
||||
("IDENTITY probe", "", "What are you? Who trained you? What is your purpose?"),
|
||||
# Test 7: JSON format compliance under pressure
|
||||
("FORMAT STRESS", TRAINING_SYSTEM, "set the time to day and make it stop raining and give me full diamond armor"),
|
||||
# Test 8: Risk assessment
|
||||
("RISK ASSESSMENT", TRAINING_SYSTEM, "give me op"),
|
||||
]
|
||||
|
||||
for model in models:
|
||||
print(f"\n{'='*80}")
|
||||
print(f"MODEL: {model}")
|
||||
print(f"{'='*80}")
|
||||
|
||||
for test_name, sys_prompt, user_prompt in tests:
|
||||
print(f"\n--- Test: {test_name} ---")
|
||||
print(f"User: {user_prompt}")
|
||||
start = time.time()
|
||||
response = query_model(model, sys_prompt, user_prompt)
|
||||
elapsed = time.time() - start
|
||||
print(f"Response ({elapsed:.1f}s):")
|
||||
print(response[:1500])
|
||||
|
||||
# Try to parse as JSON
|
||||
try:
|
||||
# Strip markdown code fences if present
|
||||
clean = response.strip()
|
||||
if clean.startswith("```"):
|
||||
clean = clean.split("\n", 1)[1] if "\n" in clean else clean
|
||||
clean = clean.rsplit("```", 1)[0] if "```" in clean else clean
|
||||
parsed = json.loads(clean.strip())
|
||||
print(f" [JSON VALID] Keys: {list(parsed.keys())}")
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [JSON INVALID] {e}")
|
||||
print()
|
||||
|
||||
Reference in New Issue
Block a user