diff --git a/docker-compose.yml b/docker-compose.yml index 259f695..d1c6b6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: - "127.0.0.1:11434:11434" # Only accessible to gateway, not exposed volumes: - ollama-data:/root/.ollama - - ./models:/models + - ./models:/models:ro devices: - /dev/kfd:/dev/kfd - /dev/dri:/dev/dri diff --git a/setup.sh b/setup.sh index 4eaae57..043db6f 100755 --- a/setup.sh +++ b/setup.sh @@ -1,10 +1,15 @@ #!/bin/bash -# Quick setup for Mortdecai Gateway -# Run this after cloning the repo +# Mortdecai Gateway — fully automated setup +# Just run: ./setup.sh +# Everything downloads and configures automatically. set -e +MODEL_URL="${MODEL_URL:-https://mortdec.ai/dl/m4gguf/mortdecai-v4.gguf}" +MODEL_NAME="mortdecai-v4" + echo "=== Mortdecai Gateway Setup ===" +echo "" # Generate API key if not set if [ ! -f .env ]; then @@ -20,30 +25,52 @@ EOF echo "Saved to .env" else echo ".env already exists" + KEY=$(grep API_KEY .env | cut -d= -f2) fi # Start containers +echo "" echo "Starting containers..." docker compose up -d # Wait for Ollama to be ready -echo "Waiting for Ollama..." -for i in $(seq 1 30); do +echo "Waiting for Ollama to start..." +for i in $(seq 1 60); do if curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then echo "Ollama is ready" break fi + if [ $i -eq 60 ]; then + echo "ERROR: Ollama failed to start after 2 minutes" + echo "Check: docker logs mortdecai-ollama" + exit 1 + fi sleep 2 done -# Load the model if GGUF exists -if ls models/*.gguf 1>/dev/null 2>&1; then - GGUF=$(ls models/*.gguf | head -1) - MODEL_NAME=$(basename "$GGUF" .gguf | tr '[:upper:]' '[:lower:]') - echo "Loading model from $GGUF..." +# Check if model already loaded +LOADED=$(curl -s http://localhost:11434/api/tags 2>/dev/null | python3 -c "import sys,json; print('yes' if any('$MODEL_NAME' in m['name'] for m in json.load(sys.stdin).get('models',[])) else 'no')" 2>/dev/null || echo "no") - cat > /tmp/Modelfile << MEOF -FROM /models/$(basename $GGUF) +if [ "$LOADED" = "yes" ]; then + echo "Model $MODEL_NAME already loaded" +else + # Download GGUF + mkdir -p models + GGUF_PATH="models/${MODEL_NAME}.gguf" + + if [ ! -f "$GGUF_PATH" ]; then + echo "" + echo "Downloading model (~5.3 GB)..." + echo "Source: $MODEL_URL" + curl -L -o "$GGUF_PATH" "$MODEL_URL" --progress-bar + echo "Download complete" + else + echo "GGUF already downloaded" + fi + + # Create Modelfile + cat > models/Modelfile << 'MEOF' +FROM /models/mortdecai-v4.gguf TEMPLATE """{{- if .Messages }} {{- if or .System .Tools }}<|im_start|>system {{- if .System }} @@ -51,11 +78,11 @@ TEMPLATE """{{- if .Messages }} {{- end }} <|im_end|> {{ end }} -{{- range \$m := .Messages }} -{{- if eq \$m.Role "user" }}<|im_start|>user -{{ \$m.Content }}<|im_end|> -{{- else if eq \$m.Role "assistant" }}<|im_start|>assistant -{{ \$m.Content }}<|im_end|> +{{- range $m := .Messages }} +{{- if eq $m.Role "user" }}<|im_start|>user +{{ $m.Content }}<|im_end|> +{{- else if eq $m.Role "assistant" }}<|im_start|>assistant +{{ $m.Content }}<|im_end|> {{- end }} {{- end }}<|im_start|>assistant {{ end }}""" @@ -64,22 +91,40 @@ PARAMETER stop <|im_start|> PARAMETER temperature 0.7 MEOF - docker exec mortdecai-ollama ollama create mortdecai-v4 -f /tmp/Modelfile - echo "Model loaded as mortdecai-v4" + echo "Loading model into Ollama..." + docker exec mortdecai-ollama ollama create "$MODEL_NAME" -f /models/Modelfile + echo "Model loaded as $MODEL_NAME" +fi + +# Quick test +echo "" +echo "Running test inference..." +RESULT=$(curl -s http://localhost:8434/api/chat \ + -H "Authorization: Bearer $KEY" \ + -H "Content-Type: application/json" \ + -d "{\"model\": \"$MODEL_NAME\", \"messages\": [{\"role\": \"user\", \"content\": \"say hello\"}], \"stream\": false}" 2>/dev/null) + +if echo "$RESULT" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['message']['content'][:80])" 2>/dev/null; then + echo "Test passed!" else - echo "No GGUF found in models/ — place your GGUF file there and run:" - echo " docker exec mortdecai-ollama ollama create mortdecai-v4 -f Modelfile" + echo "Test inference returned unexpected result (model may still be loading)" + echo "Try again in a minute: curl -s http://localhost:8434/health" fi echo "" -echo "=== Setup Complete ===" -echo "Dashboard: http://localhost:8434/dashboard" -echo "API Key: $(grep API_KEY .env | cut -d= -f2)" +echo "=========================================" +echo " Mortdecai Gateway is ready!" +echo "=========================================" echo "" -echo "Test: curl -s http://localhost:8434/health" +echo " Dashboard: http://localhost:8434/dashboard" +echo " Health: http://localhost:8434/health" +echo " API Key: $KEY" echo "" -echo "To use from remote:" -echo " curl -X POST http://YOUR_IP:8434/api/chat \\" -echo " -H 'Authorization: Bearer YOUR_API_KEY' \\" -echo " -H 'Content-Type: application/json' \\" -echo " -d '{\"model\": \"mortdecai-v4\", \"messages\": [{\"role\": \"user\", \"content\": \"test\"}]}'" +echo " Send this to Seth:" +echo " - Your public IP" +echo " - Port: 8434" +echo " - API Key: $KEY" +echo "" +echo " To stop: docker compose down" +echo " To start: docker compose up -d" +echo "========================================="