Seth
c5865feb35
- API key auth on all inference endpoints - Power/cost tracking: GPU TDP × inference time × electricity rate - Spending cap enforcement - Web dashboard with live stats - Docker compose for AMD ROCm (Strix Halo) or NVIDIA - Auto-setup script with GGUF loading - Tested against local Ollama Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
86 lines
2.2 KiB
Bash
Executable File
86 lines
2.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# Quick setup for Mortdecai Gateway
|
|
# Run this after cloning the repo
|
|
|
|
set -e
|
|
|
|
echo "=== Mortdecai Gateway Setup ==="
|
|
|
|
# Generate API key if not set
|
|
if [ ! -f .env ]; then
|
|
KEY="mk_$(openssl rand -hex 16)"
|
|
cat > .env << EOF
|
|
API_KEY=$KEY
|
|
GPU_TDP_WATTS=54
|
|
SYSTEM_OVERHEAD_WATTS=30
|
|
ELECTRICITY_RATE=0.15
|
|
SPENDING_CAP=10.00
|
|
EOF
|
|
echo "Generated API key: $KEY"
|
|
echo "Saved to .env"
|
|
else
|
|
echo ".env already exists"
|
|
fi
|
|
|
|
# Start containers
|
|
echo "Starting containers..."
|
|
docker compose up -d
|
|
|
|
# Wait for Ollama to be ready
|
|
echo "Waiting for Ollama..."
|
|
for i in $(seq 1 30); do
|
|
if curl -s http://localhost:11434/api/tags > /dev/null 2>&1; then
|
|
echo "Ollama is ready"
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Load the model if GGUF exists
|
|
if ls models/*.gguf 1>/dev/null 2>&1; then
|
|
GGUF=$(ls models/*.gguf | head -1)
|
|
MODEL_NAME=$(basename "$GGUF" .gguf | tr '[:upper:]' '[:lower:]')
|
|
echo "Loading model from $GGUF..."
|
|
|
|
cat > /tmp/Modelfile << MEOF
|
|
FROM /models/$(basename $GGUF)
|
|
TEMPLATE """{{- if .Messages }}
|
|
{{- if or .System .Tools }}<|im_start|>system
|
|
{{- if .System }}
|
|
{{ .System }}
|
|
{{- end }}
|
|
<|im_end|>
|
|
{{ end }}
|
|
{{- range \$m := .Messages }}
|
|
{{- if eq \$m.Role "user" }}<|im_start|>user
|
|
{{ \$m.Content }}<|im_end|>
|
|
{{- else if eq \$m.Role "assistant" }}<|im_start|>assistant
|
|
{{ \$m.Content }}<|im_end|>
|
|
{{- end }}
|
|
{{- end }}<|im_start|>assistant
|
|
{{ end }}"""
|
|
PARAMETER stop <|im_end|>
|
|
PARAMETER stop <|im_start|>
|
|
PARAMETER temperature 0.7
|
|
MEOF
|
|
|
|
docker exec mortdecai-ollama ollama create mortdecai-v4 -f /tmp/Modelfile
|
|
echo "Model loaded as mortdecai-v4"
|
|
else
|
|
echo "No GGUF found in models/ — place your GGUF file there and run:"
|
|
echo " docker exec mortdecai-ollama ollama create mortdecai-v4 -f Modelfile"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== Setup Complete ==="
|
|
echo "Dashboard: http://localhost:8434/dashboard"
|
|
echo "API Key: $(grep API_KEY .env | cut -d= -f2)"
|
|
echo ""
|
|
echo "Test: curl -s http://localhost:8434/health"
|
|
echo ""
|
|
echo "To use from remote:"
|
|
echo " curl -X POST http://YOUR_IP:8434/api/chat \\"
|
|
echo " -H 'Authorization: Bearer YOUR_API_KEY' \\"
|
|
echo " -H 'Content-Type: application/json' \\"
|
|
echo " -d '{\"model\": \"mortdecai-v4\", \"messages\": [{\"role\": \"user\", \"content\": \"test\"}]}'"
|