Bonsai-27B — AWQ 4-bit

Serve (sglang)

bash

pip install sglang
python -m sglang.launch_server \    --model prism-ml/Bonsai-27B-AWQ-4bit \    --port 8000 \    --dtype bfloat16

pip install sglang
python -m sglang.launch_server \    --model prism-ml/Bonsai-27B-AWQ-4bit \    --port 8000 \    --dtype bfloat16

Use

bash

curl http://localhost:8000/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{"model":"prism-ml/Bonsai-27B-AWQ-4bit","messages":[{"role":"user","content":"Who are you?"}],"max_tokens":100}'

curl http://localhost:8000/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{"model":"prism-ml/Bonsai-27B-AWQ-4bit","messages":[{"role":"user","content":"Who are you?"}],"max_tokens":100}'

Multi-GPU (8× H100)

bash

# TP=2 DP=4 — 4 replicas, each split across 2 GPUs (throughput sweet spot)python -m sglang.launch_server \    --model prism-ml/Bonsai-27B-AWQ-4bit \    --tp-size 2 --dp-size 4 \    --load-balance-method total_tokens \    --port 8000 --dtype bfloat16

# TP=2 DP=4 — 4 replicas, each split across 2 GPUs (throughput sweet spot)python -m sglang.launch_server \    --model prism-ml/Bonsai-27B-AWQ-4bit \    --tp-size 2 --dp-size 4 \    --load-balance-method total_tokens \    --port 8000 --dtype bfloat16