81 lines
6.2 KiB
Bash
Executable File
81 lines
6.2 KiB
Bash
Executable File
#!/bin/sh
|
||
# Smoke run (v1): two fake upstreams, one crossbar with a fresh SQLite file, real HTTP.
|
||
# Checks routing, leases (sticky + header), failover, recovery, streaming, queueing, pin, drain,
|
||
# usage and metrics. Prints "smoke: ok" or fails with the crossbar log.
|
||
set -eu
|
||
cd "$(dirname "$0")/.."
|
||
tmp=$(mktemp -d); trap 'kill $pids 2>/dev/null; rm -rf "$tmp"' EXIT INT TERM
|
||
pids=""
|
||
sed "s#^db .*#db = \"$tmp/crossbar.db\"#" example.toml > "$tmp/crossbar.toml"
|
||
bin/fakeupstream -listen 127.0.0.1:18081 -name alpha -models ornith-1.5-35b-a3b,small-9b -down-file "$tmp/alpha.down" -slow 600 >"$tmp/alpha.log" 2>&1 & pids="$pids $!"
|
||
bin/fakeupstream -listen 127.0.0.1:18082 -name beta -models ornith-1.5-35b-a3b -down-file "$tmp/beta.down" >"$tmp/beta.log" 2>&1 & pids="$pids $!"
|
||
bin/crossbar -config "$tmp/crossbar.toml" >"$tmp/crossbar.log" 2>&1 & pids="$pids $!"
|
||
sleep 1.5
|
||
fail() { echo "smoke: FAIL: $*" >&2; echo "--- crossbar.log"; cat "$tmp/crossbar.log"; exit 1; }
|
||
base=http://127.0.0.1:17777
|
||
conv() { printf '{"model":"ornith-1.5-35b-a3b","stream":false,"messages":[{"role":"system","content":"smoke"},{"role":"user","content":"conversation %s"}]}' "$1"; }
|
||
hdrs() { curl -s -o /dev/null -w '%{http_code} %header{X-Crossbar-Host} %header{X-Crossbar-Lease}' "$@"; }
|
||
|
||
# 1. a conversation gets a lease and keeps it; beta wins (2 slots × weight 2 vs 1 × 1)
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -d "$(conv A)" "$base/opencode-a/v1/chat/completions")
|
||
[ "$h" = "200 beta new" ] || fail "first turn should be '200 beta new', got '$h'"
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -d "$(conv A)" "$base/opencode-a/v1/chat/completions")
|
||
[ "$h" = "200 beta reused" ] || fail "second turn should reuse beta, got '$h'"
|
||
|
||
# 2. header route
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -H 'X-Crossbar-Route: hermes-x' -d "$(conv B)" "$base/v1/chat/completions")
|
||
case "$h" in "200 beta new") ;; *) fail "header route hermes-x should be '200 beta new', got '$h'";; esac
|
||
h=$(curl -s -o /dev/null -w '%{http_code}' "$base/nope/v1/models"); [ "$h" = "404" ] || fail "unknown route 404, got $h"
|
||
|
||
# 3. pin opencode-a to alpha: conversation A's next turn moves (an operator pin outranks the lease)
|
||
h=$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{"host":"alpha","pin":true}' "$base/_crossbar/routes/opencode-a")
|
||
[ "$h" = "200" ] || fail "pin returned $h"
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -d "$(conv A)" "$base/opencode-a/v1/chat/completions")
|
||
[ "$h" = "200 alpha new" ] || fail "after pin, conversation A should be '200 alpha new', got '$h'"
|
||
curl -s "$base/_crossbar/routes" | grep -q '"pinned":"alpha"' || fail "routes view does not show the pin: $(curl -s $base/_crossbar/routes)"
|
||
|
||
# 4. queue: alpha has parallel 1, queue_max 1, and answers in 600 ms → of three concurrent, one is 503
|
||
for i in 1 2 3; do (curl -s -o /dev/null -w '%{http_code}\n' -X POST -H 'Content-Type: application/json' -d "$(conv Q$i)" "$base/opencode-a/v1/chat/completions" >> "$tmp/codes") & sleep 0.1; done; wait $! 2>/dev/null || true
|
||
sleep 2.5
|
||
sort "$tmp/codes" | uniq -c | tr -s ' ' > "$tmp/counts"
|
||
grep -q '2 200' "$tmp/counts" && grep -q '1 503' "$tmp/counts" || fail "queue test wanted two 200 and one 503, got: $(cat "$tmp/counts")"
|
||
|
||
# 5. release the pin, drain alpha: new conversations go to beta, A stays on alpha
|
||
curl -s -o /dev/null -X POST -H 'Content-Type: application/json' -d '{"release":true}' "$base/_crossbar/routes/opencode-a"
|
||
h=$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' -d '{"drain":true}' "$base/_crossbar/hosts/alpha"); [ "$h" = "200" ] || fail "drain returned $h"
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -d "$(conv C)" "$base/opencode-a/v1/chat/completions")
|
||
[ "$h" = "200 beta new" ] || fail "with alpha draining a new conversation should go to beta, got '$h'"
|
||
curl -s "$base/_crossbar/hosts" | grep -q '"alpha":{[^}]*"draining":true' || fail "hosts view does not show alpha draining"
|
||
curl -s -o /dev/null -X POST -H 'Content-Type: application/json' -d '{"drain":false}' "$base/_crossbar/hosts/alpha"
|
||
|
||
# 6. failover + recovery
|
||
touch "$tmp/beta.down"; sleep 2.5
|
||
h=$(hdrs -X POST -H 'Content-Type: application/json' -d "$(conv C)" "$base/opencode-a/v1/chat/completions")
|
||
[ "$h" = "200 alpha new" ] || fail "with beta down conversation C should move to alpha, got '$h'"
|
||
curl -s "$base/_crossbar/hosts" | grep -q '"beta":{"healthy":false' || fail "hosts view does not show beta unhealthy"
|
||
rm "$tmp/beta.down"; sleep 3.5
|
||
curl -s "$base/_crossbar/hosts" | grep -q '"beta":{"healthy":true' || fail "beta did not recover after two good polls"
|
||
|
||
# 7. streaming still arrives incrementally, and the final usage chunk is untouched
|
||
start=$(date +%s%N)
|
||
curl -sN -X POST -H 'Content-Type: application/json' -d '{"model":"ornith-1.5-35b-a3b","stream":true,"messages":[{"role":"user","content":"stream me"}]}' \
|
||
"$base/opencode-a/v1/chat/completions" | while IFS= read -r line; do
|
||
[ -n "$line" ] || continue
|
||
now=$(date +%s%N); echo "$(( (now - start) / 1000000 )) $line"
|
||
done > "$tmp/stream.txt"
|
||
firstms=$(head -1 "$tmp/stream.txt" | cut -d' ' -f1); lastms=$(tail -1 "$tmp/stream.txt" | cut -d' ' -f1)
|
||
[ -n "$firstms" ] && [ "$((lastms - firstms))" -ge 600 ] || fail "stream arrived in one burst: $(cat "$tmp/stream.txt")"
|
||
grep -q '"usage"' "$tmp/stream.txt" && grep -q 'DONE' "$tmp/stream.txt" || fail "stream lost the usage chunk or DONE"
|
||
|
||
# 8. accounting and metrics
|
||
sleep 1
|
||
u=$(curl -s "$base/_crossbar/usage?by=host")
|
||
echo "$u" | grep -q '"key":"alpha"' && echo "$u" | grep -q '"key":"beta"' || fail "usage by host: $u"
|
||
echo "$u" | grep -q '"cached_tokens":[1-9]' || fail "usage has no cached tokens (SSE/JSON usage not captured): $u"
|
||
curl -s -H 'Accept: text/plain' "$base/_crossbar/usage?by=route" | grep -qi 'cache' || fail "text usage table missing"
|
||
m=$(curl -s "$base/_crossbar/metrics")
|
||
echo "$m" | grep -q 'crossbar_requests_total{route="opencode-a",host="alpha",status="503"} 1' || fail "metrics missing the 503: $m"
|
||
echo "$m" | grep -q 'crossbar_host_healthy{host="beta"} 1' || fail "metrics missing host health"
|
||
grep -q 'route=opencode-a host=' "$tmp/crossbar.log" || fail "no request log line"
|
||
echo "smoke: ok (stream spread $((lastms - firstms)) ms)"
|