32 lines
496 B
TOML
32 lines
496 B
TOML
[infer]
|
|
socket = "/tmp/infer.sock"
|
|
model = "some-model"
|
|
|
|
[slots]
|
|
main = 2
|
|
background = 3
|
|
|
|
[expect]
|
|
template_sha256 = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
|
|
n_ctx = 4096
|
|
slots = 4
|
|
|
|
[sampling]
|
|
temperature = 0.2
|
|
top_p = 0.9
|
|
top_k = 40
|
|
|
|
[limits]
|
|
poll_ms = 50
|
|
busy_wait_ms = 200
|
|
load_wait_ms = 300
|
|
idle_grace_ms = 150
|
|
liveness_ms = 100
|
|
thinking_cap = 20
|
|
thinking_overrun = 10
|
|
max_tokens = 512
|
|
queue_len = 1
|
|
retry_attempts = 2
|
|
retry_backoff_ms = [10]
|
|
retry_window_ms = 1000
|