orchard

 1# llama-swap: one model, started on demand and unloaded when idle.
 2#
 3# The ttl is the point of this file. Without it the weights sit on the card
 4# forever and this machine is also a workstation. Three minutes is short
 5# because a cold start measured 2.7 seconds, so the cost of being wrong is
 6# small and the card comes back quickly when nobody is chatting.
 7
 8healthCheckTimeout: 600
 9logLevel: info
10startPort: 10001
11
12models:
13  # Every caller asks for "local", so this name is the contract across the
14  # whole estate and swapping the weights underneath is one line here.
15  "local":
16    name: "Ornith 1.5 9B"
17    ttl: 180
18    # Measured on a 3070 with a live desktop: 7307 MiB of 8192 at this context
19    # with a q4_0 cache, 885 MiB free, and 62 tok/s. A q8_0 cache costs 271 MiB
20    # more for no measured speed, and 128k leaves only 452 MiB, which is inside
21    # the noise of the desktop's own usage.
22    #
23    # Q4_K_M because it is the smallest quant this model's publisher ships.
24    # There is no Unsloth dynamic build for 1.5, and Q5_K_M is 820 MiB heavier,
25    # which would cap context near 16k.
26    #
27    # A path on the volume rather than -hf, so the one service every other one
28    # depends on does not need HuggingFace to be reachable to start.
29    cmd: >
30      llama-server
31      --host 127.0.0.1 --port ${PORT}
32      -m /models/ornith/Ornith-1.5-9B-Q4_K_M.gguf
33      --n-gpu-layers 999
34      --flash-attn on
35      --parallel 1
36      --jinja
37      --ctx-size 65536
38      --cache-type-k q4_0
39      --cache-type-v q4_0
40      --ubatch-size 256