orchard

 1name: orchard-llm
 2
 3# Two services. The gateway is on the edge network so Caddy and every other site
 4# can reach it, and the model server is on a private one that nothing else
 5# joins, since it has no authentication of its own and the gateway is what
 6# supplies that.
 7
 8services:
 9  app:
10    build:
11      context: .
12      dockerfile: Dockerfile
13    container_name: orchard-llm
14    restart: unless-stopped
15    # Longer than Docker's ten second default so a generation in flight finishes
16    # and the call log closes cleanly rather than being killed mid write.
17    stop_grace_period: 30s
18    environment:
19      LLM_UPSTREAM: http://swap:8091
20      LLM_MODEL: local
21      LLM_DB: /data/llm.db
22    volumes:
23      - data:/data
24    depends_on:
25      swap:
26        condition: service_healthy
27    deploy:
28      resources:
29        limits:
30          cpus: "1.00"
31          memory: 512M
32    security_opt:
33      - no-new-privileges:true
34    logging:
35      driver: json-file
36      options:
37        max-size: "10m"
38        max-file: "3"
39    networks: [edge, model]
40
41  # llama.cpp behind llama-swap with the card attached. Ornith 1.5 9B at Q4_K_M,
42  # 64k of context with a q4_0 KV cache, unloaded after three idle minutes so a
43  # desktop that is also a workstation gets its card back.
44  swap:
45    build:
46      context: ./modelserver
47      dockerfile: Dockerfile
48    container_name: orchard-llm-swap
49    restart: unless-stopped
50    volumes:
51      - models:/models
52    deploy:
53      resources:
54        reservations:
55          devices:
56            - driver: nvidia
57              count: all
58              capabilities: [gpu]
59    # /v1/models answers from llama-swap's own config and loads nothing, so this
60    # never keeps the model awake and defeats the idle unload.
61    healthcheck:
62      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8091/v1/models"]
63      interval: 30s
64      timeout: 5s
65      retries: 3
66      start_period: 20s
67    security_opt:
68      - no-new-privileges:true
69    logging:
70      driver: json-file
71      options:
72        max-size: "10m"
73        max-file: "3"
74    networks: [model]
75
76volumes:
77  data:
78    name: orchard-llm-data
79  models:
80    name: orchard-llm-models
81
82networks:
83  edge:
84    name: orchard-edge
85    external: true
86  model:
87    name: orchard-llm-model