orchard

  1# The model server behind llm.bythewood.me: llama.cpp behind llama-swap.
  2#
  3# Nothing else in the estate runs one of these. Every service that wants a model
  4# goes through the gateway in front of this, which is what makes one set of
  5# weights on one card serve all of them.
  6#
  7# llama-swap is what makes the model on demand. It fronts llama.cpp, starts the
  8# model when a request names it, and unloads after a TTL, freeing the card for
  9# whatever else the machine is doing. Without it the weights sit resident
 10# forever, which on a desktop that is also a workstation is the card gone.
 11
 12FROM debian:trixie-slim AS build
 13
 14# CUDA 13.2 produces gibberish from low-bit quants, so pin away from it.
 15ARG CUDA=13-3
 16
 17RUN apt-get update && \
 18    apt-get install -y --no-install-recommends \
 19        ca-certificates curl git cmake ninja-build build-essential \
 20        libcurl4-openssl-dev && \
 21    curl -fsSLo /tmp/keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64/cuda-keyring_1.1-1_all.deb && \
 22    dpkg -i /tmp/keyring.deb && \
 23    apt-get update && \
 24    apt-get install -y --no-install-recommends \
 25        cuda-nvcc-${CUDA} cuda-cudart-dev-${CUDA} libcublas-dev-${CUDA} && \
 26    rm -rf /var/lib/apt/lists/* /tmp/keyring.deb
 27
 28ENV PATH=/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 29
 30# sm_86 alone keeps the build short, and it is the 3070 this runs on. Change
 31# CMAKE_CUDA_ARCHITECTURES for another card.
 32#
 33# FA_ALL_QUANTS is what makes flash attention work with a quantized KV cache,
 34# which is what fits 32k of context on 8GB.
 35#
 36# Static libs because upstream ships shared objects with no SONAME, which
 37# ldconfig then refuses to cache, and the resulting failure names a missing
 38# .so rather than the cause.
 39ARG LLAMA_REF=master
 40RUN git clone --depth 1 --branch ${LLAMA_REF} https://github.com/ggml-org/llama.cpp /tmp/src && \
 41    cmake -S /tmp/src -B /tmp/build -G Ninja \
 42        -DCMAKE_BUILD_TYPE=Release \
 43        -DBUILD_SHARED_LIBS=OFF \
 44        -DGGML_CUDA=ON \
 45        -DCMAKE_CUDA_ARCHITECTURES=86 \
 46        -DGGML_CUDA_FA_ALL_QUANTS=ON \
 47        -DGGML_NATIVE=OFF \
 48        -DLLAMA_CURL=ON \
 49        -DLLAMA_BUILD_TESTS=OFF \
 50        -DLLAMA_BUILD_EXAMPLES=OFF && \
 51    cmake --build /tmp/build --target llama-server -j"$(nproc)" && \
 52    strip /tmp/build/bin/llama-server
 53
 54# llama-swap ships a static Go binary, so it is fetched rather than built.
 55# Pinned by digest: a release asset can be replaced in place.
 56ARG SWAP_VERSION=252
 57ARG SWAP_SHA256=5f858d72a90ae28aa7c75d34df25d05f9d7daec8c47b9503fb8a264bc6d95d25
 58RUN curl -fsSL -o /tmp/swap.tgz \
 59        "https://github.com/mostlygeek/llama-swap/releases/download/v${SWAP_VERSION}/llama-swap_${SWAP_VERSION}_linux_amd64.tar.gz" && \
 60    echo "${SWAP_SHA256}  /tmp/swap.tgz" | sha256sum -c - && \
 61    tar -xzf /tmp/swap.tgz -C /tmp && \
 62    install -m 0755 /tmp/llama-swap /usr/local/bin/llama-swap
 63
 64
 65FROM debian:trixie-slim
 66
 67ARG CUDA=13-3
 68
 69# libgomp1 and libcurl4 are llama-server's runtime needs; curl is the
 70# healthcheck, since this image has no shell-less alternative.
 71RUN apt-get update && \
 72    apt-get install -y --no-install-recommends \
 73        ca-certificates curl libgomp1 libcurl4 && \
 74    curl -fsSLo /tmp/keyring.deb https://developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64/cuda-keyring_1.1-1_all.deb && \
 75    dpkg -i /tmp/keyring.deb && \
 76    apt-get update && \
 77    apt-get install -y --no-install-recommends \
 78        cuda-cudart-${CUDA} libcublas-${CUDA} && \
 79    rm -rf /var/lib/apt/lists/* /tmp/keyring.deb
 80
 81COPY --from=build /tmp/build/bin/llama-server /usr/local/bin/llama-server
 82COPY --from=build /usr/local/bin/llama-swap /usr/local/bin/llama-swap
 83COPY config.yaml /etc/llama-swap/config.yaml
 84
 85# A malformed config would otherwise surface as a container that starts and
 86# then exits during a deploy. This turns it into a failed build instead.
 87RUN llama-swap --config /etc/llama-swap/config.yaml --validate
 88
 89# Weights live on a volume, not in the image, so a rebuild does not re-download
 90# several gigabytes.
 91ENV LLAMA_CACHE=/models
 92
 93# 65532 like the other images here, so one number owns every volume.
 94RUN groupadd -g 65532 app && useradd -u 65532 -g app -M -d /models app && \
 95    mkdir -p /models && chown 65532:65532 /models
 96VOLUME ["/models"]
 97USER 65532:65532
 98
 99# /v1/models lists what is configured without loading anything, so the
100# healthcheck never keeps the model awake. Asking /health would defeat the TTL
101# this image exists to provide.
102HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
103    CMD curl -fsS http://127.0.0.1:8091/v1/models >/dev/null || exit 1
104
105EXPOSE 8091
106ENTRYPOINT ["llama-swap", "--config", "/etc/llama-swap/config.yaml", "--listen", "0.0.0.0:8091"]