taproot

 1#!/bin/sh
 2# Use the shared model gateway when there is one, and be a model server when
 3# there is not.
 4#
 5# llm.bythewood.me holds one set of weights on one card for the whole estate, so
 6# on the desktop this container should not load a second copy and evict it. On a
 7# laptop, or with the gateway down, there is no gateway to use and the container
 8# has to be the thing that serves the model, which is what it always was.
 9#
10# The choice is made once at start, and which way it went is printed, because a
11# container that quietly loaded 6GB of weights it did not need looks identical
12# to one that did the right thing.
13set -eu
14
15GATEWAY="${LLM_URL:-https://llm.bythewood.me}"
16MODELS_JSON="${HOME:-/home/ai}/.pi/agent/models.json"
17
18# The key lives in a volume so replacing the container does not lose it, the
19# same way the restic credentials do. An environment variable wins, which is
20# what a one-off `docker run` uses.
21KEY_FILE="${HOME:-/home/ai}/.llm/key"
22KEY="${LLM_KEY:-}"
23if [ -z "$KEY" ] && [ -r "$KEY_FILE" ]; then
24	KEY=$(tr -d " \t\r\n" < "$KEY_FILE")
25fi
26
27# A key is required, so a machine with no key falls through to its own model
28# rather than sitting there getting 401s from a gateway it cannot use.
29gateway_answers() {
30	[ -n "$KEY" ] || return 1
31	curl -fsS --max-time 5 \
32		-H "Authorization: Bearer $KEY" \
33		"$GATEWAY/v1/models" >/dev/null 2>&1
34}
35
36point_pi_at() {
37	tmp="$MODELS_JSON.tmp"
38	jq --arg url "$1/v1" --arg key "$2" \
39		'.providers.local.baseUrl = $url | .providers.local.apiKey = $key' \
40		"$MODELS_JSON" > "$tmp" && mv "$tmp" "$MODELS_JSON"
41}
42
43if gateway_answers; then
44	point_pi_at "$GATEWAY" "$KEY"
45	echo "using the model gateway at $GATEWAY, no weights loaded here" >&2
46	# Nothing to serve, but the container is a place to exec into, so it has
47	# to stay up. It runs with --init, which reaps what tmux leaves behind.
48	exec sleep infinity
49fi
50
51if [ -n "$KEY" ]; then
52	echo "the gateway at $GATEWAY did not answer, serving a model here instead" >&2
53else
54	echo "no LLM_KEY set, serving a model here" >&2
55fi
56point_pi_at "http://127.0.0.1:8000" "none"
57exec /usr/local/bin/llama-server "$@"