taproot

  1# taproot
  2#
  3# Every operation is a make target, so nothing here needs a docker command typed
  4# by hand. Three cover most days:
  5#
  6#   make up          create what is missing and start it; safe to re-run
  7#   make update      rebuild the image and replace the running container
  8#   make doctor      what exists, what is running, what to type
  9#
 10# A machine that has never run this wants `make install`, which is those plus
 11# the three things no image can carry: the git key, the backup credentials, and
 12# the data itself. `make help` lists everything else.
 13#
 14# Add C=aiagent to any target, it defaults to webdev, which is the one you live
 15# in.
 16#
 17# Run from a clone of this repo, which is the build context.
 18#
 19# The docker socket is root-owned, so every command goes through sudo. On a host
 20# where docker needs no sudo (Docker Desktop), turn it off:  make up SUDO=
 21
 22SUDO   ?= sudo
 23DOCKER ?= $(SUDO) docker
 24
 25C ?= webdev
 26REGISTRY ?= overshard
 27KEY ?= $(HOME)/.ssh/home_key
 28
 29VOLUMES = bythewood-code bythewood-claude bythewood-ssh bythewood-restic bythewood-models \
 30	bythewood-llm
 31
 32NAME   = bythewood-$(C)
 33IMAGE  = $(REGISTRY)/$(C):latest
 34WEBDEV = bythewood-webdev
 35
 36RUN_webdev = --detach --name bythewood-webdev --init --restart unless-stopped \
 37	--publish 8000:8000 \
 38	--volume bythewood-code:/home/dev/code \
 39	--volume bythewood-claude:/home/dev/.claude \
 40	--volume bythewood-ssh:/home/dev/.ssh \
 41	--volume bythewood-restic:/home/dev/.restic \
 42	--volume /var/run/docker.sock:/var/run/docker.sock
 43
 44RUN_aiagent = --detach --name bythewood-aiagent --init --gpus all \
 45	--volume bythewood-code:/home/ai/code \
 46	--volume bythewood-ssh:/home/ai/.ssh \
 47	--volume bythewood-models:/models \
 48	--volume bythewood-llm:/home/ai/.llm \
 49	--volume /var/run/docker.sock:/var/run/docker.sock
 50
 51RUN_ARGS = $(RUN_$(C))
 52
 53.DEFAULT_GOAL := help
 54
 55# install runs its prerequisites in order and a parallel make would not.
 56.NOTPARALLEL:
 57
 58.PHONY: help install up update doctor shell build stop key restic password \
 59	backup snapshots restore sync dotfiles models serve llm-key \
 60	require-container require-webdev
 61
 62help:
 63	@echo "make install        a machine that has never run this, start to finish"
 64	@echo "make up             create what is missing and start it; safe to re-run"
 65	@echo "make update         rebuild the image and replace the running container"
 66	@echo "make doctor         what exists, what is running, what to type"
 67	@echo ""
 68	@echo "make shell          attach via tmux"
 69	@echo "make build          build the image without touching the container"
 70	@echo "make stop           stop it, without replacing anything"
 71	@echo ""
 72	@echo "make key            copy the git ssh key in from this machine"
 73	@echo "make restic         enter or repair the B2 backup credentials"
 74	@echo "make password       print a suggested password to paste into 1Password"
 75	@echo ""
 76	@echo "make backup         take a restic snapshot now"
 77	@echo "make snapshots      last snapshot per host, and what the repo costs"
 78	@echo "make restore        pull everything back from the latest snapshot"
 79	@echo "make sync           pull every repo under ~/code, clone any new ones"
 80	@echo ""
 81	@echo "make models         fetch the model weights once; needs the network"
 82	@echo "make serve MODEL=   run a different model without rebuilding"
 83	@echo "make llm-key KEY=    store the model gateway key aiagent uses"
 84	@echo ""
 85	@echo "add C=aiagent to any of the first block. it defaults to webdev."
 86
 87# The whole of a fresh machine. Every step is idempotent except the restore,
 88# which is skipped when there is already code in the volume to lose.
 89install: up key restic
 90	@echo ""
 91	@if $(DOCKER) exec $(WEBDEV) sh -c 'test -z "$$(ls -A /home/dev/code)"' 2>/dev/null; then \
 92		echo "code/ is empty, restoring the latest snapshot"; \
 93		echo ""; \
 94		$(DOCKER) exec -it $(WEBDEV) restic-restore; \
 95	else \
 96		echo "code/ already has something in it, so the restore is skipped."; \
 97		echo "to pull the snapshot down anyway: make restore"; \
 98	fi
 99	@echo ""
100	@$(MAKE) --no-print-directory doctor
101	@echo ""
102	@echo "get in with: make shell"
103
104# Idempotent, so it repairs as readily as it installs. It never replaces a
105# running container, that is what `update` is for.
106up: require-container
107	for v in $(VOLUMES); do $(DOCKER) volume create $$v >/dev/null; done
108	$(DOCKER) image inspect $(IMAGE) >/dev/null 2>&1 || \
109		$(DOCKER) build --tag $(IMAGE) -f containers/$(C)/Dockerfile .
110	state=$$($(DOCKER) inspect --format '{{.State.Status}}' $(NAME) 2>/dev/null); \
111	case "$$state" in \
112	running) echo "$(NAME) is already running" ;; \
113	"")      echo "creating $(NAME)"; $(DOCKER) run $(RUN_ARGS) $(IMAGE) ;; \
114	*)       echo "starting $(NAME) (was $$state)"; $(DOCKER) start $(NAME) ;; \
115	esac
116	test "$(C)" != "webdev" || $(MAKE) --no-print-directory dotfiles
117	@echo ""
118	$(MAKE) --no-print-directory doctor
119
120# Replacing webdev from inside webdev cannot be done directly: `docker rm
121# --force` on your own container kills the make doing the removing, so the
122# `docker run` after it never happens. A throwaway docker:cli survives that.
123#
124# Nothing below may contain $$(MAKE). GNU make runs any recipe line carrying
125# that string even under `-n`, so a dry run would really replace the container.
126update: require-container
127	$(DOCKER) build --tag $(IMAGE) -f containers/$(C)/Dockerfile .
128	@self=$$(cat /etc/hostname 2>/dev/null); \
129	target=$$($(DOCKER) inspect --format '{{.Id}}' $(NAME) 2>/dev/null | cut -c1-12); \
130	if [ -n "$$target" ] && [ "$$self" = "$$target" ]; then \
131		echo ""; \
132		echo "replacing the container you are sitting in."; \
133		echo "this shell will drop in about two seconds. to get back in:"; \
134		echo ""; \
135		echo "    make shell C=$(C)"; \
136		echo ""; \
137		$(DOCKER) rm --force bythewood-swap >/dev/null 2>&1 || true; \
138		$(DOCKER) run --rm --detach --name bythewood-swap \
139			--volume /var/run/docker.sock:/var/run/docker.sock docker:cli \
140			sh -c "sleep 2; docker rm --force $(NAME); docker run $(RUN_ARGS) $(IMAGE)" >/dev/null; \
141	else \
142		$(DOCKER) rm --force $(NAME) >/dev/null 2>&1 || true; \
143		$(DOCKER) run $(RUN_ARGS) $(IMAGE); \
144		if [ "$(C)" = webdev ]; then \
145			$(DOCKER) exec bythewood-webdev sh -c \
146				"ln -snf /home/dev/code/taproot/dotfiles/claude/skills /home/dev/.claude/skills && \
147				 ln -snf /home/dev/code/taproot/dotfiles/claude/status-line.sh /home/dev/.claude/status-line.sh"; \
148		fi; \
149		echo ""; \
150		echo "$(NAME) replaced. check it with: make doctor"; \
151	fi
152
153# Read only. Every line is either fine or carries the command that fixes it,
154# and every one of those commands is a make target.
155doctor:
156	@probe() { \
157		st=$$($(DOCKER) inspect --format '{{.State.Status}}' "bythewood-$$1" 2>/dev/null); \
158		img=$$($(DOCKER) image inspect $(REGISTRY)/$$1:latest >/dev/null 2>&1 && echo yes || echo no); \
159		if [ "$$img" = no ]; then \
160			printf '  %-10s %-14s %s\n' "$$1" "no image" "-> make up C=$$1"; \
161		elif [ -z "$$st" ]; then \
162			printf '  %-10s %-14s %s\n' "$$1" "not created" "-> make up C=$$1"; \
163		elif [ "$$st" != running ]; then \
164			printf '  %-10s %-14s %s\n' "$$1" "$$st" "-> make up C=$$1"; \
165		else \
166			printf '  %-10s %-14s\n' "$$1" "running"; \
167		fi; \
168	}; \
169	echo "containers"; \
170	probe webdev; \
171	probe aiagent; \
172	echo ""; \
173	echo "volumes"; \
174	sizes=$$($(DOCKER) system df -v 2>/dev/null | awk '/^VOLUME NAME/{v=1;next} v && NF==3{print $$1"="$$3}'); \
175	for v in $(VOLUMES); do \
176		if $(DOCKER) volume inspect $$v >/dev/null 2>&1; then \
177			size=$$(echo "$$sizes" | sed -n "s/^$$v=//p"); \
178			printf '  %-24s %s\n' "$$v" "ok, $${size:-size unknown}"; \
179		else printf '  %-24s %s\n' "$$v" "MISSING  -> make up"; fi; \
180	done; \
181	echo ""; \
182	echo "webdev setup"; \
183	if $(DOCKER) exec $(WEBDEV) test -s /home/dev/.ssh/home_key 2>/dev/null; then \
184		echo "  git ssh key     ok"; \
185	else \
186		echo "  git ssh key     MISSING  -> make key"; \
187	fi; \
188	$(DOCKER) exec $(WEBDEV) restic-setup --check >/dev/null 2>&1; \
189	case $$? in \
190	0)   echo "  restic          ok" ;; \
191	127) echo "  restic          unknown  -> image predates restic-setup, run: make update" ;; \
192	*)   echo "  restic          NOT SET UP  -> make restic" ;; \
193	esac; \
194	if $(DOCKER) exec $(WEBDEV) test -L /home/dev/.claude/skills 2>/dev/null; then \
195		echo "  claude dotfiles ok"; \
196	else \
197		echo "  claude dotfiles MISSING  -> make dotfiles"; \
198	fi
199
200shell: require-container
201	$(DOCKER) exec -it $(NAME) tmux
202
203build: require-container
204	$(DOCKER) build --tag $(IMAGE) -f containers/$(C)/Dockerfile .
205
206stop: require-container
207	$(DOCKER) stop $(NAME)
208
209# The git identity is the one thing that neither the image nor a restore can
210# carry, since restic will not have run yet on a fresh machine. Both containers
211# mount bythewood-ssh, so copying it once serves them both.
212#
213# The chown is explicit because docker cp lands the file as root, and the 600
214# because ssh refuses anything looser and a key copied off Windows arrives with
215# no usable mode at all.
216key: require-webdev
217	@test -s "$(KEY)" || { \
218		echo "no key at $(KEY)" >&2; \
219		echo "put your github key there, or point at it: make key KEY=/path/to/key" >&2; \
220		exit 1; \
221	}
222	@$(DOCKER) cp "$(KEY)" $(WEBDEV):/home/dev/.ssh/home_key
223	@$(DOCKER) exec --user root $(WEBDEV) sh -c \
224		"chown dev:dev /home/dev/.ssh /home/dev/.ssh/home_key && \
225		 chmod 700 /home/dev/.ssh && chmod 600 /home/dev/.ssh/home_key"
226	@echo "git key installed from $(KEY)"
227
228# Interactive, so it needs the tty that -it gives it. Suggests a password for a
229# new repository rather than asking you to invent one.
230restic: require-webdev
231	@$(DOCKER) exec -it $(WEBDEV) restic-setup
232
233# Run out of the checkout rather than the container, so a password can be minted
234# before there is anything to put it in. Nothing is written either way, and the
235# password goes to stdout on its own with the note on stderr, so it pipes clean.
236password:
237	@sh containers/webdev/scripts/restic-setup.sh --password
238
239backup: require-webdev
240	@$(DOCKER) exec $(WEBDEV) restic-backup
241
242snapshots: require-webdev
243	@$(DOCKER) exec $(WEBDEV) restic-status
244
245# Moves anything already in the volumes aside before it writes, and says where.
246restore: require-webdev
247	@$(DOCKER) exec -it $(WEBDEV) restic-restore
248
249sync: require-webdev
250	@$(DOCKER) exec $(WEBDEV) code-sync
251
252# /home/dev/.claude is a volume and shadows whatever the image puts there, so
253# these have to be linked into the running container. Safe to re-run.
254dotfiles:
255	$(DOCKER) exec bythewood-webdev sh -c \
256		"ln -snf /home/dev/code/taproot/dotfiles/claude/skills /home/dev/.claude/skills && \
257		 ln -snf /home/dev/code/taproot/dotfiles/claude/status-line.sh /home/dev/.claude/status-line.sh && \
258		 ln -snf /home/dev/code/taproot/dotfiles/neovim/init.lua /home/dev/.config/nvim/init.lua"
259	echo "claude skills, status line and neovim config linked"
260
261# The only step that reaches Hugging Face, weights land in the volume.
262# --list-devices cannot stand in for this, it exits before -hf is resolved.
263models:
264	-$(DOCKER) rm --force aiagent-fetch
265	$(DOCKER) run --detach --name aiagent-fetch --gpus all \
266		--env LLAMA_ARG_OFFLINE=false \
267		--volume bythewood-models:/models $(REGISTRY)/aiagent:latest
268	echo "downloading weights, several GB, this takes a while"
269	while ! $(DOCKER) exec aiagent-fetch curl -sf -o /dev/null http://127.0.0.1:8000/health 2>/dev/null; do \
270		$(DOCKER) ps -q -f name=aiagent-fetch | grep -q . || \
271			{ echo "fetch failed:"; $(DOCKER) logs --tail 20 aiagent-fetch; exit 1; }; \
272		sleep 10; \
273	done
274	$(DOCKER) rm --force aiagent-fetch >/dev/null
275	echo "weights are in the bythewood-models volume"
276
277# Where aiagent's key for the model gateway is kept, so replacing the container
278# does not lose it. Mint the key on the gateway first, which prints it once:
279#
280#     cd ~/code/orchard && make llm-key NAME=aiagent
281#
282# With no key here aiagent starts its own model instead, which is the right
283# answer on a machine that is not the one running the gateway.
284llm-key:
285	@test -n "$(KEY)" || { \
286		echo "paste the key the gateway printed:" >&2; \
287		echo "" >&2; \
288		echo "  make llm-key KEY=orch-..." >&2; \
289		exit 1; \
290	}
291	@$(DOCKER) volume create bythewood-llm >/dev/null
292	@printf '%s' "$(KEY)" | $(DOCKER) run --rm -i \
293		--volume bythewood-llm:/k alpine:3 \
294		sh -c 'cat > /k/key && chmod 600 /k/key && chown 1001:1001 /k/key'
295	@echo "key stored. it takes effect on the next: make update C=aiagent"
296
297# A one-off in the foreground, so it neither rebuilds the image nor disturbs the
298# aiagent container. Weights it pulls stay in the volume.
299serve:
300	@test -n "$(MODEL)" || { \
301		echo "usage: make serve MODEL=<hf-repo>:<quant>" >&2; \
302		exit 1; \
303	}
304	$(DOCKER) run --rm --gpus all --volume bythewood-models:/models \
305		$(REGISTRY)/aiagent:latest -hf $(MODEL) --alias local
306
307require-container:
308	@test -d "containers/$(C)" || { \
309		echo "there is no container called '$(C)'. one of:" >&2; \
310		echo "  webdev" >&2; \
311		echo "  aiagent" >&2; \
312		exit 1; \
313	}
314
315require-webdev:
316	@$(DOCKER) inspect --format '{{.State.Running}}' $(WEBDEV) 2>/dev/null | grep -q true || { \
317		echo "$(WEBDEV) is not running, and this target works inside it." >&2; \
318		echo "start it with: make up" >&2; \
319		exit 1; \
320	}