orchard

  1// Command chat serves chat.bythewood.me: a conversation with a small model
  2// running on Isaac's own card, with tools for anything it cannot know, history
  3// in SQLite, and an incognito mode that writes nothing down.
  4package main
  5
  6import (
  7	"context"
  8	"encoding/json"
  9	"flag"
 10	"fmt"
 11	"html/template"
 12	"log/slog"
 13	"mime/multipart"
 14	"net/http"
 15	"os"
 16	"strconv"
 17	"strings"
 18	"time"
 19
 20	"chat.bythewood.me/tools"
 21	"chat.bythewood.me/web"
 22	"github.com/yuin/goldmark"
 23	"github.com/yuin/goldmark/extension"
 24)
 25
 26const (
 27	listenAddr = ":8000"
 28	sourceURL  = "https://github.com/overshard/orchard/tree/main/sites/chat.bythewood.me"
 29	selfSource = "chat"
 30)
 31
 32// csp is the same shape as every other site here. 'unsafe-inline' is for the
 33// analytics collector snippet, which is an inline script by design, and for the
 34// handful of inline style attributes the templates set.
 35func csp() string {
 36	return strings.Join([]string{
 37		"default-src 'self'",
 38		"script-src 'self' 'unsafe-inline' https://analytics.bythewood.me",
 39		"style-src 'self' 'unsafe-inline'",
 40		"img-src 'self' data:",
 41		"font-src 'self'",
 42		"connect-src 'self' https://analytics.bythewood.me",
 43		"base-uri 'self'",
 44		"form-action 'self'",
 45		"frame-ancestors 'none'",
 46	}, "; ")
 47}
 48
 49type site struct {
 50	auth    *web.Authenticator
 51	llm     *LLM
 52	engine  *Engine
 53	store   *Store
 54	runs    *Runs
 55	queue   *Queue
 56	hub     *Hub
 57	comp    *Compactor
 58	tpl     *template.Template
 59	md      goldmark.Markdown
 60	label   string
 61	ctxSize int
 62	dev     bool
 63}
 64
 65func round1(f float64) float64 { return float64(int(f*10+0.5)) / 10 }
 66
 67func main() {
 68	var (
 69		addr    = flag.String("addr", env("CHAT_ADDR", listenAddr), "listen address")
 70		llmURL  = flag.String("llm", env("LLM_URL", "http://orchard-llm:8000"), "model gateway base url")
 71		llmKey  = flag.String("llm-key", os.Getenv("LLM_KEY"), "api key for the model gateway")
 72		verify  = flag.String("verify", "", "development only: check sessions against this url instead of auth")
 73		model   = flag.String("model", env("LLM_MODEL", "local"), "model name the server answers to")
 74		label   = flag.String("model-name", env("LLM_NAME", "Ornith 1.5 9B"), "readable model name, shown in the UI and told to the model")
 75		dbPath  = flag.String("db", env("CHAT_DB", "data/chat.db"), "conversation database")
 76		wikiURL = flag.String("wiki", env("WIKI_URL", "http://orchard-wiki:8000"), "offline wikipedia base url")
 77		ctxSize = flag.Int("ctx", envInt("LLM_CTX", 32768), "model context window in tokens")
 78		health  = flag.Bool("healthcheck", false, "probe the local server and exit")
 79	)
 80	flag.Parse()
 81	web.SetupLogging()
 82
 83	if *health {
 84		if err := web.HealthCheck("http://127.0.0.1"+*addr+"/healthz", 3*time.Second); err != nil {
 85			os.Exit(1)
 86		}
 87		return
 88	}
 89
 90	// Past the healthcheck branch, so a HEALTHCHECK invocation does not start a
 91	// queue it will never flush. This was the only one of the eleven sites not
 92	// shipping, which is why logging.bythewood.me had no record of chat at all.
 93	web.ShipLogs(selfSource, web.HTTPSink())
 94
 95	store, err := OpenStore(*dbPath)
 96	if err != nil {
 97		slog.Error("open store", "err", err)
 98		os.Exit(1)
 99	}
100	defer store.Close()
101
102	tools.WikiBase = *wikiURL
103
104	llm := NewLLM(*llmURL, *model, *llmKey)
105	// The verifier can only be moved in a development build. Reloaded is false
106	// in the shipped image, so this cannot be turned on however the environment
107	// is set, the same fence assets_disk.go has.
108	auth := web.NewAuthenticator()
109	if Reloaded && *verify != "" {
110		slog.Warn("checking sessions against a development verifier", "url", *verify)
111		auth = web.NewAuthenticatorAt(*verify)
112	}
113
114	s := &site{
115		auth: auth,
116		llm:  llm, engine: NewEngine(llm, *label), store: store, label: *label,
117		comp: NewCompactor(llm, *ctxSize), ctxSize: *ctxSize, dev: Reloaded,
118		runs: NewRuns(), queue: NewQueue(), hub: NewHub(),
119		md: goldmark.New(goldmark.WithExtensions(extension.GFM),
120			goldmark.WithRendererOptions()),
121	}
122	s.engine.Render = s.render
123	// The remember tool writes to this process's own database rather than to a
124	// service, so it is handed the store rather than a url.
125	s.engine.Deps().Memory = memoryStore{store}
126	s.engine.Deps().History = historyStore{store}
127	// The rate limit boxes survive a restart. Without this every deploy asked a
128	// host that was already refusing, which is how a ban gets renewed rather
129	// than expiring.
130	s.engine.RestoreGuard(store, store.Penalties())
131	s.engine.RestoreSpend(store.Spend(tools.SearchHost))
132	// Finished runs are kept a while so a tab coming back can still read one,
133	// and swept after that rather than held until the process restarts.
134	go func() {
135		for range time.Tick(5 * time.Minute) {
136			s.runs.Sweep(time.Now())
137		}
138	}()
139	if err := s.loadTemplates(); err != nil {
140		slog.Error("templates", "err", err)
141		os.Exit(1)
142	}
143
144	mux := http.NewServeMux()
145	// Everything is gated. This site can read Isaac's own infrastructure
146	// through its tools, so there is no anonymous surface at all, not even a
147	// landing page, which is the difference between this and the dashboards
148	// that show a signed out visitor something.
149	// The root is the one public page: what this is and a way in. Everything
150	// past it needs a session, and a signed in visitor gets the app here rather
151	// than the pitch.
152	mux.HandleFunc("GET /{$}", s.root)
153	mux.HandleFunc("GET /c/{id}", s.auth.RequireAuth(s.page))
154	// Memory. Reading and deleting are plain pages and plain posts, and the
155	// only way to write is the one that goes through the model.
156	mux.HandleFunc("GET /api/memory", s.auth.RequireAuthJSON(s.memoryList))
157	mux.HandleFunc("POST /api/memory/teach", s.auth.RequireAuthJSON(s.memoryTeach))
158	mux.HandleFunc("DELETE /api/memory/{id}", s.auth.RequireAuthJSON(s.memoryDelete))
159	mux.HandleFunc("DELETE /api/memory", s.auth.RequireAuthJSON(s.memoryForget))
160	// Stylesheets and scripts, which the landing page needs and which carry
161	// nothing a session would protect.
162	mux.Handle("GET /static/", s.static())
163	// Signing in happens on auth.bythewood.me, and this is here so an old
164	// bookmark lands somewhere useful rather than on a redirect loop.
165	mux.HandleFunc("GET /login", func(w http.ResponseWriter, r *http.Request) {
166		http.Redirect(w, r, web.LoginURL(r), http.StatusSeeOther)
167	})
168	// Not gated, because the container's own health check calls it over
169	// loopback with no cookie and it says nothing.
170	mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) {
171		w.Header().Set("Cache-Control", "no-store")
172		fmt.Fprintln(w, "ok")
173	})
174	mux.HandleFunc("POST /api/send", s.auth.RequireAuthJSON(s.send))
175	mux.HandleFunc("GET /api/attach/{id}", s.auth.RequireAuthJSON(s.attach))
176	mux.HandleFunc("POST /api/stop/{id}", s.auth.RequireAuthJSON(s.stop))
177	mux.HandleFunc("GET /api/conversations", s.auth.RequireAuthJSON(s.listConversations))
178	mux.HandleFunc("GET /api/conversation/{id}", s.auth.RequireAuthJSON(s.getConversation))
179	mux.HandleFunc("DELETE /api/conversation/{id}", s.auth.RequireAuthJSON(s.deleteConversation))
180	mux.HandleFunc("DELETE /api/conversations", s.auth.RequireAuthJSON(s.deleteAll))
181	mux.HandleFunc("GET /api/status", s.auth.RequireAuthJSON(s.status))
182	mux.HandleFunc("GET /api/events", s.auth.RequireAuthJSON(s.events))
183	// The readings behind a chart. Gated like everything else here, and read
184	// only: both go out to a public source and neither touches this estate.
185	mux.HandleFunc("GET /api/widget/ticker", s.auth.RequireAuthJSON(s.widgetTicker))
186	mux.HandleFunc("GET /api/widget/weather", s.auth.RequireAuthJSON(s.widgetWeather))
187
188	slog.Info("chat listening", "addr", *addr, "llm", *llmURL, "model", *label,
189		"db", *dbPath, "reloaded", Reloaded)
190	web.Serve(*addr, web.Chain(mux, web.Recovered, web.Logged, web.SecurityHeaders(csp())))
191}
192
193func env(k, def string) string {
194	if v := os.Getenv(k); v != "" {
195		return v
196	}
197	return def
198}
199
200func envInt(k string, def int) int {
201	if v := os.Getenv(k); v != "" {
202		if n, err := strconv.Atoi(v); err == nil {
203			return n
204		}
205	}
206	return def
207}
208
209func (s *site) loadTemplates() error {
210	// Reading assets off disk means their hashes change while the process is
211	// up, so the reload path drops the cached versions with the templates.
212	if Reloaded {
213		resetAssetVersions()
214	}
215	t, err := template.New("").Funcs(template.FuncMap{
216		"when":  fmtWhen,
217		"asset": assetURL,
218	}).ParseFS(assets(), "templates/*.html")
219	if err != nil {
220		return err
221	}
222	s.tpl = t
223	return nil
224}
225
226func (s *site) static() http.Handler {
227	fs := http.FileServerFS(assets())
228	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
229		// Assets come off disk in development so a CSS edit shows up on
230		// reload, and out of the binary in the release build. That split is
231		// why an embedded stylesheet does not silently ignore an edit.
232		if Reloaded {
233			w.Header().Set("Cache-Control", "no-store")
234		} else {
235			w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
236		}
237		fs.ServeHTTP(w, r)
238	})
239}
240
241// root shows the app to anyone signed in and the landing page to everyone else.
242func (s *site) root(w http.ResponseWriter, r *http.Request) {
243	if s.auth.Authenticated(r) {
244		s.page(w, r)
245		return
246	}
247	s.landing(w, r)
248}
249
250func (s *site) landing(w http.ResponseWriter, r *http.Request) {
251	if Reloaded {
252		if err := s.loadTemplates(); err != nil {
253			http.Error(w, err.Error(), 500)
254			return
255		}
256	}
257	w.Header().Set("Content-Type", "text/html; charset=utf-8")
258	w.Header().Set("Cache-Control", "no-store")
259	if err := s.tpl.ExecuteTemplate(w, "landing.html", map[string]any{
260		"Source": sourceURL, "Model": s.label, "Ctx": kfmt(s.ctxSize),
261	}); err != nil {
262		slog.Error("render landing", "err", err)
263	}
264}
265
266func (s *site) page(w http.ResponseWriter, r *http.Request) {
267	if Reloaded {
268		if err := s.loadTemplates(); err != nil {
269			http.Error(w, err.Error(), 500)
270			return
271		}
272	}
273	convs, _ := s.store.List(40)
274	nConv, nMsg := s.store.Count()
275	data := map[string]any{
276		"Conversations": convs,
277		"Source":        sourceURL,
278		"Dev":           s.dev,
279		"Model":         s.label,
280		"Stats":         map[string]int{"Conversations": nConv, "Messages": nMsg},
281		"Ctx":           *(&s.ctxSize),
282		"Active":        r.PathValue("id"),
283	}
284	w.Header().Set("Content-Type", "text/html; charset=utf-8")
285	if err := s.tpl.ExecuteTemplate(w, "app.html", data); err != nil {
286		slog.Error("render", "err", err)
287	}
288}
289
290type sendReq struct {
291	Message string `json:"message"`
292	ConvID  string `json:"conversation_id"`
293	// The id the browser made up for a conversation that does not exist yet, so
294	// a turn started before the first row is written is still findable if the
295	// tab goes away in the middle of it.
296	RunID     string `json:"run_id"`
297	Incognito bool   `json:"incognito"`
298}
299
300// A turn's whole upload, across every file on it. The per file ceiling is in
301// attach.go and this is the one that stops ten of them at once.
302const maxUploadBytes = 64 << 20
303
304// readSend accepts either JSON or a multipart form, since a turn carrying files
305// cannot be JSON and a turn without them should not have to be a form.
306func readSend(w http.ResponseWriter, r *http.Request) (sendReq, []filePart, error) {
307	var req sendReq
308	if !strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
309		if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
310			return req, nil, fmt.Errorf("bad request")
311		}
312		req.Message = strings.TrimSpace(req.Message)
313		return req, nil, nil
314	}
315	r.Body = http.MaxBytesReader(w, r.Body, maxUploadBytes)
316	if err := r.ParseMultipartForm(16 << 20); err != nil {
317		return req, nil, fmt.Errorf("that is more than %s of attachments", humanSize(maxUploadBytes))
318	}
319	// Anything over the memory limit was spooled to a temp file, so the parts
320	// are turned into text here and the spill is dropped before the turn runs,
321	// rather than sitting there for the twelve minutes a turn may take.
322	defer func() { _ = r.MultipartForm.RemoveAll() }()
323
324	req.Message = strings.TrimSpace(r.FormValue("message"))
325	req.ConvID = strings.TrimSpace(r.FormValue("conversation_id"))
326	req.RunID = strings.TrimSpace(r.FormValue("run_id"))
327	req.Incognito = r.FormValue("incognito") == "true"
328	var headers []*multipart.FileHeader
329	if r.MultipartForm != nil {
330		headers = r.MultipartForm.File["files"]
331	}
332	return req, readFiles(headers), nil
333}
334
335// send starts one turn and streams it. Server sent events rather than a
336// websocket because the traffic is one way and this survives a proxy.
337//
338// The turn itself runs detached, so the browser is a reader and not the thing
339// the work depends on. Closing the tab drops the reader and the turn carries on.
340func (s *site) send(w http.ResponseWriter, r *http.Request) {
341	req, parts, err := readSend(w, r)
342	if err != nil {
343		http.Error(w, err.Error(), 400)
344		return
345	}
346	// A turn carrying files needs no message, since the files are the question.
347	if req.Message == "" && len(parts) == 0 {
348		http.Error(w, "empty message", 400)
349		return
350	}
351
352	// A new conversation has no id until its first turn is stored, so the
353	// browser sends one it made up and the run is keyed by that until the store
354	// hands over the real one.
355	key := req.ConvID
356	if key == "" {
357		key = req.RunID
358	}
359	if key == "" {
360		http.Error(w, "no conversation or run id", 400)
361		return
362	}
363
364	var session string
365	if c, err := r.Cookie(web.SessionCookie); err == nil {
366		session = c.Value
367	}
368
369	rn := s.runs.Start(key)
370	// Incognito is left out of every one of these. A mode that writes nothing
371	// down must not announce itself to a tab on another device either.
372	if !req.Incognito {
373		s.hub.Publish(HubEvent{Kind: "started", ConvID: key})
374	}
375	// Detached on purpose. r.Context() dies with the tab, and a turn that has
376	// spent two minutes fetching should not be thrown away because a phone
377	// locked.
378	ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 12*time.Minute)
379	rn.setCancel(cancel)
380	go func() {
381		defer cancel()
382		defer rn.Finish()
383		defer func() {
384			if p := recover(); p != nil {
385				slog.Error("a turn panicked", "err", p)
386				rn.Emit(Event{Kind: "error", Text: "that turn failed"})
387			}
388		}()
389		s.turn(ctx, rn, key, req, parts, session)
390	}()
391	s.streamRun(w, r, rn)
392}
393
394// streamRun writes a run to one browser: everything it has already produced,
395// then whatever comes next until the turn ends or this reader goes away.
396// events is the meta stream every open tab holds, so a turn started anywhere is
397// known everywhere. It carries which conversation changed and never what was
398// said: a tab is told to go and read, and the run is still the only place an
399// answer is assembled.
400func (s *site) events(w http.ResponseWriter, r *http.Request) {
401	w.Header().Set("Content-Type", "text/event-stream")
402	w.Header().Set("Cache-Control", "no-store")
403	w.Header().Set("X-Accel-Buffering", "no")
404	rc := http.NewResponseController(w)
405	_ = rc.SetWriteDeadline(time.Time{})
406
407	ch, cancel := s.hub.Subscribe()
408	defer cancel()
409
410	// An immediate frame, so a proxy that buffers until it sees output lets the
411	// stream through rather than holding it until the first real event.
412	if _, err := fmt.Fprint(w, ": open\n\n"); err != nil || rc.Flush() != nil {
413		return
414	}
415	// A comment every half minute keeps the connection through Cloudflare and
416	// Caddy, neither of which will hold an idle stream open indefinitely.
417	beat := time.NewTicker(30 * time.Second)
418	defer beat.Stop()
419
420	for {
421		select {
422		case <-r.Context().Done():
423			return
424		case ev, ok := <-ch:
425			if !ok {
426				return
427			}
428			b := ev.frame()
429			if b == nil {
430				continue
431			}
432			if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
433				return
434			}
435			if rc.Flush() != nil {
436				return
437			}
438		case <-beat.C:
439			if _, err := fmt.Fprint(w, ": beat\n\n"); err != nil || rc.Flush() != nil {
440				return
441			}
442		}
443	}
444}
445
446func (s *site) streamRun(w http.ResponseWriter, r *http.Request, tr *turnRun) {
447	w.Header().Set("Content-Type", "text/event-stream")
448	w.Header().Set("Cache-Control", "no-store")
449	w.Header().Set("X-Accel-Buffering", "no")
450	// A type assertion for http.Flusher does not survive the request logger,
451	// which wraps the writer in a recorder that only promotes three methods.
452	// ResponseController follows Unwrap and works through it.
453	rc := http.NewResponseController(w)
454	// No write bound, since a turn runs for minutes and any deadline here is a
455	// ceiling on how long the stream may stay open.
456	_ = rc.SetWriteDeadline(time.Time{})
457
458	backlog, ch, live := tr.Follow()
459	write := func(b []byte) bool {
460		if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil {
461			return false
462		}
463		return rc.Flush() == nil
464	}
465	for _, b := range backlog {
466		if !write(b) {
467			if live {
468				tr.Unfollow(ch)
469			}
470			return
471		}
472	}
473	if !live {
474		return
475	}
476	defer tr.Unfollow(ch)
477	for {
478		select {
479		case <-r.Context().Done():
480			return
481		case b, ok := <-ch:
482			if !ok {
483				return
484			}
485			if !write(b) {
486				return
487			}
488		}
489	}
490}
491
492// turn is the work, with no http in it. It writes into the run rather than to a
493// response, which is what lets it outlive the request that started it.
494func (s *site) turn(ctx context.Context, rn *turnRun, key string, req sendReq, parts []filePart, session string) {
495	prompt := composeTurn(req.Message, parts)
496	emit := func(e Event) { rn.Emit(e) }
497	tr := NewTrace(emit)
498
499	// One turn at a time. There is one card behind this and llama.cpp runs it
500	// with a single slot, so two turns at once interleave and both take longer
501	// than they would have taken in order. The waiting is shown rather than
502	// hidden, since a tab sat on "thinking" because another is ahead looks
503	// broken and one that says it is second in line looks like a queue.
504	release, ok := s.queue.Enter(ctx, func(q QueueState) {
505		emit(Event{Kind: "status", Text: waitingLabel(q)})
506	})
507	if !ok {
508		emit(Event{Kind: "error", Text: "that turn was stopped before it started"})
509		return
510	}
511	defer release()
512	// Every model call this turn makes hangs off this context, including the
513	// ones the tools start, so marking it here is what keeps the gateway from
514	// writing down what the local database is not writing down either.
515	if req.Incognito {
516		ctx = WithIncognito(ctx)
517	}
518
519	// Load the conversation and build the window before anything else, since
520	// compaction may need a model call of its own.
521	var history []Message
522	var conv Conversation
523	var stored []Stored
524	if !req.Incognito && req.ConvID != "" {
525		conv, _ = s.store.Get(req.ConvID)
526		stored, _ = s.store.Messages(req.ConvID)
527		var summary string
528		var covered int
529		var changed bool
530		history, summary, covered, changed = s.comp.Window(ctx, conv, stored)
531		if changed {
532			emit(Event{Kind: "status", Text: "compacting"})
533			_ = s.store.SetSummary(req.ConvID, summary, covered)
534			tr.Add(Step{Kind: "compact", Label: "rewrote the older history as a summary",
535				Out: summary, Meta: itoa(covered) + " earlier messages replaced"})
536		}
537	}
538
539	// Warm the weights while the window is being built rather than after.
540	go s.llm.Warm(context.WithoutCancel(ctx))
541
542	// Retrieval is against what the user typed, not the composed prompt, since
543	// the text of an attachment would swamp the scoring with its own words.
544	recalled := s.store.Relevant(req.Message, factsPerTurn)
545	if len(recalled) > 0 {
546		tr.Add(Step{Kind: "memory", Label: "recalled what it knows about Isaac",
547			In: req.Message, Out: memoryBlock(recalled),
548			Meta: itoa(len(recalled)) + " of the stored facts scored against this question"})
549	}
550	reply, used, srcs, widgets, stats, err := s.engine.Run(ctx, history, prompt, session, memoryBlock(recalled), tr, emit)
551	// Whether the turn worked or not, whatever it spent has been spent, and a
552	// failed turn is exactly when the counts matter most.
553	s.engine.SaveSpend(s.store.SaveSpend)
554	if err != nil {
555		emit(Event{Kind: "error", Text: err.Error()})
556		return
557	}
558
559	// Whatever the model wrote, the stored copy has no leaked markup in it.
560	reply.Content, _ = salvageCalls(reply.Content, func(string) bool { return false })
561
562	summaries := make([]ToolSummary, 0, len(used))
563	for _, u := range used {
564		summaries = append(summaries, ToolSummary{
565			Name: u.Name, Args: shortArgs(string(u.Args)),
566			MS: u.Elapsed.Milliseconds(), OK: u.Err == "", Err: u.Err,
567			Age: snapshotAge(u.Content),
568		})
569	}
570
571	convID := req.ConvID
572	// The bar shows this. Without it a conversation named on its first turn
573	// keeps saying "New conversation" until the tab navigates away and back.
574	title := ""
575	if !req.Incognito {
576		if convID == "" {
577			if id, e := s.store.NewConversation(""); e == nil {
578				convID = id
579				// The run was keyed by the id the browser made up. Move it, so
580				// a tab reopening this conversation by its real id finds the
581				// turn that is still writing into it.
582				s.runs.Rekey(key, convID)
583				// The browser made the old key up, so a tab that was told a
584				// turn had started under it needs the real id to follow.
585				s.hub.Publish(HubEvent{Kind: "started", ConvID: convID})
586			}
587		}
588		if convID != "" {
589			user := Stored{Role: RoleUser, Content: prompt}
590			if len(parts) > 0 {
591				user.Display, user.Files = req.Message, attachments(parts)
592			}
593			_ = s.store.Append(convID, user)
594			_ = s.store.Append(convID, Stored{Role: RoleAssistant, Content: reply.Content,
595				Tools: summaries, Sources: srcs, Widgets: widgets, Steps: tr.Steps()})
596			if len(stored) == 0 {
597				seed := titleSeed(req.Message, parts)
598				titleStart := time.Now()
599				if t := s.comp.Title(context.WithoutCancel(ctx), seed, reply.Content); t != "" {
600					title = t
601					_ = s.store.SetTitle(convID, t)
602					tr.Add(Step{Kind: "title", Label: "named the conversation",
603						In: seed, Out: t, MS: time.Since(titleStart).Milliseconds()})
604				}
605			}
606			s.store.Checkpoint()
607		}
608	}
609
610	// After the answer is on its way, never in front of it. Incognito is
611	// excluded: a mode that writes nothing down cannot be the one that teaches
612	// it something to write down later.
613	if !req.Incognito {
614		go func() {
615			// Recovered here and not by the middleware, which only wraps the
616			// handler. A panic on this goroutine would take the process down
617			// and lose every conversation in flight.
618			defer func() {
619				if r := recover(); r != nil {
620					slog.Error("the memory pass panicked", "err", r)
621				}
622			}()
623			bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Minute)
624			defer cancel()
625			s.Remember(bg, req.Message, reply.Content)
626		}()
627	}
628
629	done := map[string]any{"kind": "done", "conversation_id": convID, "title": title,
630		"tools": summaries, "html": s.renderCited(reply.Content, srcs), "incognito": req.Incognito,
631		"steps":   tr.Steps(),
632		"sources": srcs,
633		"files":   attachments(parts),
634		"stats": map[string]any{
635			"prompt_tokens": stats.Prompt, "completion_tokens": stats.Completion,
636			"decode_tps": round1(stats.Decode), "prefill_tps": round1(stats.Prefill),
637			"ctx": s.ctxSize,
638		}}
639	rn.Emit(done)
640	if !req.Incognito && convID != "" {
641		s.hub.Publish(HubEvent{Kind: "finished", ConvID: convID, Title: title})
642	}
643}
644
645// attach lets a tab that went away pick a turn back up. It is the same stream
646// send writes, from the beginning, so a browser that missed the first half sees
647// it replayed and then follows the rest.
648func (s *site) attach(w http.ResponseWriter, r *http.Request) {
649	id := r.PathValue("id")
650	rn, ok := s.runs.Get(id)
651	if !ok {
652		// Nothing running and nothing kept. The conversation endpoint has the
653		// messages, so this is not an error, there is just nothing to follow.
654		http.Error(w, "no turn to attach to", http.StatusNotFound)
655		return
656	}
657	s.streamRun(w, r, rn)
658}
659
660// stop cancels a turn. A browser that has stopped listening is not a reason to
661// stop the work, so this is the only thing that is.
662func (s *site) stop(w http.ResponseWriter, r *http.Request) {
663	writeJSON(w, map[string]any{"stopped": s.runs.Cancel(r.PathValue("id"))})
664}
665
666// waitingLabel says where in the queue a turn is, in words rather than a
667// number on its own, since "2" beside a spinner reads as an error code.
668func waitingLabel(q QueueState) string {
669	switch {
670	case q.Ahead <= 0:
671		return "waiting for the card"
672	case q.Ahead == 1:
673		return "waiting, one turn ahead"
674	default:
675		return "waiting, " + itoa(q.Ahead) + " turns ahead"
676	}
677}
678
679// render turns the model's markdown into HTML on the server, so the browser
680// never has to parse markdown and the sanitising happens in one place.
681func (s *site) render(md string) string {
682	var sb strings.Builder
683	if err := s.md.Convert([]byte(md), &sb); err != nil {
684		return "<p>" + template.HTMLEscapeString(md) + "</p>"
685	}
686	return sb.String()
687}
688
689// renderCited is the same render with the citation numbers turned into links.
690// The markdown is stored with its numbers rather than its anchors, so a change
691// to how a pill looks does not need every old message rewritten.
692func (s *site) renderCited(md string, srcs []Source) string {
693	return linkCitations(s.render(md), srcs)
694}
695
696func (s *site) listConversations(w http.ResponseWriter, r *http.Request) {
697	convs, err := s.store.List(60)
698	if err != nil {
699		http.Error(w, err.Error(), 500)
700		return
701	}
702	writeJSON(w, map[string]any{"conversations": convs})
703}
704
705func (s *site) getConversation(w http.ResponseWriter, r *http.Request) {
706	id := r.PathValue("id")
707	msgs, err := s.store.Messages(id)
708	if err != nil {
709		http.Error(w, err.Error(), 500)
710		return
711	}
712	type out struct {
713		Role    Role          `json:"role"`
714		HTML    string        `json:"html"`
715		Text    string        `json:"text"`
716		Files   []Attachment  `json:"files,omitempty"`
717		Tools   []ToolSummary `json:"tools,omitempty"`
718		Sources []Source      `json:"sources,omitempty"`
719		Widgets []Widget      `json:"widgets,omitempty"`
720		Steps   []Step        `json:"steps,omitempty"`
721	}
722	rendered := make([]out, 0, len(msgs))
723	for _, m := range msgs {
724		o := out{Role: m.Role, Text: m.Shown(), Files: m.Files, Tools: m.Tools,
725			Sources: m.Sources, Widgets: m.Widgets, Steps: m.Steps}
726		if m.Role == RoleAssistant {
727			o.HTML = s.renderCited(m.Content, m.Sources)
728		}
729		rendered = append(rendered, o)
730	}
731	conv, _ := s.store.Get(id)
732	// Whether a turn is still writing into this conversation, so a tab that
733	// comes back knows to attach and follow rather than render what is stored
734	// and stop, which is how a half finished turn looked like a lost one.
735	running := false
736	if rn, ok := s.runs.Get(id); ok {
737		running = rn.Running()
738	}
739	writeJSON(w, map[string]any{"id": id, "title": conv.Title,
740		"messages": rendered, "running": running})
741}
742
743func (s *site) deleteConversation(w http.ResponseWriter, r *http.Request) {
744	id := r.PathValue("id")
745	if err := s.store.Delete(id); err != nil {
746		http.Error(w, err.Error(), 500)
747		return
748	}
749	s.hub.Publish(HubEvent{Kind: "changed", ConvID: id})
750	writeJSON(w, map[string]any{"deleted": id})
751}
752
753func (s *site) deleteAll(w http.ResponseWriter, r *http.Request) {
754	if err := s.store.DeleteAll(); err != nil {
755		http.Error(w, err.Error(), 500)
756		return
757	}
758	s.hub.Publish(HubEvent{Kind: "changed"})
759	writeJSON(w, map[string]any{"deleted": "all"})
760}
761
762func (s *site) status(w http.ResponseWriter, r *http.Request) {
763	nConv, nMsg := s.store.Count()
764	out := map[string]any{
765		"model": s.label, "up": s.llm.Healthy(r.Context()), "ctx": s.ctxSize,
766		"tools": tools.Default().Names(), "conversations": nConv, "messages": nMsg,
767	}
768	// Search being unavailable is the one tool failure worth saying out loud,
769	// because a turn without it answers from memory and reads like an ordinary
770	// answer. Everything else is narrow enough to report itself in the turn.
771	if left, down := s.engine.SearchDown(); down {
772		out["search_down"] = true
773		out["search_back_in"] = left.String()
774	}
775	minute, hour, day := s.engine.SearchSpend()
776	out["search_left"] = map[string]int{"minute": minute, "hour": hour, "day": day}
777	writeJSON(w, out)
778}
779
780func writeJSON(w http.ResponseWriter, v any) {
781	w.Header().Set("Content-Type", "application/json")
782	_ = json.NewEncoder(w).Encode(v)
783}
784
785// kfmt matches the meter in the app, which divides by 1024 because a context
786// window is a power of two and 65536 has to read as the 64k it was set to.
787func kfmt(n int) string {
788	if n < 1024 {
789		return strconv.Itoa(n)
790	}
791	return strconv.Itoa(n/1024) + "k"
792}
793
794// snapshotAge reads the date a tool's data was taken, when it has one. Any tool
795// answering from a snapshot rather than the live source reports it the same
796// way, so the chip needs no per tool knowledge.
797func snapshotAge(content any) string {
798	m, ok := content.(map[string]any)
799	if !ok {
800		return ""
801	}
802	s, _ := m["snapshot_date"].(string)
803	return s
804}