orchard

  1// Command search serves search.bythewood.me: ask a question, it runs
  2// DuckDuckGo, fetches and cleans the pages it finds, and writes an answer whose
  3// every sentence is checked against the passage it cites. The model is a 4B in
  4// a container with the GPU attached, started on demand, so asking nothing costs
  5// nothing.
  6package main
  7
  8import (
  9	"context"
 10	"encoding/json"
 11	"flag"
 12	"fmt"
 13	"html/template"
 14	"log/slog"
 15	"net/http"
 16	"os"
 17	"strings"
 18	"time"
 19
 20	"search.bythewood.me/web"
 21)
 22
 23const (
 24	listenAddr = ":8000"
 25
 26	// The public repository this site lives in. Every gated site here links to
 27	// its own source from its landing page.
 28	sourceURL = "https://github.com/overshard/orchard/tree/main/sites/search.bythewood.me"
 29
 30	// Must match the first hostname label, like every other source label.
 31	selfSource = "search"
 32
 33	// analyticsID is in the page source of every site; it is identity, not a
 34	// credential. Its own property, since a copied one silently files this
 35	// site's traffic under whichever site it was copied from.
 36	analyticsID = "5782ea95-0169-4095-b94a-0b2b420440ed"
 37)
 38
 39type site struct {
 40	engine   *Engine
 41	store    *Store
 42	hist     *History
 43	llm      *LLM
 44	sessions *Sessions
 45	budget   *Budget
 46	queue    *Queue
 47	assets   *Assets
 48	auth     *web.Authenticator
 49
 50	// devOpen skips the auth check so the UI can be worked on without a
 51	// session. It is gated on Reloaded, which is false in any build made with
 52	// -tags embed, so the shipped image cannot turn this on however the
 53	// environment is set.
 54	devOpen bool
 55}
 56
 57func devOpen() bool {
 58	if !Reloaded {
 59		return false
 60	}
 61	if os.Getenv("SEARCH_DEV_NOAUTH") == "" {
 62		return false
 63	}
 64	slog.Warn("auth is bypassed, this is a development build only")
 65	return true
 66}
 67
 68func env(key, fallback string) string {
 69	if v := os.Getenv(key); v != "" {
 70		return v
 71	}
 72	return fallback
 73}
 74
 75func main() {
 76	healthcheck := flag.Bool("healthcheck", false, "probe a running server on this host and exit")
 77	flag.Parse()
 78
 79	web.SetupLogging()
 80
 81	if *healthcheck {
 82		if err := web.HealthCheck("http://127.0.0.1:8000/healthz", 3*time.Second); err != nil {
 83			slog.Info(fmt.Sprintf("healthcheck: %v", err))
 84			os.Exit(1)
 85		}
 86		return
 87	}
 88
 89	// Tees stdout records to logging.bythewood.me; see web/shipper.go. It goes
 90	// after the healthcheck branch so a HEALTHCHECK does not start a queue it
 91	// will never flush.
 92	shipper := web.ShipLogs(selfSource, web.HTTPSink())
 93	defer shipper.Close()
 94
 95	dataDir := env("SITE_DATA", "build/data")
 96	store, err := OpenStore(dataDir)
 97	if err != nil {
 98		slog.Error("startup failed", slog.Any("err", err))
 99		os.Exit(1)
100	}
101	defer store.Close()
102
103	llm := NewLLM(env("LLM_URL", "http://orchard-llm:8000"), os.Getenv("LLM_KEY"))
104
105	budget := NewBudget()
106	hist, err := OpenHistory(dataDir)
107	if err != nil {
108		slog.Error("history open failed", slog.Any("err", err))
109		os.Exit(1)
110	}
111	defer hist.Close()
112
113	s := &site{
114		queue:    NewQueue(),
115		hist:     hist,
116		assets:   NewAssets(assets()),
117		auth:     web.NewAuthenticator(),
118		devOpen:  devOpen(),
119		budget:   budget,
120		engine:   NewEngine(store, llm, budget),
121		store:    store,
122		llm:      llm,
123		sessions: NewSessions(),
124	}
125
126	// Everything here is behind auth. This site spends Isaac's GPU and searches
127	// from his address on every question, so an open one would be a stranger's
128	// search engine running on his hardware.
129	if _, err := s.loadTemplates(); err != nil {
130		slog.Error("startup failed", slog.Any("err", err))
131		os.Exit(1)
132	}
133
134	mux := http.NewServeMux()
135	// The root is public and explains what this is, the way every other gated
136	// site here does. The tool itself is behind auth, because it spends a GPU
137	// and searches from Isaac's address on every question.
138	mux.HandleFunc("GET /{$}", s.landing)
139	mux.HandleFunc("GET /search", s.gate(s.app))
140	mux.HandleFunc("GET /stream", s.gate(s.ask))
141	mux.HandleFunc("POST /reset", s.gateJSON(s.reset))
142	mux.HandleFunc("GET /budget", s.gateJSON(s.budgetState))
143
144	// The history of what was asked, which is the one thing the cache does not
145	// hold and the only page here that can delete anything.
146	mux.HandleFunc("GET /history", s.gate(s.historyPage))
147	mux.HandleFunc("POST /rate", s.gateJSON(s.rate))
148	mux.HandleFunc("POST /forget", s.gateJSON(s.forget))
149
150	// Signing in happens on auth.bythewood.me. This stays so an old bookmark
151	// or a typed /login still lands somewhere sensible.
152	mux.HandleFunc("GET /login", func(w http.ResponseWriter, r *http.Request) {
153		http.Redirect(w, r, web.LoginURL(r), http.StatusSeeOther)
154	})
155
156	// Unauthenticated on purpose: the health strip on dash probes this over the
157	// bridge, and Caddy refuses it from outside.
158	mux.HandleFunc("GET /healthz", s.healthz)
159
160	mux.Handle("GET /static/", s.assets.Handler())
161
162	slog.Info("search serving",
163		slog.String("addr", listenAddr),
164		slog.String("llm", llm.BaseURL),
165		slog.Bool("assets_from_disk", Reloaded))
166	// Recovered is outermost so a panic in the pipeline becomes a 500 rather
167	// than taking the process and every question in flight with it.
168	handler := web.Chain(mux, web.Recovered, web.Logged)
169
170	if err := web.Serve(listenAddr, handler); err != nil {
171		slog.Error("server stopped", slog.Any("err", err))
172		os.Exit(1)
173	}
174}
175
176// loadTemplates parses from whichever source this build uses. In development
177// that is the disk, so a template edit shows on reload rather than at the next
178// rebuild.
179func (s *site) loadTemplates() (*template.Template, error) {
180	return template.New("").Funcs(template.FuncMap{
181		"hostname": hostname,
182		"asset":    s.assets.URL,
183		"num":      formatNum,
184		"reason":   reasonText,
185	}).ParseFS(assets(), "templates/*.html")
186}
187
188func (s *site) render(w http.ResponseWriter, name string, data any) {
189	tmpl, err := s.loadTemplates()
190	if err != nil {
191		slog.Error("template parse failed", slog.Any("err", err))
192		http.Error(w, "template error", http.StatusInternalServerError)
193		return
194	}
195	w.Header().Set("Content-Type", "text/html; charset=utf-8")
196	if err := tmpl.ExecuteTemplate(w, name, data); err != nil {
197		slog.Error("render failed", slog.Any("err", err))
198	}
199}
200
201func (s *site) gate(next http.HandlerFunc) http.HandlerFunc {
202	if s.devOpen {
203		return next
204	}
205	return s.auth.RequireAuth(next)
206}
207
208func (s *site) gateJSON(next http.HandlerFunc) http.HandlerFunc {
209	if s.devOpen {
210		return next
211	}
212	return s.auth.RequireAuthJSON(next)
213}
214
215// landing is what a signed out visitor sees. It says what this is and where the
216// source is, and nothing about what has been asked.
217func (s *site) landing(w http.ResponseWriter, r *http.Request) {
218	pages, chunks, sites := s.store.Stats()
219	s.render(w, "landing.html", map[string]any{
220		"Pages":         pages,
221		"Chunks":        chunks,
222		"Sites":         sites,
223		"SourceURL":     sourceURL,
224		"Authenticated": s.devOpen || s.auth.Authenticated(r),
225		"Analytics":     !Reloaded,
226		"AnalyticsID":   analyticsID,
227	})
228}
229
230func (s *site) app(w http.ResponseWriter, r *http.Request) {
231	pages, chunks, _ := s.store.Stats()
232	s.render(w, "app.html", map[string]any{
233		"Pages":       pages,
234		"Chunks":      chunks,
235		"LLMUp":       s.llm.Healthy(r.Context()),
236		"SessionID":   NewSessionID(),
237		"Budget":      s.budget.State(),
238		"Ambient":     AmbientFacts(),
239		"SourceURL":   sourceURL,
240		"Analytics":   !Reloaded,
241		"AnalyticsID": analyticsID,
242	})
243}
244
245func (s *site) reset(w http.ResponseWriter, r *http.Request) {
246	s.sessions.Reset(r.URL.Query().Get("sid"))
247	w.WriteHeader(http.StatusNoContent)
248}
249
250// ask streams the pipeline. Every step reports as it happens, because a
251// question can take fifteen seconds and a spinner that says nothing is the
252// difference between "working" and "broken".
253func (s *site) ask(w http.ResponseWriter, r *http.Request) {
254	question := strings.TrimSpace(r.URL.Query().Get("q"))
255	sid := r.URL.Query().Get("sid")
256	incognito := r.URL.Query().Get("incognito") == "1"
257	// chat's deep_search wants the history row skipped on every question it
258	// asks, since chat is already keeping that conversation, without claiming
259	// the turn was incognito when it was not.
260	nohistory := incognito || r.URL.Query().Get("nohistory") == "1"
261	if question == "" {
262		http.Error(w, "no question", http.StatusBadRequest)
263		return
264	}
265
266	// ResponseController rather than a type assertion for http.Flusher, because
267	// the request logger wraps the writer and an assertion would see the
268	// wrapper. It follows Unwrap down to the real one.
269	rc := http.NewResponseController(w)
270
271	// web/server.go sets no write bound for this site, and this clears any
272	// per-connection deadline anyway, so an answer that takes minutes is never
273	// cut mid-frame. It doubles as the check that this writer can be flushed.
274	if err := rc.SetWriteDeadline(time.Time{}); err != nil {
275		http.Error(w, "streaming unsupported", http.StatusInternalServerError)
276		return
277	}
278
279	w.Header().Set("Content-Type", "text/event-stream")
280	w.Header().Set("Cache-Control", "no-cache")
281	w.Header().Set("Connection", "keep-alive")
282	w.Header().Set("X-Accel-Buffering", "no")
283	rc.Flush()
284
285	send := func(event string, payload any) {
286		blob, err := json.Marshal(payload)
287		if err != nil {
288			return
289		}
290		fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event, blob)
291		rc.Flush()
292	}
293
294	// Longer than one question needs, because it now covers waiting for the
295	// people ahead as well as answering.
296	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Minute)
297	defer cancel()
298	// Every model call the pipeline makes hangs off this, so marking it here is
299	// what keeps the gateway from writing down a question this site is not
300	// writing down either.
301	if incognito {
302		ctx = WithIncognito(ctx)
303	}
304
305	// One question runs at a time, since there is one GPU and the model server
306	// holds a single slot. Waiting is shown rather than hidden.
307	release, ok := s.queue.Enter(ctx, func(q QueueState) {
308		send("queued", q)
309	})
310	if !ok {
311		return // the client went away while waiting
312	}
313	defer release()
314
315	progress := Progress(func(step, detail string) {
316		send("status", map[string]string{"step": step, "detail": detail})
317	})
318
319	history := s.sessions.History(sid)
320	if len(history) > 0 {
321		send("status", map[string]string{"step": "followup", "detail": "following on from the last answer"})
322	}
323
324	ans, err := s.engine.Run(ctx, question, history, progress)
325	if err != nil {
326		send("failed", map[string]string{"error": err.Error()})
327		return
328	}
329
330	if sid != "" {
331		s.sessions.Append(sid, Turn{Question: question, Answer: ans.Text})
332	}
333
334	// Incognito skips this row and the gateway's copy of the prompts, and
335	// nothing else. The pages fetched on the way still go in the archive, which
336	// Isaac decided is fine: it is public articles with no question attached,
337	// so nothing there reads back as what was asked. The row is what would.
338	var logged int64
339	if !nohistory {
340		id, err := s.hist.Log(ans, s.stamp())
341		if err != nil {
342			slog.Warn("history write", slog.Any("err", err))
343		}
344		logged = id
345	}
346
347	pages, chunks, _ := s.store.Stats()
348	send("answer", map[string]any{
349		"id":         logged,
350		"incognito":  incognito,
351		"budget":     s.budget.State(),
352		"question":   ans.Query,
353		"standalone": ans.Standalone,
354		"shape":      ans.Shape,
355		"skill":      ans.Skill,
356		"text":       ans.Text,
357		"html":       ans.HTML,
358		"sources":    ans.Sources,
359		"links":      ans.Links,
360		"citations":  ans.Citations,
361		"checks":     ans.Checks,
362		"deps":       ans.Deps,
363		"passages":   ans.Passages,
364		"queries":    ans.Queries,
365		"elapsed":    ans.Elapsed,
366		"warnings":   ans.Warnings,
367		"retried":    ans.Retried,
368		"support":    ans.Support,
369		"pages":      pages,
370		"chunks":     chunks,
371	})
372}
373
374// budgetState is polled by the page so the search allowance is visible before
375// someone runs into it rather than after.
376func (s *site) budgetState(w http.ResponseWriter, r *http.Request) {
377	w.Header().Set("Content-Type", "application/json")
378	json.NewEncoder(w).Encode(s.budget.State())
379}
380
381func (s *site) healthz(w http.ResponseWriter, r *http.Request) {
382	fmt.Fprintln(w, "ok")
383}
384
385func hostname(raw string) string {
386	raw = strings.TrimPrefix(strings.TrimPrefix(raw, "https://"), "http://")
387	if i := strings.IndexByte(raw, '/'); i > 0 {
388		raw = raw[:i]
389	}
390	return strings.TrimPrefix(raw, "www.")
391}
392
393// stamp says what produced an answer. The model comes off the last response
394// rather than the config, so it names the repository and quant actually loaded.
395func (s *site) stamp() Stamp {
396	return Stamp{
397		Model:    s.llm.Served(),
398		Prompts:  promptVersion(),
399		Sampling: samplingVersion(),
400		Build:    map[bool]string{true: "dev", false: "release"}[Reloaded],
401	}
402}
403
404func (s *site) historyPage(w http.ResponseWriter, r *http.Request) {
405	only := r.URL.Query().Get("only")
406	entries, err := s.hist.List(200, 0, only)
407	if err != nil {
408		slog.Error("history read", slog.Any("err", err))
409		http.Error(w, "history unavailable", http.StatusInternalServerError)
410		return
411	}
412	total, rated := s.hist.Count()
413	s.render(w, "history.html", map[string]any{
414		"Entries": entries,
415		"Only":    only,
416		"Total":   total,
417		"Rated":   rated,
418	})
419}
420
421// rate takes the thumb. The reason is a short enum rather than free text
422// because a bare thumb cannot say which step went wrong, and which step went
423// wrong is the entire value of collecting it.
424func (s *site) rate(w http.ResponseWriter, r *http.Request) {
425	var in struct {
426		ID      int64  `json:"id"`
427		Verdict int    `json:"verdict"`
428		Reason  string `json:"reason"`
429		Note    string `json:"note"`
430	}
431	if err := json.NewDecoder(r.Body).Decode(&in); err != nil || in.ID == 0 {
432		http.Error(w, "bad request", http.StatusBadRequest)
433		return
434	}
435	if !validReason(in.Reason) {
436		in.Reason = ""
437	}
438	if err := s.hist.Rate(in.ID, in.Verdict, in.Reason, truncate(in.Note, 500)); err != nil {
439		slog.Warn("rate failed", slog.Any("err", err))
440		http.Error(w, "could not save that", http.StatusInternalServerError)
441		return
442	}
443	w.Header().Set("Content-Type", "application/json")
444	fmt.Fprint(w, `{"ok":true}`)
445}
446
447// The reasons map onto the steps of the pipeline, so a month of them says
448// where to spend the effort rather than only how often it was wrong.
449// The wording matches the buttons on the answer, so a row on the history page
450// reads back as the thing that was actually clicked.
451var reasons = map[string]string{
452	"wrong":   "it is wrong",             // synthesis or validation
453	"stale":   "already happened",        // shape and planning
454	"missed":  "answered something else", // routing and shape
455	"sources": "bad sources",             // retrieval
456}
457
458func validReason(r string) bool { _, ok := reasons[r]; return ok }
459
460// reasonText is what the history page shows on a row. The stored value is the
461// key, so the wording can change without rewriting what was already collected.
462func reasonText(key string) string {
463	if text, ok := reasons[key]; ok {
464		return text
465	}
466	return key
467}
468
469func (s *site) forget(w http.ResponseWriter, r *http.Request) {
470	var in struct {
471		ID  int64 `json:"id"`
472		All bool  `json:"all"`
473	}
474	if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
475		http.Error(w, "bad request", http.StatusBadRequest)
476		return
477	}
478	var err error
479	if in.All {
480		err = s.hist.DeleteAll()
481	} else if in.ID > 0 {
482		err = s.hist.Delete(in.ID)
483	} else {
484		http.Error(w, "bad request", http.StatusBadRequest)
485		return
486	}
487	if err != nil {
488		slog.Warn("forget failed", slog.Any("err", err))
489		http.Error(w, "could not delete that", http.StatusInternalServerError)
490		return
491	}
492	w.Header().Set("Content-Type", "application/json")
493	fmt.Fprint(w, `{"ok":true}`)
494}