orchard

   1package main
   2
   3import (
   4	"context"
   5	"fmt"
   6	"log/slog"
   7	"net/http"
   8	"sort"
   9	"strings"
  10	"sync"
  11	"time"
  12
  13	"search.bythewood.me/skills"
  14)
  15
  16const (
  17	serpTTL    = 6 * time.Hour
  18	pageTTL    = 14 * 24 * time.Hour
  19	maxSources = 5
  20
  21	// Below this share of cited sentences holding up, the answer is treated as
  22	// wrong rather than imperfect, and the pipeline searches again with what it
  23	// learned. Most of the answer failing means the search found the wrong
  24	// pages, which no amount of rewriting fixes.
  25	retryBelowSupport = 0.5
  26)
  27
  28// Source is one document that made it into the answer's evidence.
  29type Source struct {
  30	N         int
  31	URL       string
  32	Title     string
  33	Site      string
  34	Published string
  35	FromCache bool
  36}
  37
  38// Passage is a numbered piece of evidence. The model is shown these IDs and
  39// never a URL, so a citation it emits always resolves back to real text.
  40type Passage struct {
  41	ID     int
  42	Source int
  43	Text   string
  44}
  45
  46// Citation records the validation verdict for one sentence of the answer.
  47type Citation struct {
  48	Sentence  string
  49	PassageID int
  50	Source    int
  51	Supported bool
  52	Checked   bool
  53	// Repaired names the passage a failing claim was re-pointed at, when a
  54	// different passage turned out to support it.
  55	Repaired int
  56	Note     string
  57}
  58
  59// Answer is the finished product.
  60type Answer struct {
  61	Query      string
  62	Standalone string
  63	Shape      Shape
  64	Text       string
  65	HTML       string
  66	Sources    []Source
  67	Passages   []Passage
  68	Citations  []Citation
  69	Queries    []string
  70	Elapsed    string
  71	Warnings   []string
  72	Retried    bool
  73	Support    float64
  74
  75	// Skill names the handler that answered, when something other than a web
  76	// search did. Empty means the normal pipeline.
  77	Skill string
  78
  79	// Links are checked URLs for the things the answer named, which the source
  80	// list does not cover: sources say where the research came from, not where
  81	// the subject lives.
  82	Links []EntityLink
  83
  84	// Checks and Deps are the code shape's own verification, since a file that
  85	// does not parse and a package that does not exist are what go wrong in an
  86	// answer somebody is going to paste.
  87	Checks []CodeCheck
  88	Deps   []Dependency
  89}
  90
  91// Progress reports pipeline steps to whoever is watching. The web handler wires
  92// it to an SSE stream; tests leave it nil.
  93type Progress func(step, detail string)
  94
  95func (p Progress) send(step, detail string) {
  96	if p != nil {
  97		p(step, detail)
  98	}
  99}
 100
 101// Engine runs the pipeline.
 102type Engine struct {
 103	store  *Store
 104	llm    *LLM
 105	client *http.Client
 106	budget *Budget
 107	skills *skills.Registry
 108}
 109
 110func NewEngine(store *Store, llm *LLM, budget *Budget) *Engine {
 111	return &Engine{
 112		store: store, llm: llm, client: newHTTPClient(),
 113		budget: budget, skills: skills.Default(),
 114	}
 115}
 116
 117func (e *Engine) skillDeps() skills.Deps {
 118	return skills.Deps{HTTP: e.client, UA: browserUA, Now: localNow}
 119}
 120
 121// skillAnswer converts a skill result into the answer the rest of the site
 122// expects. Rendering happens here rather than in the skill, since the template
 123// and its renderer belong to this package.
 124func skillAnswer(question, standalone string, r *skills.Result, start time.Time) *Answer {
 125	a := &Answer{
 126		Query: question, Standalone: standalone, Skill: r.Skill,
 127		Shape: Shape(r.Shape), Text: r.Text, HTML: renderMarkdown(r.Text),
 128		Support: 1,
 129		Elapsed: time.Since(start).Round(10 * time.Millisecond).String(),
 130	}
 131	if a.Shape == "" {
 132		a.Shape = ShapeFactual
 133	}
 134	for i, src := range r.Sources {
 135		a.Sources = append(a.Sources, Source{
 136			N: i + 1, URL: src.URL, Title: src.Title, Site: src.Site,
 137		})
 138	}
 139	return a
 140}
 141
 142// Run executes the pipeline: plan, gather, select, synthesize, validate, and
 143// retry once if the answer did not hold up.
 144func (e *Engine) Run(ctx context.Context, question string, history []Turn, pr Progress) (*Answer, error) {
 145	start := time.Now()
 146
 147	// The model loads while the search and the fetches are in flight.
 148	go e.llm.Warm(ctx)
 149
 150	// A follow-up is resolved before anything else reads it, the router
 151	// included. "odds on the match" names no event, no team and no date, so a
 152	// router shown those words can only answer none, and a follow-up could
 153	// never reach a skill however plainly it asked for one. The rewrite is what
 154	// turns it into a question a card can match.
 155	standalone := question
 156	if len(history) > 0 {
 157		pr.send("followup", "reading the previous answer")
 158		standalone = e.rewriteFollowup(ctx, question, history)
 159		standalone = keepURLs(question, standalone)
 160	}
 161
 162	// Routing is the classification a 4B is reliably good at. The skill names
 163	// are an enum in the grammar rather than an instruction in the prompt, so
 164	// the model cannot name a handler that does not exist, and every card
 165	// carries the near misses that should not fire it as well as the phrasings
 166	// that should.
 167	//
 168	// A keyword matcher used to do this and it was wrong in both directions,
 169	// claiming "what is the S&P 500" for the quote skill and missing "do i need
 170	// a jacket today" entirely. The matcher survives inside the registry as the
 171	// fallback for when the model is unreachable.
 172	pr.send("route", "working out what kind of question this is")
 173	if res, name := e.skills.Run(ctx, e.llm, standalone, e.skillDeps()); res != nil {
 174		// A skill that returns addresses found where the answer is rather than
 175		// what it says, so the rest of the pipeline runs over those pages
 176		// instead of over a search.
 177		if len(res.URLs) > 0 {
 178			pr.send("skill", "reading "+listHosts(res.URLs))
 179			a, err := e.readGiven(ctx, question, standalone, res, start, pr)
 180			if err == nil {
 181				return a, nil
 182			}
 183			// A page behind a login or a wall is common enough that falling
 184			// back to a search beats an error, and the warning says which
 185			// happened.
 186			slog.Warn("given page unreadable, searching instead", slog.Any("err", err))
 187			pr.send("skill", "could not read that page, searching instead")
 188			ans := &Answer{Query: question, Standalone: standalone}
 189			ans.Warnings = append(ans.Warnings, fmt.Sprintf(
 190				"%s could not be read, so this answers from a search rather than from the page you gave", listHosts(res.URLs)))
 191			return e.search(ctx, ans, start, pr)
 192		}
 193		pr.send("skill", "answered from "+name)
 194		return skillAnswer(question, standalone, res, start), nil
 195	}
 196
 197	return e.search(ctx, &Answer{Query: question, Standalone: standalone}, start, pr)
 198}
 199
 200// readGiven answers from the pages the question named. Everything after the
 201// gathering is the normal pipeline, so the answer is written to a contract,
 202// every sentence is checked against the passage it cites, and the page is
 203// fetched through the same cache and the same browser headers a search result
 204// is.
 205func (e *Engine) readGiven(ctx context.Context, question, standalone string, res *skills.Result, start time.Time, pr Progress) (*Answer, error) {
 206	ans := &Answer{Query: question, Standalone: standalone, Shape: ShapeSummary, Skill: res.Skill}
 207	given := make([]Result, 0, len(res.URLs))
 208	for _, u := range res.URLs {
 209		given = append(given, Result{URL: u, Title: u})
 210	}
 211	if err := e.answerFrom(ctx, ans, given, contractFor(ShapeSummary), pr); err != nil {
 212		return nil, err
 213	}
 214	ans.HTML = renderMarkdown(ans.Text)
 215	ans.Elapsed = time.Since(start).Round(100 * time.Millisecond).String()
 216	return ans, nil
 217}
 218
 219// search is the pipeline proper: plan, gather, and one self-correction round.
 220func (e *Engine) search(ctx context.Context, ans *Answer, start time.Time, pr Progress) (*Answer, error) {
 221	question, standalone := ans.Query, ans.Standalone
 222	// A warning raised before the search, such as the page the question named
 223	// being unreadable, has to survive a retry throwing this answer away.
 224	carried := append([]string(nil), ans.Warnings...)
 225
 226	pr.send("plan", "working out what to search for")
 227	plan := e.plan(ctx, standalone, "")
 228	ans.Shape = plan.Shape
 229	ans.Queries = plan.Queries
 230	contract := contractFor(plan.Shape)
 231	pr.send("plan", fmt.Sprintf("%s question, searching: %s", plan.Shape, strings.Join(plan.Queries, " / ")))
 232
 233	if err := e.round(ctx, ans, plan, contract, pr); err != nil {
 234		return nil, err
 235	}
 236
 237	// One self-correction round. A mostly-unsupported answer is evidence the
 238	// search missed, so the retry re-plans with the failure described rather
 239	// than rewording the same evidence.
 240	//
 241	// A code answer is judged on the code. Its prose is three caveats, so one
 242	// mis-cited caveat drops the support rate under the floor and would throw
 243	// away a file that parsed, and a file that did not parse is worth a second
 244	// search however well the caveats held up.
 245	var badCode bool
 246	if ans.Shape == ShapeCode {
 247		badCode = codeFailed(ans.Checks, ans.Deps)
 248	}
 249	// The support rate cannot see this one. Every sentence about a match that
 250	// was already played is true and cited and holds up, and the answer is
 251	// still not the one that was asked for, so the second search is triggered
 252	// by the dates rather than by the validator.
 253	var lookedBack bool
 254	if ans.Shape == ShapeUpcoming {
 255		lookedBack = noFutureDate(ans.Text, localNow())
 256	}
 257	if badCode || lookedBack || (ans.Shape != ShapeCode && ans.Support < retryBelowSupport && len(ans.Citations) > 0) {
 258		hint := e.failureHint(ans)
 259		if badCode {
 260			pr.send("retry", "the code did not hold up, searching again")
 261			hint = codeHint(ans.Checks, ans.Deps)
 262		} else if lookedBack {
 263			pr.send("retry", "that only found things that have already happened, searching again")
 264			hint = upcomingHint()
 265		} else {
 266			pr.send("retry", fmt.Sprintf("only %.0f%% of that held up, searching again", ans.Support*100))
 267		}
 268		retryPlan := e.plan(ctx, standalone, hint)
 269		// The retry keeps the first plan's shape, since the failure was in the
 270		// answer and re-classifying can only move a code question off the
 271		// contract that was right for it.
 272		retryPlan.Shape = ans.Shape
 273		retry := &Answer{Query: question, Standalone: standalone, Shape: retryPlan.Shape,
 274			Queries: retryPlan.Queries, Retried: true, Warnings: carried}
 275		if err := e.round(ctx, retry, retryPlan, contractFor(retryPlan.Shape), pr); err == nil && betterAnswer(retry, ans, badCode, lookedBack) {
 276			if badCode {
 277				retry.Warnings = append(retry.Warnings, "the first attempt was thrown away because its code did not check out")
 278			} else if lookedBack {
 279				retry.Warnings = append(retry.Warnings, "the first attempt only found things that had already happened, so it was searched again")
 280			} else {
 281				retry.Warnings = append(retry.Warnings,
 282					fmt.Sprintf("first attempt was rejected, %.0f%% of its sentences were unsupported", (1-ans.Support)*100))
 283			}
 284			ans = retry
 285		}
 286	}
 287
 288	ans.HTML = renderMarkdown(ans.Text)
 289	ans.Elapsed = time.Since(start).Round(100 * time.Millisecond).String()
 290	return ans, nil
 291}
 292
 293// round is one full attempt: gather, select, synthesize, validate, repair.
 294func (e *Engine) round(ctx context.Context, ans *Answer, plan Plan, contract Contract, pr Progress) error {
 295	pr.send("search", fmt.Sprintf("running %d search%s",
 296		len(plan.Queries), map[bool]string{true: "", false: "es"}[len(plan.Queries) == 1]))
 297	results := e.gather(ctx, plan.Queries, ans, pr)
 298	if len(results) == 0 {
 299		if len(ans.Warnings) > 0 {
 300			return fmt.Errorf("%s", ans.Warnings[0])
 301		}
 302		return fmt.Errorf("no search results for that question")
 303	}
 304	return e.answerFrom(ctx, ans, results, contract, pr)
 305}
 306
 307// answerFrom is everything after the evidence is chosen: fetch, select,
 308// synthesize, validate, repair. It does not care whether the pages came from a
 309// search or from the question.
 310func (e *Engine) answerFrom(ctx context.Context, ans *Answer, results []Result, contract Contract, pr Progress) error {
 311	pr.send("fetch", fmt.Sprintf("reading %d pages", min(len(results), maxSources*2)))
 312	sources, passages, links := e.collect(ctx, results, ans.Standalone, contract, pr)
 313	if len(passages) == 0 {
 314		return fmt.Errorf("nothing readable found for that question")
 315	}
 316	ans.Sources, ans.Passages = sources, passages
 317
 318	pr.send("write", "writing the answer")
 319	text, err := e.synthesize(ctx, ans.Standalone, passages, contract)
 320	if err != nil {
 321		return err
 322	}
 323	ans.Text = strings.TrimSpace(dropMeta(dropMissingFields(tidyCitations(text))))
 324	if contract.Shape == ShapeCode {
 325		pr.send("code", "parsing every file and looking up what it imports")
 326		// Before anything reads the answer, since a marker left in a file is
 327		// pasted into it and the fix has to happen once, at the source.
 328		ans.Text = stripCodeCitations(ans.Text)
 329		blocks := codeBlocks(ans.Text)
 330		// After the blocks are read, since that is where the file name comes
 331		// from when the model wrote it as a comment.
 332		ans.Text = liftFileComments(ans.Text)
 333		ans.Checks = checkCode(blocks)
 334		ans.Warnings = append(ans.Warnings, codeWarnings(ans.Checks)...)
 335		ans.Warnings = append(ans.Warnings, unusedConstants(blocks)...)
 336		if len(blocks) == 0 {
 337			ans.Warnings = append(ans.Warnings, "this came back with no code in it, so it is an explanation rather than something to paste")
 338		}
 339	}
 340	// Only the time sensitive shapes, since "scheduled for April 2026" in a
 341	// recipe is not a claim about now.
 342	if contract.Shape == ShapeStatus || contract.Shape == ShapeNews || contract.Shape == ShapeUpcoming {
 343		now := localNow()
 344		ans.Warnings = append(ans.Warnings, staleFutures(ans.Text, now)...)
 345		ans.Warnings = append(ans.Warnings, staleNow(ans.Text, now)...)
 346		if contract.Shape == ShapeUpcoming {
 347			ans.Warnings = append(ans.Warnings, pastOnly(ans.Text, now)...)
 348			ans.Warnings = append(ans.Warnings, sooner(ans.Text, passages, now)...)
 349			ans.Warnings = append(ans.Warnings, scheduleAge(ans.Sources, now)...)
 350		}
 351	}
 352
 353	// Linking runs alongside validation rather than after it. Neither needs the
 354	// other's result and both are several model or network calls.
 355	var wg sync.WaitGroup
 356	wg.Add(1)
 357	go func() {
 358		defer wg.Done()
 359		ans.Links = e.linkEntities(ctx, ans.Standalone, text, links)
 360	}()
 361	if contract.Shape == ShapeCode {
 362		wg.Add(1)
 363		go func() {
 364			defer wg.Done()
 365			ans.Deps = verifyDeps(ctx, e.client, codeBlocks(ans.Text))
 366		}()
 367	}
 368
 369	pr.send("check", "checking every sentence against its source")
 370	// The prose only, because a line of code is not a claim: entailing it costs
 371	// a model call that can only fail, and `rows[0]` reads as a citation.
 372	ans.Citations = e.validate(ctx, proseOnly(ans.Text), passages)
 373	ans.Support = supportRate(ans.Citations)
 374	// An answer with nothing to check renders without the verification panel,
 375	// which looks the same as an answer that had nothing wrong with it. On a
 376	// site whose claim is that every sentence is checked, silence is the
 377	// misleading option, so it says so instead.
 378	if countChecked(ans.Citations) == 0 && strings.TrimSpace(ans.Text) != "" {
 379		note := "nothing in this answer carries a citation, so none of it was checked against a source"
 380		if contract.Shape == ShapeCode {
 381			// The code itself was parsed and its packages looked up, so saying
 382			// nothing was checked would be wrong in the other direction.
 383			note = "the writing around the code carries no citation, so only the code itself was checked"
 384		}
 385		ans.Warnings = append(ans.Warnings, note)
 386	}
 387
 388	wg.Wait()
 389	ans.Warnings = append(ans.Warnings, depWarnings(ans.Deps)...)
 390	if len(ans.Links) > 0 {
 391		pr.send("check", fmt.Sprintf("found %d verified link%s", len(ans.Links),
 392			map[bool]string{true: "", false: "s"}[len(ans.Links) == 1]))
 393	}
 394	return nil
 395}
 396
 397// betterAnswer decides whether the second attempt replaces the first. For code
 398// that is whether it fixed what was broken, and for everything else whether
 399// more of it held up.
 400func betterAnswer(retry, first *Answer, wasCode, lookedBack bool) bool {
 401	switch {
 402	case wasCode:
 403		return !codeFailed(retry.Checks, retry.Deps) && len(retry.Checks) > 0
 404	case lookedBack:
 405		return !noFutureDate(retry.Text, localNow())
 406	}
 407	return retry.Support > first.Support
 408}
 409
 410func countChecked(cs []Citation) int {
 411	n := 0
 412	for _, c := range cs {
 413		if c.Checked {
 414			n++
 415		}
 416	}
 417	return n
 418}
 419
 420func supportRate(cs []Citation) float64 {
 421	checked, ok := 0, 0
 422	for _, c := range cs {
 423		if !c.Checked {
 424			continue
 425		}
 426		checked++
 427		if c.Supported {
 428			ok++
 429		}
 430	}
 431	// Nothing checked is not the same as nothing wrong, but the caller warns
 432	// about that case rather than reading a rate that has no denominator.
 433	if checked == 0 {
 434		return 1
 435	}
 436	return float64(ok) / float64(checked)
 437}
 438
 439// failureHint describes what went wrong so the second plan does not repeat the
 440// first one's mistake.
 441func (e *Engine) failureHint(ans *Answer) string {
 442	var bad []string
 443	for _, c := range ans.Citations {
 444		if c.Checked && !c.Supported {
 445			bad = append(bad, stripCitations(c.Sentence))
 446		}
 447		if len(bad) >= 3 {
 448			break
 449		}
 450	}
 451	return fmt.Sprintf(
 452		"A previous search used the queries %q and the pages it found did not support the answer. "+
 453			"These claims could not be backed by anything found: %s. "+
 454			"Write different queries that would find pages actually stating the answer.",
 455		strings.Join(ans.Queries, ", "), strings.Join(bad, " | "))
 456}
 457
 458// Plan is the output of the planning step.
 459type Plan struct {
 460	Queries []string `json:"queries"`
 461	Shape   Shape    `json:"shape"`
 462}
 463
 464func (e *Engine) plan(ctx context.Context, question, hint string) Plan {
 465	schema := map[string]any{
 466		"type": "object",
 467		"properties": map[string]any{
 468			"queries": map[string]any{
 469				"type": "array", "items": map[string]any{"type": "string"},
 470				"minItems": 1, "maxItems": 3,
 471			},
 472			"shape": map[string]any{
 473				"type": "string",
 474				"enum": shapeNames(),
 475			},
 476		},
 477		"required":             []string{"queries", "shape"},
 478		"additionalProperties": false,
 479	}
 480	system := strings.Join([]string{
 481		AmbientContext(),
 482		"You plan a web search. Return short keyword queries that would find pages answering the question, and the shape of answer it wants.",
 483		"recipe: the user wants something they can cook from. comparison: two or more options weighed.",
 484		"code: the user wants something they can paste into a file and run, such as a script, a program, a config file, a query or a command. Anything asking to write, build or give code is this. So is a question about how or where to do something on a computer whose answer is a command, a query or a config, even when it is worded as the best way to do it, because what that reader wants is the command.",
 485		"Write code queries naming the language, the library, the platform and the version, and prefer official documentation over roundups.",
 486		"howto: ordered steps a person carries out in an interface or on hardware, where nothing gets typed into a file.",
 487		"news: something that already happened, including sports results and recent events.",
 488		"upcoming: the user wants the date or time of something that has not happened yet, such as the next match, a launch, a release or when a thing starts. Anything asking when the next one is, or what is coming up, is this rather than news.",
 489		"factual: everything else.",
 490		"For news, write queries that would find what happened, using words like result, final score, or the current month and year, not words like schedule, fixtures or upcoming.",
 491		"For upcoming, write the opposite: queries carrying schedule, fixtures, upcoming or next along with the current month and year, since a query about a result finds the match before the one being asked about.",
 492		"A team plays in more than one competition, so one of those queries asks for the next fixture in any competition rather than naming a league, since a league schedule leaves the cup out and calls its own next match the next match.",
 493		"status: the user wants to know where an ongoing thing stands now, such as a court case, an investigation or a rollout. Write queries carrying the current month and year so the newest coverage is found rather than the first report.",
 494		"No sentences, no quotes, no search operators.",
 495	}, " ")
 496	user := question
 497	if hint != "" {
 498		user = question + "\n\n" + hint
 499	}
 500
 501	var out Plan
 502	if err := e.llm.Structured(ctx, system, user, 250, schema, &out); err != nil || len(out.Queries) == 0 {
 503		slog.Warn("plan failed, falling back", slog.Any("err", err))
 504		return Plan{Queries: []string{question}, Shape: guessShape(question)}
 505	}
 506	for i, q := range out.Queries {
 507		out.Queries[i] = strings.TrimSpace(q)
 508	}
 509	if out.Shape == "" {
 510		out.Shape = guessShape(question)
 511	}
 512	return out
 513}
 514
 515// rewriteFollowup turns "what about the vegetarian version" into a question
 516// that stands on its own, because the search engine has no conversation.
 517func (e *Engine) rewriteFollowup(ctx context.Context, question string, history []Turn) string {
 518	schema := map[string]any{
 519		"type":                 "object",
 520		"properties":           map[string]any{"question": map[string]any{"type": "string"}},
 521		"required":             []string{"question"},
 522		"additionalProperties": false,
 523	}
 524	var b strings.Builder
 525	for _, t := range history {
 526		fmt.Fprintf(&b, "Q: %s\nA: %s\n\n", t.Question, truncate(t.Answer, 700))
 527	}
 528	var out struct {
 529		Question string `json:"question"`
 530	}
 531	err := e.llm.Structured(ctx,
 532		AmbientContext()+" Rewrite the user's follow-up into one standalone question that carries over whatever it refers to from the conversation. Keep it short. If it already stands alone, return it unchanged.",
 533		fmt.Sprintf("Conversation so far:\n\n%sFollow-up: %s", b.String(), question),
 534		200, schema, &out)
 535	if err != nil || strings.TrimSpace(out.Question) == "" {
 536		return question
 537	}
 538	return strings.TrimSpace(out.Question)
 539}
 540
 541func (e *Engine) gather(ctx context.Context, queries []string, ans *Answer, pr Progress) []Result {
 542	var all []Result
 543	seen := map[string]bool{}
 544	for _, q := range queries {
 545		results := e.store.CachedSERP(q, serpTTL)
 546		if results == nil {
 547			// A cached query costs nothing, so the budget only guards the ones
 548			// that actually leave the house.
 549			if st := e.budget.State(); st.Left == 0 {
 550				ans.Warnings = append(ans.Warnings, fmt.Sprintf(
 551					"search allowance is spent, skipped %q (room opens in about %ds)", q, st.ResetIn))
 552				continue
 553			}
 554			// Only uncached queries pace, so a repeat question still returns at
 555			// once. The wait is reported because a silent pause looks like a
 556			// hang, and this one is deliberate.
 557			if waited := e.budget.Wait(ctx); waited > 500*time.Millisecond {
 558				pr.send("search", fmt.Sprintf("paused %.1fs between searches", waited.Seconds()))
 559			}
 560			if ctx.Err() != nil {
 561				return all
 562			}
 563			e.budget.Spend()
 564			var err error
 565			results, err = SearchDDG(e.client, q, 8)
 566			if err != nil {
 567				if err == errRateLimited {
 568					e.budget.Limited()
 569				}
 570				ans.Warnings = append(ans.Warnings, fmt.Sprintf("search %q: %v", q, err))
 571				continue
 572			}
 573			if err := e.store.PutSERP(q, results); err != nil {
 574				slog.Warn("serp cache write", slog.Any("err", err))
 575			}
 576		}
 577		for _, r := range results {
 578			if !seen[r.URL] {
 579				seen[r.URL] = true
 580				all = append(all, r)
 581			}
 582		}
 583	}
 584	return all
 585}
 586
 587// collect fetches pages in parallel, then picks the chunks that answer the
 588// question rather than the ones that happen to come first.
 589func (e *Engine) collect(ctx context.Context, results []Result, question string, contract Contract, pr Progress) ([]Source, []Passage, []Link) {
 590	if len(results) > maxSources*2 {
 591		results = results[:maxSources*2]
 592	}
 593
 594	type fetched struct {
 595		idx    int
 596		page   *Page
 597		id     int64
 598		cached bool
 599	}
 600	var (
 601		mu   sync.Mutex
 602		got  []fetched
 603		wg   sync.WaitGroup
 604		sema = make(chan struct{}, 5)
 605		done int
 606	)
 607	for i, r := range results {
 608		wg.Add(1)
 609		go func(i int, r Result) {
 610			defer wg.Done()
 611			sema <- struct{}{}
 612			defer func() { <-sema }()
 613
 614			var (
 615				p      *Page
 616				id     int64
 617				cached bool
 618			)
 619			if hit := e.store.CachedPage(r.URL, pageTTL); hit != nil {
 620				p, id, cached = hit, e.store.PageID(r.URL), true
 621			} else {
 622				var err error
 623				p, err = Fetch(e.client, r.URL)
 624				if err != nil {
 625					slog.Debug("fetch skipped", slog.String("url", r.URL), slog.Any("err", err))
 626					return
 627				}
 628				if id, err = e.store.PutPage(p); err != nil {
 629					slog.Warn("page cache write", slog.Any("err", err))
 630				}
 631			}
 632			mu.Lock()
 633			got = append(got, fetched{i, p, id, cached})
 634			done++
 635			pr.send("fetch", fmt.Sprintf("read %s", hostname(r.URL)))
 636			mu.Unlock()
 637		}(i, r)
 638	}
 639	wg.Wait()
 640
 641	// Search order normally, but two shapes have a better ordering than the
 642	// search engine's.
 643	switch {
 644	// A status question is answered by whichever source is newest, so a dated
 645	// page outranks a well ranked stale one.
 646	// A schedule is the same, since an old page lists a fixture that has since
 647	// moved and a new one lists the one being asked about.
 648	case contract.Shape == ShapeStatus || contract.Shape == ShapeUpcoming:
 649		sort.SliceStable(got, func(a, b int) bool {
 650			return got[a].page.Published > got[b].page.Published
 651		})
 652	// "breakfast burrito recipe" is the exact phrase every roundup is written
 653	// to rank for, so the first page of results is listicles naming twelve
 654	// recipes and giving the quantities for none. A page publishing a
 655	// schema.org Recipe is an actual recipe, and it is the only place the
 656	// quantities exist, so it goes first whatever the search engine thought.
 657	// A page with no code on it cannot show how the code is written, whatever
 658	// it ranks for, and the first page of results for anything with a language
 659	// name in it is content marketing.
 660	case contract.Shape == ShapeCode:
 661		sort.SliceStable(got, func(a, b int) bool {
 662			ca, cb := codeWeight(got[a].page), codeWeight(got[b].page)
 663			if ca != cb {
 664				return ca > cb
 665			}
 666			return got[a].idx < got[b].idx
 667		})
 668	case contract.Shape == ShapeRecipe:
 669		sort.SliceStable(got, func(a, b int) bool {
 670			ra, rb := hasStructuredRecipe(got[a].page), hasStructuredRecipe(got[b].page)
 671			if ra != rb {
 672				return ra
 673			}
 674			return got[a].idx < got[b].idx
 675		})
 676	default:
 677		sort.Slice(got, func(a, b int) bool { return got[a].idx < got[b].idx })
 678	}
 679	if len(got) > maxSources {
 680		got = got[:maxSources]
 681	}
 682
 683	var sources []Source
 684	var passages []Passage
 685	var links []Link
 686	for i, f := range got {
 687		n := i + 1
 688		links = append(links, e.store.PageLinks(f.id)...)
 689		sources = append(sources, Source{
 690			N: n, URL: f.page.URL, Title: f.page.Title,
 691			Site: f.page.Site, Published: f.page.Published, FromCache: f.cached,
 692		})
 693
 694		// Relevance first, document order as the fallback when the question's
 695		// words do not appear (which happens on pages that answer it anyway).
 696		// A summary is the exception: it follows the page from the top, and
 697		// ranking it against "summary of this" plus an address would shuffle
 698		// the page into the order of words that are not about anything.
 699		var chunks []string
 700		if contract.Shape == ShapeSummary {
 701			chunks = e.store.PageChunks(f.id, contract.PerSource)
 702		} else if chunks = e.store.RankPassages(f.id, question, contract.PerSource); len(chunks) == 0 {
 703			chunks = e.store.PageChunks(f.id, contract.PerSource)
 704		}
 705		for _, text := range chunks {
 706			passages = append(passages, Passage{ID: len(passages) + 1, Source: n, Text: text})
 707		}
 708	}
 709	if len(passages) > contract.MaxPassages {
 710		passages = passages[:contract.MaxPassages]
 711	}
 712	return sources, passages, links
 713}
 714
 715func (e *Engine) synthesize(ctx context.Context, question string, passages []Passage, contract Contract) (string, error) {
 716	var b strings.Builder
 717	for _, p := range passages {
 718		fmt.Fprintf(&b, "[%d] %s\n\n", p.ID, truncate(p.Text, 1800))
 719	}
 720	system := strings.Join([]string{
 721		AmbientContext(),
 722		"You answer using only the numbered passages provided.",
 723		"Cite every factual sentence with the passage number it came from, like [3]. A sentence may carry more than one.",
 724		"Never state anything the passages do not support, and never fill a gap from your own knowledge.",
 725		"If the passages do not answer the question that was asked, say so in the first sentence and then say what they do cover. Do not answer a nearby question instead.",
 726		"Never put a citation on a statement that something is missing, unknown or not specified, and prefer leaving that line out entirely.",
 727		"Write markdown. Use **bold** for the facts worth skimming.",
 728		contract.Instruction,
 729	}, " ")
 730	user := fmt.Sprintf("Passages:\n\n%s\nQuestion: %s", b.String(), question)
 731	if contract.Reminder != "" {
 732		user += "\n\n" + contract.Reminder
 733	}
 734	return e.llm.Complete(ctx, system, user, contract.MaxTokens)
 735}
 736
 737// validate checks each cited sentence against the passage it cites, then tries
 738// to repair the ones that fail by looking for a passage that does support them.
 739func (e *Engine) validate(ctx context.Context, text string, passages []Passage) []Citation {
 740	byID := map[int]Passage{}
 741	for _, p := range passages {
 742		byID[p.ID] = p
 743	}
 744
 745	var out []Citation
 746	for _, sentence := range splitClaims(text) {
 747		ids := citedIDs(sentence)
 748		if len(ids) == 0 {
 749			continue
 750		}
 751		claim := stripCitations(sentence)
 752		for _, id := range ids {
 753			p, ok := byID[id]
 754			if !ok {
 755				out = append(out, Citation{
 756					Sentence: plainText(sentence), PassageID: id, Checked: true,
 757					Note: "cites a passage that does not exist",
 758				})
 759				continue
 760			}
 761			c := Citation{Sentence: plainText(sentence), PassageID: id, Source: p.Source}
 762			supported, err := e.entails(ctx, p.Text, claim)
 763			if err == nil {
 764				c.Checked = true
 765				c.Supported = supported
 766			}
 767			if c.Checked && !c.Supported {
 768				// The claim may be true and merely mis-cited, which is a
 769				// different problem from an invented one and worth telling
 770				// apart.
 771				if alt := e.findSupport(ctx, claim, passages, id); alt > 0 {
 772					c.Supported = true
 773					c.Repaired = alt
 774					c.Note = fmt.Sprintf("cited [%d] but [%d] is what supports it", id, alt)
 775				} else {
 776					c.Note = "no fetched passage states this"
 777				}
 778			}
 779			out = append(out, c)
 780		}
 781	}
 782	// An answer that cited nothing at all falls straight through the loop
 783	// above and renders as an answer nobody checked, which on this site reads
 784	// the same as an answer nothing was wrong with. A short answer is where
 785	// this happens, since one sentence carries every citation the model was
 786	// going to write and sometimes it writes none, so each claim gets a pass
 787	// through findSupport instead. The note says the citation was found rather
 788	// than written, because the difference matters.
 789	if len(out) == 0 {
 790		for _, sentence := range splitClaims(text) {
 791			claim := stripCitations(sentence)
 792			if len(contentWords(claim)) < 3 {
 793				continue
 794			}
 795			c := Citation{Sentence: plainText(sentence), Checked: true}
 796			if id := e.findSupport(ctx, claim, passages, 0); id > 0 {
 797				c.PassageID, c.Source, c.Supported, c.Repaired = id, byID[id].Source, true, id
 798				c.Note = fmt.Sprintf("no citation was written, [%d] is what states it", id)
 799			} else {
 800				c.Note = "no citation was written and no passage states this"
 801			}
 802			out = append(out, c)
 803		}
 804	}
 805	return out
 806}
 807
 808func (e *Engine) entails(ctx context.Context, passage, claim string) (bool, error) {
 809	schema := map[string]any{
 810		"type":                 "object",
 811		"properties":           map[string]any{"supported": map[string]any{"type": "boolean"}},
 812		"required":             []string{"supported"},
 813		"additionalProperties": false,
 814	}
 815	var v struct {
 816		Supported bool `json:"supported"`
 817	}
 818	err := e.llm.Structured(ctx,
 819		strings.Join([]string{
 820			"You check whether a passage supports a claim.",
 821			"Answer true if the passage states the claim, directly implies it, or describes the same step or item in different words.",
 822			"A paraphrase is still supported. A claim naming something the passage never mentions is not.",
 823			"Answer only with the JSON field.",
 824		}, " "),
 825		fmt.Sprintf("Passage:\n%s\n\nClaim:\n%s", truncate(passage, 1800), claim),
 826		40, schema, &v)
 827	return v.Supported, err
 828}
 829
 830// findSupport looks for another passage that backs a failing claim. It checks
 831// the most textually similar ones first rather than all of them, since every
 832// check is a model call.
 833func (e *Engine) findSupport(ctx context.Context, claim string, passages []Passage, skip int) int {
 834	type scored struct {
 835		id    int
 836		score int
 837	}
 838	var cands []scored
 839	words := contentWords(claim)
 840	for _, p := range passages {
 841		if p.ID == skip {
 842			continue
 843		}
 844		cands = append(cands, scored{p.ID, overlap(words, contentWords(p.Text))})
 845	}
 846	sort.Slice(cands, func(a, b int) bool { return cands[a].score > cands[b].score })
 847
 848	byID := map[int]Passage{}
 849	for _, p := range passages {
 850		byID[p.ID] = p
 851	}
 852	for i, c := range cands {
 853		if i >= 3 || c.score == 0 {
 854			break
 855		}
 856		if ok, err := e.entails(ctx, byID[c.id].Text, claim); err == nil && ok {
 857			return c.id
 858		}
 859	}
 860	return 0
 861}
 862
 863func contentWords(s string) map[string]bool {
 864	out := map[string]bool{}
 865	for _, f := range strings.Fields(strings.ToLower(s)) {
 866		w := strings.Map(func(r rune) rune {
 867			if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
 868				return r
 869			}
 870			return -1
 871		}, f)
 872		if len(w) > 3 && !stopword[w] {
 873			out[w] = true
 874		}
 875	}
 876	return out
 877}
 878
 879func overlap(a, b map[string]bool) int {
 880	n := 0
 881	for w := range a {
 882		if b[w] {
 883			n++
 884		}
 885	}
 886	return n
 887}
 888
 889func citedIDs(s string) []int {
 890	var ids []int
 891	seen := map[int]bool{}
 892	for i := 0; i < len(s); i++ {
 893		if s[i] != '[' {
 894			continue
 895		}
 896		j := strings.IndexByte(s[i:], ']')
 897		if j < 0 {
 898			break
 899		}
 900		n, ok := 0, j > 1
 901		for _, c := range s[i+1 : i+j] {
 902			if c < '0' || c > '9' {
 903				ok = false
 904				break
 905			}
 906			n = n*10 + int(c-'0')
 907		}
 908		if ok && n > 0 && !seen[n] {
 909			seen[n] = true
 910			ids = append(ids, n)
 911		}
 912		i += j
 913	}
 914	return ids
 915}
 916
 917func stripCitations(s string) string {
 918	var b strings.Builder
 919	depth := 0
 920	for _, r := range s {
 921		switch r {
 922		case '[':
 923			depth++
 924		case ']':
 925			if depth > 0 {
 926				depth--
 927			}
 928		default:
 929			if depth == 0 {
 930				b.WriteRune(r)
 931			}
 932		}
 933	}
 934	return strings.Join(strings.Fields(b.String()), " ")
 935}
 936
 937// splitClaims breaks an answer into the units worth checking.
 938//
 939// A run of list items citing the same passage is one claim, not one per line.
 940// An ingredient list produced fifteen separate model calls that all asked the
 941// same question of the same passage, which was slow and told the reader
 942// nothing. Prose splits on sentences as before.
 943const shortItem = 80
 944
 945func splitClaims(text string) []string {
 946	var out []string
 947	var bullets []string
 948	var bulletCite string
 949
 950	flush := func() {
 951		if len(bullets) == 0 {
 952			return
 953		}
 954		out = append(out, strings.Join(bullets, "; "))
 955		bullets = nil
 956		bulletCite = ""
 957	}
 958
 959	for _, line := range strings.Split(text, "\n") {
 960		t := strings.TrimSpace(line)
 961		if t == "" {
 962			flush()
 963			continue
 964		}
 965		if item, ordered, ok := listItem(t); ok {
 966			// A numbered list is a sequence of steps and each step is its own
 967			// claim, since one wrong step in a recipe is the whole problem. A
 968			// short bulleted run is a list of things and reads as one claim.
 969			if ordered || len(item) > shortItem {
 970				flush()
 971				out = append(out, item)
 972				continue
 973			}
 974			ids := citedIDs(t)
 975			key := ""
 976			if len(ids) > 0 {
 977				key = fmt.Sprint(ids)
 978			}
 979			// A change of citation ends the run, since the group is only one
 980			// claim while every line points at the same evidence.
 981			if bulletCite != "" && key != bulletCite {
 982				flush()
 983			}
 984			bulletCite = key
 985			bullets = append(bullets, item)
 986			continue
 987		}
 988		flush()
 989		out = append(out, splitSentences(t)...)
 990	}
 991	flush()
 992
 993	var kept []string
 994	for _, c := range out {
 995		if len(strings.TrimSpace(c)) > 15 {
 996			kept = append(kept, strings.TrimSpace(c))
 997		}
 998	}
 999	return kept
1000}
1001
1002// listItem recognises a markdown bullet or numbered line, returning its text
1003// and whether the list was ordered.
1004func listItem(t string) (text string, ordered bool, ok bool) {
1005	if strings.HasPrefix(t, "- ") || strings.HasPrefix(t, "* ") || strings.HasPrefix(t, "+ ") {
1006		return strings.TrimSpace(t[2:]), false, true
1007	}
1008	for i := 0; i < len(t) && i < 3; i++ {
1009		if t[i] >= '0' && t[i] <= '9' {
1010			continue
1011		}
1012		if i > 0 && (t[i] == '.' || t[i] == ')') && i+1 < len(t) && t[i+1] == ' ' {
1013			return strings.TrimSpace(t[i+2:]), true, true
1014		}
1015		break
1016	}
1017	return "", false, false
1018}
1019
1020// splitSentences is deliberately crude. It only has to find units small enough
1021// to check individually, so a split inside an abbreviation costs nothing.
1022func splitSentences(text string) []string {
1023	var out []string
1024	var cur strings.Builder
1025	for i, r := range text {
1026		cur.WriteRune(r)
1027		if r == '.' || r == '!' || r == '?' {
1028			rest := text[i+1:]
1029			if rest == "" || rest[0] == ' ' || rest[0] == '\n' {
1030				if s := strings.TrimSpace(cur.String()); len(s) > 15 {
1031					out = append(out, s)
1032				}
1033				cur.Reset()
1034			}
1035		}
1036	}
1037	if s := strings.TrimSpace(cur.String()); len(s) > 15 {
1038		out = append(out, s)
1039	}
1040	return out
1041}
1042
1043// dropMissingFields removes lines whose whole content is that the passages did
1044// not say something. The prompt asks the model to leave them out and it writes
1045// them anyway, and "Total Time: Not specified" is noise in a recipe rather than
1046// an answer.
1047func dropMissingFields(text string) string {
1048	return strings.TrimSpace(eachProseLine(text, func(line string) (string, bool) {
1049		l := strings.ToLower(plainText(line))
1050		if strings.Contains(l, "not specified") || strings.Contains(l, "not mentioned") ||
1051			strings.Contains(l, "not provided") || strings.Contains(l, "not given") {
1052			// Only when the line is a field, not when it is a real sentence
1053			// saying the sources fall short of the question.
1054			if len(l) < 90 && strings.Count(l, " ") < 12 {
1055				return "", false
1056			}
1057		}
1058		return line, true
1059	}))
1060}
1061
1062// plainText strips the markdown emphasis so a claim reads as a sentence in the
1063// validation list rather than as source.
1064func plainText(s string) string {
1065	s = strings.ReplaceAll(s, "**", "")
1066	s = strings.ReplaceAll(s, "`", "")
1067	s = strings.ReplaceAll(s, "__", "")
1068	return strings.Join(strings.Fields(s), " ")
1069}
1070
1071func truncate(s string, n int) string {
1072	if len(s) <= n {
1073		return s
1074	}
1075	return s[:n] + "..."
1076}
1077
1078// keepURLs puts back any address the follow-up rewrite dropped. The rewrite is
1079// a model call and an address is the one thing in a question that cannot
1080// survive being paraphrased, since a character of it changes what gets read.
1081func keepURLs(question, standalone string) string {
1082	for _, u := range skills.URLsIn(question) {
1083		if !strings.Contains(standalone, u) {
1084			standalone += " " + u
1085		}
1086	}
1087	return standalone
1088}
1089
1090// listHosts names the pages being read the way a person would say them.
1091func listHosts(urls []string) string {
1092	out := make([]string, 0, len(urls))
1093	for _, u := range urls {
1094		out = append(out, hostname(u))
1095	}
1096	return strings.Join(out, ", ")
1097}