orchard

  1package main
  2
  3import (
  4	"context"
  5	"encoding/json"
  6	"regexp"
  7	"strings"
  8
  9	"chat.bythewood.me/tools"
 10)
 11
 12// What stops a turn ending on "I could look that up for you".
 13//
 14// The turn loop breaks the moment the model replies without a tool call, which
 15// treats every reply as an answer. Two of them are not: a reply that offers to
 16// go and check, and a reply that states facts about the world nothing in the
 17// turn checked. Both read as an answer to the loop and neither is one, so the
 18// model gets one more pass with the tools still on the table rather than the
 19// turn ending there.
 20
 21// A deferral in the shapes a small model actually writes them. It is checked
 22// first because it is free and catches the common case without a model call.
 23//
 24// Every pattern needs a first person subject, since "you can search for it on
 25// their site" is advice and not a deferral, and an answer that mentions
 26// searching in passing must not be thrown away.
 27var deferrals = []*regexp.Regexp{
 28	regexp.MustCompile(`(?i)\b(i|we)\s+(can|could|will|shall|am able to)\s+(now\s+)?(go\s+)?(and\s+)?(search|look|check|find|fetch|pull|dig|research|browse|see)\b`),
 29	regexp.MustCompile(`(?i)\b(let me|i'?ll|i will|i'?m going to|i am going to)\s+(go\s+)?(and\s+)?(search|look|check|find|fetch|pull|dig|research|browse|see|grab|get)\b`),
 30	regexp.MustCompile(`(?i)\bwould you like me to\b`),
 31	regexp.MustCompile(`(?i)\bdo you want me to\b`),
 32	regexp.MustCompile(`(?i)\bshall i\b`),
 33	regexp.MustCompile(`(?i)\bif you'?d? (like|want)\b.{0,40}\b(search|look|check|find)\b`),
 34	regexp.MustCompile(`(?i)\bjust (say|let me know|tell me)\b.{0,30}\b(and|so)? ?i'?ll\b`),
 35	regexp.MustCompile(`(?i)\bi (do not|don'?t) have (access to|real ?time|current|up ?to ?date|live)\b`),
 36	regexp.MustCompile(`(?i)\bmy (training data|knowledge) (only )?(goes|extends|cuts off|ends)\b`),
 37	regexp.MustCompile(`(?i)\b(i|we) (do not|don'?t) have (anything|any information|any details|much) (to report|on that|about that)\b`),
 38}
 39
 40// Where a deferral has to start to count. A reply that puts the work off says
 41// so at the top, and one that answers and then offers to dig further has
 42// answered. Without the position an answer ending on "if you want I can check
 43// the other two" is thrown away and the whole turn is spent again.
 44const deferralHead = 240
 45
 46// The length past which even an early hedge is not a deferral, since a model
 47// that opened with one and then wrote two thousand characters did the work.
 48const deferralMax = 1200
 49
 50// isDeferral reports whether a reply puts the work off rather than doing it.
 51func isDeferral(reply string) bool {
 52	s := strings.TrimSpace(reply)
 53	if s == "" || len(s) > deferralMax {
 54		return false
 55	}
 56	for _, re := range deferrals {
 57		if m := re.FindStringIndex(s); m != nil && m[0] < deferralHead {
 58			return true
 59		}
 60	}
 61	return false
 62}
 63
 64// A refusal is not a deferral and the head and length rules above do not reach
 65// it. This one is matched anywhere in a reply of any length, which is only safe
 66// because it says outright that the reply is not answering the question.
 67var refusal = regexp.MustCompile(`(?i)\b(i|we) (cannot|can'?t|could not|couldn'?t) answer (this|that|it|your question)\b.{0,60}\bfrom (the )?tool results\b`)
 68
 69// The tool results die with the turn and only the answers survive, so a model
 70// reading its own earlier answer treats it as evidence it still holds and
 71// writes it out again. That is not a deferral and the patterns above miss it.
 72//
 73// Six word shingles against every earlier answer put the rehashes already in
 74// this site's history at 0.34, 0.40 and 0.59, and every real follow-up at 0.08.
 75const (
 76	repeatShingle = 6
 77	repeatOverlap = 0.30
 78	// Below this a draft is too short for the overlap to mean anything. A one
 79	// line answer to "why?" shares its whole vocabulary with what came before.
 80	repeatMinShingles = 20
 81)
 82
 83var wordish = regexp.MustCompile(`[a-z0-9]+`)
 84
 85// repeatsAnswered reports whether most of the draft is already in what this
 86// conversation has answered.
 87func repeatsAnswered(draft string, previous []string) bool {
 88	d := shingles(draft)
 89	if len(d) < repeatMinShingles {
 90		return false
 91	}
 92	seen := map[string]bool{}
 93	for _, p := range previous {
 94		for s := range shingles(p) {
 95			seen[s] = true
 96		}
 97	}
 98	if len(seen) == 0 {
 99		return false
100	}
101	hit := 0
102	for s := range d {
103		if seen[s] {
104			hit++
105		}
106	}
107	return float64(hit)/float64(len(d)) >= repeatOverlap
108}
109
110func shingles(s string) map[string]bool {
111	w := wordish.FindAllString(strings.ToLower(s), -1)
112	out := make(map[string]bool, len(w))
113	for i := 0; i+repeatShingle <= len(w); i++ {
114		out[strings.Join(w[i:i+repeatShingle], " ")] = true
115	}
116	return out
117}
118
119// The gate. It runs when the model stops calling tools and wants to answer,
120// which is the only moment where both "is this actually an answer" and "is
121// there enough behind it" can be asked at once.
122//
123// Two fields and one of them an enum, because a 4B handed a free reasoning
124// field beside a constrained one writes the reasoning and then contradicts it.
125// The query is what makes a verdict actionable: a step that says more research
126// is needed without saying what to look for sends the model back to the search
127// it already ran.
128type verdict struct {
129	Verdict string `json:"verdict"`
130	Query   string `json:"query"`
131}
132
133var verdictSchema = map[string]any{
134	"type": "object",
135	"properties": map[string]any{
136		"verdict": map[string]any{
137			"type": "string",
138			"enum": []string{"answered", "research"},
139		},
140		"query": map[string]any{"type": "string"},
141	},
142	"required":             []string{"verdict", "query"},
143	"additionalProperties": false,
144}
145
146const gateSystem = `You are checking a draft answer before it is sent. Answer only with the JSON object you were given a schema for.
147
148"answered" means the draft answers the question with real specifics and every fact in it either came from the tool results, or was established earlier in this conversation, or is something no tool could change, like arithmetic, code, or an opinion asked for. A draft that disagrees with the background section is never "answered", whatever else is true of it.
149
150"research" means the draft is not ready to send. Choose it when any of these are true:
151- A background section is present and the draft disagrees with it. Compare them claim by claim before anything else. The background is the opening section of a Wikipedia article, so on a definition, a name, a date, an origin or who did something it is right and a draft that says otherwise is wrong, however confident the draft sounds. Two limits: the background does not cover everything, so a fact it is simply silent on is not a disagreement, and it was taken on the date it states, so a difference about something that could have changed since then means the background is old rather than the draft wrong, and that is "answered".
152- The draft offers to look something up, says it will check, or asks whether it should.
153- The draft says it does not know, has no access, has nothing to report, or cannot answer from what it has.
154- The draft states a name, title, date, number, price or event that no tool result above supports.
155- The question asks about something current, local, priced, scheduled or newsworthy and no tool was called.
156- The draft answers part of the question and leaves the rest.
157- The draft repeats an earlier answer in this conversation instead of addressing what the new question adds to it.
158- The draft quotes a page or credits a figure to a source that is not in the tool results above. Pages read in earlier turns are gone and cannot be quoted from memory.
159
160When the verdict is "research", query is the single web search that would close the biggest gap, written as a person would type it. When the verdict is "answered", query is an empty string.`
161
162// Whether the question needs something the weights cannot hold. This is asked
163// of the question and never of the draft, because a draft that invents an
164// answer reads exactly like one that knows it, and the eight rule check below
165// let "no big news today" through on a Monday afternoon for that reason.
166type freshness struct {
167	NeedsFresh bool   `json:"needs_fresh"`
168	Query      string `json:"query"`
169}
170
171var freshnessSchema = map[string]any{
172	"type": "object",
173	"properties": map[string]any{
174		"needs_fresh": map[string]any{"type": "boolean"},
175		"query":       map[string]any{"type": "string"},
176	},
177	"required":             []string{"needs_fresh", "query"},
178	"additionalProperties": false,
179}
180
181// One question rather than the eight the draft check weighs at once, since a 9B
182// asked for a single judgement gets it right far more often. Measured over
183// thirty two of Isaac's own questions this reached sixteen of eighteen held
184// out, against a draft check that missed "any big news today?" outright.
185const freshnessSystem = `Decide whether answering the user's question correctly needs information you could not have from training alone, because it changes over time or has happened since.
186
187true when the question touches news, current events, prices, markets, scores, odds, fixtures, schedules, opening or closing, weather, or what is happening now.
188true whenever the question carries a time word like today, tonight, this weekend, yesterday, this week, right now, currently, or latest, even if the subject sounds ordinary.
189true when the question asks what has been going on with something.
190
191false when the answer is a definition, an explanation, how something works, history, code, arithmetic, or a recipe, none of which change.
192
193When needs_fresh is true, query is the single web search that would answer it, written as a person would type it. When it is false, query is an empty string.`
194
195// The second narrow question, asked of a draft that fetched nothing and passed
196// the freshness check.
197//
198// Nineteen of sixty four turns on 2026-09-08 called no tool, and the wrong ones
199// were not about anything current: they were specifics written from memory. The
200// dirty rice with 23g of protein, the Marlin 195 and the Howa 158 that are not
201// real rifles, CDX described as what Common Crawl uses under the hood. The
202// contract has said to look a subject up since it was written.
203//
204// Asked of the draft rather than the question, because the question is often
205// vague and the draft is where the invented specifics actually are.
206type grounding struct {
207	NeedsCheck bool   `json:"needs_check"`
208	Query      string `json:"query"`
209}
210
211var groundingSchema = map[string]any{
212	"type": "object",
213	"properties": map[string]any{
214		"needs_check": map[string]any{"type": "boolean"},
215		"query":       map[string]any{"type": "string"},
216	},
217	"required":             []string{"needs_check", "query"},
218	"additionalProperties": false,
219}
220
221const groundingSystem = `A draft answer was written without looking anything up. Decide whether it states specifics that ought to have been checked.
222
223true when the draft names a product, model number, part number, version, company, person, book, film or song.
224true when it gives a figure presented as fact: a price, a measurement, a nutrition number, a count, a capacity, a date.
225true when it describes what a named tool, service, format or standard does or is used for.
226
227false when the draft is explanation, reasoning, opinion, code the user asked to be written, arithmetic on figures the user supplied, or ordinary conversation.
228false when every specific in it came from what the user said in the question.
229
230When needs_check is true, query is the single web search that would check the most load bearing specific, written as a person would type it. When it is false, query is an empty string.`
231
232// needsChecking reports whether a draft written from memory states specifics.
233// A failure is a no, for the same reason every other gate fails open.
234func (e *Engine) needsChecking(ctx context.Context, draft string) (grounding, Stats) {
235	msgs := []Message{
236		{Role: RoleSystem, Content: groundingSystem},
237		{Role: RoleUser, Content: "Draft:\n" + trim(strings.TrimSpace(draft), 1800)},
238	}
239	var g grounding
240	st, err := e.llm.Structured(ctx, msgs, gateTokens, groundingSchema, &g)
241	if err != nil {
242		return grounding{}, st
243	}
244	return g, st
245}
246
247// needsFresh reports whether the question wants current information. A failure
248// is a no, for the same reason the draft check fails open: a turn that cannot
249// reach the model to ask is not a turn to send round again.
250func (e *Engine) needsFresh(ctx context.Context, question string) (freshness, Stats) {
251	msgs := []Message{
252		{Role: RoleSystem, Content: freshnessSystem},
253		{Role: RoleUser, Content: "Question: " + strings.TrimSpace(question)},
254	}
255	var f freshness
256	st, err := e.llm.Structured(ctx, msgs, gateTokens, freshnessSchema, &f)
257	if err != nil {
258		return freshness{}, st
259	}
260	return f, st
261}
262
263// enough asks whether the draft can be sent. Anything that goes wrong is a yes,
264// because a gate that fails closed would turn a working turn into a loop over a
265// model that is not answering the gate either.
266func (e *Engine) enough(ctx context.Context, question, draft, background string, results, previous []string) (verdict, Stats) {
267	var b strings.Builder
268	b.WriteString("Question:\n")
269	b.WriteString(strings.TrimSpace(question))
270	if len(previous) > 0 {
271		// Without this the gate cannot tell a fresh answer from the last one
272		// written out again, which is the whole failure on a follow-up.
273		b.WriteString("\n\nAnswers already given earlier in this conversation:\n")
274		for _, p := range previous {
275			b.WriteString("- ")
276			b.WriteString(trimLine(p, 600))
277			b.WriteByte('\n')
278		}
279	}
280	b.WriteString("\n\nTool results in this turn:\n")
281	if len(results) == 0 {
282		b.WriteString("(none, no tool was called)")
283	} else {
284		for _, r := range results {
285			b.WriteString("- ")
286			b.WriteString(trimLine(r, 600))
287			b.WriteByte('\n')
288		}
289	}
290	if background != "" {
291		b.WriteString("\n\nBackground, looked up locally rather than by the model:\n")
292		b.WriteString(background)
293		b.WriteByte('\n')
294	}
295	b.WriteString("\n\nDraft answer:\n")
296	b.WriteString(trimLine(strings.TrimSpace(draft), 2000))
297
298	msgs := []Message{
299		{Role: RoleSystem, Content: gateSystem},
300		{Role: RoleUser, Content: b.String()},
301	}
302	var v verdict
303	st, err := e.llm.Structured(ctx, msgs, gateTokens, verdictSchema, &v)
304	if err != nil || v.Verdict != "research" {
305		return verdict{Verdict: "answered"}, st
306	}
307	return v, st
308}
309
310// The nudge that goes back into the conversation. The draft itself is never
311// appended, because a model handed its own deferral writes it again.
312// Arithmetic.
313//
314// The contract has asked for calc since the tool existed and it was called zero
315// times on 2026-09-08, across a day of adding up calories. Three answers had
316// wrong sums in them, one of them contradicting a total the same conversation
317// had already given. Asking was never going to work, for the same reason it did
318// not work for tool calls generally, so a draft that adds up in prose is sent
319// back to do it with the tool.
320//
321// Deterministic and free, and deliberately not an attempt to check the sum
322// here. Working out which numbers in a sentence are the addends is the part
323// that goes wrong, and calc gets it right by construction.
324var (
325	totalWord = regexp.MustCompile(`(?i)\b(total|totals|totalling|altogether|all together|adds up to|comes to|sums? to|in total|grand total)\b`)
326	// A citation marker is a number to a regex and is not one to a reader.
327	citeNum = regexp.MustCompile(`\[\d{1,3}\]`)
328	numeral = regexp.MustCompile(`\d[\d,]*(?:\.\d+)?`)
329)
330
331// countsUpInProse is true for a draft that states a total over several numbers
332// it worked out itself. Three is the floor: two numbers and a total is usually
333// a comparison, and one is a quantity rather than a sum.
334func countsUpInProse(draft string) bool {
335	clean := citeNum.ReplaceAllString(draft, " ")
336	if !totalWord.MatchString(clean) {
337		return false
338	}
339	// Inside a fence the numbers belong to code somebody is about to run.
340	clean = strings.Join(outsideFences(clean), "\n")
341	return len(numeral.FindAllString(clean, -1)) >= 3 && totalWord.MatchString(clean)
342}
343
344// outsideFences drops fenced code, since arithmetic in an example is not a
345// claim about a total.
346func outsideFences(s string) []string {
347	var out []string
348	fenced := false
349	for _, line := range strings.Split(s, "\n") {
350		t := strings.TrimSpace(line)
351		if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") {
352			fenced = !fenced
353			continue
354		}
355		if !fenced {
356			out = append(out, line)
357		}
358	}
359	return out
360}
361
362func calcNudge() string {
363	return "You added those up yourself. Call calc with the figures and write the total it gives you, " +
364		"rather than the one you worked out. If some of the figures are missing, say which."
365}
366
367func researchNudge(query string) string {
368	q := strings.TrimSpace(query)
369	if q == "" {
370		return "That does not answer the question. Call a tool now and find out. " +
371			"Do not offer to look something up and do not say what you would do next, there is nobody to answer you."
372	}
373	return "That does not answer the question yet. Search for " + quoted(q) + " now, and keep going until you have the specifics. " +
374		"Do not offer to look something up and do not say what you would do next, there is nobody to answer you."
375}
376
377// What a draft contradicted by the snapshot is told. It names the tool and the
378// subject, since a model sent back without both goes and searches the web for
379// what is already on this machine.
380func wikiNudge(subject string) string {
381	s := strings.TrimSpace(subject)
382	if s == "" {
383		return "Part of that does not match what the offline Wikipedia says. Call wikipedia now, read it, and correct the answer."
384	}
385	return "Part of that does not match what the offline Wikipedia says about " + quoted(s) + ". " +
386		"Call wikipedia with " + quoted(s) + " now, read the article, and correct the answer rather than repeating it."
387}
388
389// What a rehash is told, which has to say why rather than just no. A model sent
390// back with the generic nudge writes the same answer a third time, since as far
391// as it can see it already has the material.
392func repeatNudge() string {
393	return "That is the answer you already gave, and it does not address what this question adds. " +
394		"The tool results from the earlier turns are gone and the pages behind them are not in front of you, so nothing there can be quoted or checked. " +
395		"Call a tool now and get what this question needs."
396}
397
398// Quoted so a multi word query is not read as part of the sentence around it.
399func quoted(s string) string { return "\"" + strings.ReplaceAll(s, "\"", "") + "\"" }
400
401// gate is the whole check, and it returns the nudge to send the turn back with
402// or an empty string to let the draft stand. The two free checks run first and
403// the model is only asked when neither fired, which is a second saved on every
404// one of the failures this exists for. previous is what this conversation has
405// already answered, most recent last.
406func (e *Engine) gate(ctx context.Context, question, draft string, previous []string, used []tools.Result, emit func(Event)) (string, Stats) {
407	// Sending a turn back to research when the search endpoint is in the
408	// penalty box is a guaranteed loop: it cannot succeed, and every pass costs
409	// a model call and another failed request against a host that is already
410	// refusing. Every nudge below that names a search is held back for that
411	// reason, and the local snapshot is not, since correcting a draft against a
412	// container on the bridge needs nothing that is refusing us.
413	_, searchDown := e.SearchDown()
414	if searchDown {
415		return e.gateOffline(ctx, question, draft, used, emit)
416	}
417	// A news rundown is finished when it arrives. The gate's own rules read a
418	// list of twenty stories as an answer that covers part of the question and
419	// leaves the rest, so left to itself it sends the turn back to research one
420	// of them and the rundown becomes a single story write up.
421	if calledNews(used) {
422		return "", Stats{}
423	}
424	// Same for a turn that was told to remember something and did. There is no
425	// question under it to research, and the gate reading it as one sent the
426	// turn off to look up the film again and answer with where to stream it,
427	// having stored nothing.
428	if calledTool(used, tools.Remember.Name) {
429		return "", Stats{}
430	}
431	// Asked of the question and before anything reads the draft, since the
432	// failure this catches is a draft that sounds like an answer. A turn that
433	// already fetched something is left alone, because the question needing
434	// current information is only a problem when nothing went and got it.
435	var st Stats
436	if len(used) == 0 {
437		f, fst := e.needsFresh(ctx, question)
438		st.merge(fst)
439		if f.NeedsFresh {
440			emit(Event{Kind: "status", Text: "looking it up"})
441			return researchNudge(f.Query), st
442		}
443	}
444	if isDeferral(draft) || refusal.MatchString(draft) {
445		emit(Event{Kind: "status", Text: "looking it up"})
446		return researchNudge(""), st
447	}
448	// Only when nothing was fetched. A turn that did the work and then restated
449	// some of what it said before has answered, and taking it away would cost
450	// the tool calls it already spent.
451	if len(used) == 0 && repeatsAnswered(draft, previous) {
452		emit(Event{Kind: "status", Text: "looking it up"})
453		return repeatNudge(), st
454	}
455	// Before the model check, since it costs nothing and the model check has
456	// never once objected to a wrong sum.
457	if !calledTool(used, tools.Calc.Name) && countsUpInProse(draft) {
458		emit(Event{Kind: "status", Text: "adding it up"})
459		return calcNudge(), st
460	}
461	// A draft written from memory that states specifics. The freshness check
462	// above only catches what changes over time, and the specifics that were
463	// wrong were mostly things that do not: a rifle that does not exist, a
464	// protein figure off by eighteen grams.
465	if len(used) == 0 {
466		g, gst := e.needsChecking(ctx, draft)
467		st.merge(gst)
468		if g.NeedsCheck {
469			emit(Event{Kind: "status", Text: "checking it"})
470			return researchNudge(g.Query), st
471		}
472	}
473	emit(Event{Kind: "status", Text: "checking the answer"})
474	// Only when the turn fetched nothing, since that is the case the gate has
475	// no evidence for. A turn that called tools already gave it something to
476	// work with, and a second opinion there would argue with what was fetched.
477	var background string
478	if len(used) == 0 {
479		background = e.background(ctx, question)
480	}
481	results := make([]string, 0, len(used))
482	for _, r := range used {
483		if r.Err != "" {
484			results = append(results, r.Name+" failed: "+r.Err)
485			continue
486		}
487		body, _ := json.Marshal(r.Content)
488		results = append(results, r.Name+": "+string(body))
489	}
490	v, est := e.enough(ctx, question, draft, background, results, previous)
491	st.merge(est)
492	if v.Verdict != "research" {
493		return "", st
494	}
495	emit(Event{Kind: "status", Text: "looking it up"})
496	// The snapshot already has the article, so sending it to a web search for
497	// something a local call answers in milliseconds is the slower way to be
498	// right.
499	if background != "" {
500		return wikiNudge(subjectOf(question)), st
501	}
502	return researchNudge(v.Query), st
503}
504
505// gateOffline is the gate with the search host refusing us. The only thing that
506// can be acted on is a draft the local snapshot disagrees with, so that is the
507// only thing checked, and anything else is let through as the honest answer the
508// turn managed.
509func (e *Engine) gateOffline(ctx context.Context, question, draft string, used []tools.Result, emit func(Event)) (string, Stats) {
510	if len(used) > 0 {
511		return "", Stats{}
512	}
513	background := e.background(ctx, question)
514	if background == "" {
515		return "", Stats{}
516	}
517	emit(Event{Kind: "status", Text: "checking the answer"})
518	v, st := e.enough(ctx, question, draft, background, nil, nil)
519	if v.Verdict != "research" {
520		return "", st
521	}
522	emit(Event{Kind: "status", Text: "looking it up"})
523	return wikiNudge(subjectOf(question)), st
524}
525
526func calledNews(used []tools.Result) bool { return calledTool(used, tools.News.Name) }
527
528func calledTool(used []tools.Result, name string) bool {
529	for _, r := range used {
530		if r.Name == name && r.Err == "" {
531			return true
532		}
533	}
534	return false
535}