orchard

  1// The conversation loop. A turn is not one big generation: the model decides
  2// what it needs, tools fetch it, and only then does it write. That is the same
  3// insight search rests on and it is what makes a 4B usable here.
  4package main
  5
  6import (
  7	"context"
  8	"encoding/json"
  9	"fmt"
 10	"sort"
 11	"strings"
 12	"time"
 13
 14	"chat.bythewood.me/tools"
 15)
 16
 17const (
 18	// Enough for a comparison that searches per thing, reads the best page, and
 19	// then goes back for whatever the gate says is missing. The repeat ledger
 20	// below is what stops a model spending these on the same call over and
 21	// over, which is why this can be more than the four it used to be.
 22	maxToolRounds = 6
 23
 24	// How many times a reply that is not an answer gets sent back. Two, because
 25	// a model that has ignored the instruction twice is not going to take it on
 26	// the third go and the turn still owes the user something.
 27	maxGates = 2
 28
 29	// Budgets. The answer gets the big one because it is the only step whose
 30	// output the user reads.
 31	// Enough for several tool calls in one round. It was 900, which a model
 32	// asked to total a bank statement spent on one arithmetic expression before
 33	// being cut off mid string.
 34	toolTurnTokens = 1600
 35	answerTokens   = 2400
 36	// The gate emits an enum and a search query and nothing else.
 37	gateTokens = 120
 38)
 39
 40// Event is what the browser is told while a turn runs.
 41//
 42// The answer arrives as `block` and `tail` rather than raw text. A block is a
 43// finished piece of markdown already rendered to HTML, and the tail is the
 44// unfinished paragraph after it, as plain text. That way the reader sees
 45// formatting appear as it is settled instead of reading plain text and then
 46// having the whole message reflow under them when the turn ends.
 47type Event struct {
 48	Kind string `json:"kind"` // status, tool, tool_done, widget, step, block, tail, done, error
 49	Text string `json:"text,omitempty"`
 50	HTML string `json:"html,omitempty"`
 51	Tool string `json:"tool,omitempty"`
 52	Args string `json:"args,omitempty"`
 53	MS   int64  `json:"ms,omitempty"`
 54	OK   bool   `json:"ok,omitempty"`
 55
 56	// The subject of a chart, sent as soon as the tool that named it returns so
 57	// the panel is drawing while the answer is still being written.
 58	Widget *tools.Widget `json:"widget,omitempty"`
 59
 60	// One entry in the record of what this turn did, sent as it happens so the
 61	// list fills in rather than appearing all at once at the end.
 62	Step *Step `json:"step,omitempty"`
 63}
 64
 65type Engine struct {
 66	// Render is the same markdown renderer the finished message uses, so what
 67	// streams in and what is stored cannot disagree.
 68	Render func(string) string
 69
 70	llm  *LLM
 71	reg  *tools.Registry
 72	deps *tools.Deps
 73	now  func() time.Time
 74	// modelName is what the model is told it is. It is the readable name
 75	// rather than llama-swap's "local" alias, which is a routing key and means
 76	// nothing to a reader.
 77	modelName string
 78	place     string
 79	tz        string
 80}
 81
 82// Deps is the shared dependency set, which the widget endpoints borrow so their
 83// calls go through the same breaker and the same spend ceiling a tool's would.
 84// They pass no session, since neither endpoint reads anything of Isaac's.
 85func (e *Engine) Deps() *tools.Deps { return e.deps }
 86
 87func NewEngine(llm *LLM, modelName string) *Engine {
 88	return &Engine{
 89		llm: llm, reg: tools.Default(), deps: tools.NewDeps(), now: time.Now,
 90		modelName: modelName,
 91		place:     "Yadkin Valley, North Carolina", tz: "America/New_York",
 92	}
 93}
 94
 95// ambient is what a person sitting here would know without being told. Without
 96// the date the model cannot tell what "this weekend" means, and without the
 97// place it answers a question about the weather as though it were nowhere.
 98//
 99// The time is stated to the hour rather than the minute on purpose: every
100// system prompt opens with this block and llama.cpp caches the prompt prefix it
101// has already processed, so a clock that ticks every minute means no turn ever
102// reuses another's work.
103func (e *Engine) ambient() string {
104	loc, err := time.LoadLocation(e.tz)
105	if err != nil {
106		loc = time.UTC
107	}
108	t := e.now().In(loc)
109	return fmt.Sprintf("Today is %s. It is around %s. The user is in %s, "+
110		"which is what to use for weather, local news and anything asking what is nearby. "+
111		"It is not a hint about what an unfamiliar name means.",
112		t.Format("Monday, 2 January 2006"), t.Format("3 PM MST"), e.place)
113}
114
115// identity is first in the prompt because a small model asked what it is will
116// otherwise answer with whatever name dominated its training data, and several
117// of them say Claude. It is a training artifact rather than a jailbreak, and
118// the only fix is telling it what it actually is.
119const identity = `You are %s, an open weights model running through llama.cpp on Isaac's own RTX 3070, in a chat application he wrote. You are not Claude, ChatGPT, Gemini, or any hosted assistant, and you were not made by Anthropic, OpenAI or Google. If you are asked what you are, say which model you are and that you run locally on his hardware. Do not claim to be anything else, and do not apologise for what you are.
120
121`
122
123const contract = `You are Isaac's assistant. He is a software engineer who self hosts everything he runs, has a family, camps and hikes, and asks direct questions and wants direct answers.
124
125Use a tool whenever the answer depends on something you cannot know from memory: anything current, local, priced, scheduled, on a page, or checkable against a real record. Do not guess a fact a tool can give you, and do not tell the user to go look it up themselves.
126
127Look a subject up before answering about it, whether it is a person, company, product, game, film, event, place, species, or a technical term or concept. That includes "who is X", "what is X" and "tell me about X". Being sure you know is not evidence, and a real name, a date or a definition you half remember is the part most likely to be wrong. Start with wikipedia, which is local and costs nothing and is there so you do not have to answer from memory, and go to web_search when it has no article or when the question is about something current.
128
129Tools:
130- Call a tool rather than describing what one would return.
131- Never offer to look something up and never ask whether you should. There is nobody to answer you, so an offer ends the turn with nothing in it. If a tool would help, call it now.
132- Never answer a question about the world from memory when a tool could check it. Your training data is old and this is what the tools are for.
133- wikipedia is an offline snapshot on this machine. It answers instantly, it cannot be rate limited, and it carries each article's opening section only, so it is the cheapest way to get the background right before deciding whether anything needs searching. It knows nothing after its snapshot date, so never use it for news, prices, scores or anything that changed recently.
134- web_search gives titles, urls and snippets. Call web_fetch on a url when you need what the page actually says.
135- news reads a fixed list of publishers and is what to call for any question about what is happening or what happened over a period, rather than searching. Pass the window the question actually used, so today means today and this weekend means the weekend just gone, and pass the topic only when one was named. It hands back each publisher's own headline for you to rewrite plainly.
136- An attached file is already in this conversation in full. There is no url or path for it, so never try to fetch one, and never guess where it might be on a disk.
137- Search once per thing you are comparing. One search rarely covers a comparison or a build.
138- Use calc for totals rather than adding in your head.
139- If a tool errors or is rate limited, say so plainly and answer with what you have. Never treat a missing tool as a reason not to answer.
140- A search that finds nothing for what you assumed the question meant, and one clear hit for something else, has told you the assumption was wrong. Take the hit and answer about that, rather than reporting that the thing you invented could not be found.
141- markets and weather draw a chart above your answer, so the reader can already see the price against its range, or the week with its rain and pollen. Say what it means rather than reading it out: the direction and why it matters, the day the rain arrives, whether the pollen is worth staying in for. Listing seven days of numbers underneath the panel that shows them is the one thing not to do.
142- remember is long term memory, kept between conversations. Call it when he asks you to remember, note or forget something, and when he states a preference, a plan or something about himself worth keeping. Saying you will remember it does not remember it, the call does. List first when you need an id to correct or drop one, and keep each fact to one plain sentence about him.
143- The orchard_ tools read Isaac's own infrastructure: his logs, uptime monitoring, analytics, git repositories and dashboard. Use them for any question about his own sites rather than guessing or searching the web, and say which one you read. They only read, so nothing you do with them can change anything.
144- orchard_code reads the source of his repositories. It is the only way to see his code, so never fetch a url for it and never write code you say you read without having read it. Walk down to the file: list the top of the repository, then the directory, then read the file.
145- chat_history searches earlier conversations. Call it when he refers to something from another chat, asks what was decided before, or when a question only makes sense against something already settled. What it returns was true when it was said, so anything dated or priced in it needs looking up again.
146- Anything you present as current has to be current. A page and a snippet carry the date they were written, and today's date is at the top of this prompt, so compare the two before you write today, now or currently. A three day old incident reported as happening now is worse than saying you could not find anything from today.
147
148Follow-ups:
149- A follow-up is a new question. What you answered before covers what it says and nothing more, so anything this question adds needs a tool call before you answer it.
150- Tool results do not survive the turn that fetched them. Your earlier answers are still here and the pages behind them are not, so never quote a page or credit a figure to a source you read in an earlier turn. Fetch it again if you need what it said.
151- When the user pushes back, corrects you, or asks why, go and look. Rewriting the answer you already gave tells him nothing he does not have, and a correction usually means the first search missed the thing he is asking about.
152- When he tells you what something is, that is now what it is. Drop your own reading of it completely, including the searches you built on it, and look the thing up under the name he gave it. Repeating the earlier answer after being told its premise was wrong is the worst thing you can do here.
153- An unfamiliar name is a name. Look it up as one before deciding it must be a place, a river or a landmark near him.
154
155Answers:
156- Not every message is a question. When he is chatting, agreeing, joking or thinking out loud, answer like a person would in a line or two and call nothing. Never tell him you do not know what he is asking.
157- Lead with the answer. No preamble, no restating the question, no closing offer of more help.
158- Never write a web address. When you are given a numbered list of sources, end the sentence with the number it came from, like [2].
159- Say plainly when you are unsure or when sources disagree. A short honest answer beats a confident wrong one.
160- Every name, title, date, number and price you write has to come from a tool result in this turn, from an answer you already gave in this conversation, or from what the user told you. Anything else needs a tool call before you write it.
161- This is the only reply the user gets, so put everything you found in it.
162- Never invent a product, a song, a part number, a price or a source. Check it or say you are not sure.
163- Follow the format and constraints asked for exactly. Given a budget, a word count or a unit, hit it and show the total.
164- Markdown for structure. Bold only for labels, never mid sentence for emphasis. The exception is a rundown of many items, like the news, where the few words carrying each item are bolded so the list can be skimmed.
165- When a tool result gives you a shape to answer in, use that shape exactly, headings and bullets and all. It was asked for.
166- No em dashes and no semicolons. Use a comma, a full stop, or the word they stand in for.`
167
168// Memory is what the engine is handed for this turn, already filtered down to
169// what the question touched. The engine does no retrieval of its own, so the
170// same turn can be run in a test with a fixed set of facts.
171func (e *Engine) systemWith(memory string) Message {
172	m := e.system()
173	m.Content += memory
174	return m
175}
176
177func (e *Engine) system() Message {
178	name := e.modelName
179	if name == "" {
180		name = "a small local model"
181	}
182	return Message{Role: RoleSystem,
183		Content: fmt.Sprintf(identity, name) + e.ambient() + "\n\n" + contract}
184}
185
186// RestoreGuard hands the guard its persistence and whatever the last process
187// left behind.
188func (e *Engine) RestoreGuard(store tools.PenaltyStore, saved map[string][2]int64) {
189	e.deps.Guard.Restore(store, saved)
190}
191
192// RestoreSpend puts back what the last process spent, so a deploy is not a
193// fresh day's allowance.
194func (e *Engine) RestoreSpend(at []time.Time) {
195	e.deps.Budgets.Restore(tools.SearchHost, at)
196}
197
198// SearchSpend is what the page shows, so the pool draining is visible before it
199// is gone.
200func (e *Engine) SearchSpend() (minute, hour, day int) {
201	m, h, d, _ := e.deps.Budgets.Left(tools.SearchHost)
202	return m, h, d
203}
204
205// SaveSpend hands the current counts back to the caller's store. It runs after
206// a turn rather than per request, since a write on the request path costs more
207// than losing one turn's counts to a hard kill.
208func (e *Engine) SaveSpend(save func(host string, at []time.Time)) {
209	save(tools.SearchHost, e.deps.Budgets.Spent(tools.SearchHost))
210}
211
212// SearchDown reports whether the search endpoint is in the penalty box and for
213// how much longer, so the page can say so before a turn discovers it.
214func (e *Engine) SearchDown() (time.Duration, bool) {
215	left, ok := e.deps.Guard.Down()[tools.SearchHost]
216	return left, ok
217}
218
219// Run drives one user turn and emits events as it goes.
220// Run drives one user turn. The session is the caller's own, forwarded to the
221// orchard tools so each site checks it rather than this one holding a
222// credential of its own.
223func (e *Engine) Run(ctx context.Context, history []Message, user, session, memory string, tr *Trace, emit func(Event)) (Message, []tools.Result, []Source, []tools.Widget, Stats, error) {
224	deps := e.deps.WithSession(session)
225	// A tool decides for itself what incognito means for it, so the flag rides
226	// on the per turn copy rather than only on this process's own requests.
227	deps.Incognito = IsIncognito(ctx)
228	// Which widgets have already gone out, since the sink holds every one the
229	// turn has produced and each round would otherwise resend the earlier ones.
230	sentWidgets := map[string]bool{}
231	sys := e.systemWith(memory)
232	msgs := append([]Message{sys}, history...)
233	msgs = append(msgs, Message{Role: RoleUser, Content: user})
234	tr.Add(Step{Kind: "prompt", Label: "system prompt built",
235		In: user, Out: sys.Content,
236		Meta: itoa(len(sys.Content)) + " characters, " + itoa(len(history)) + " earlier messages in the window"})
237
238	var used []tools.Result
239	// The first move on any question naming a thing, since the snapshot is on
240	// this machine and is newer than the weights. It goes in after the history
241	// so the cached prompt prefix survives, and it is recorded as a tool call
242	// because that is what it is and the reader should see its age.
243	if res, msg, ok := e.opening(ctx, user); ok {
244		emit(Event{Kind: "tool", Tool: res.Name, Args: shortArgs(string(res.Args))})
245		emit(Event{Kind: "tool_done", Tool: res.Name, MS: res.Elapsed.Milliseconds(), OK: true})
246		msgs = append(msgs, msg)
247		used = append(used, res)
248		tr.Add(Step{Kind: "wikipedia", Label: "looked the subject up before answering",
249			In: subjectOf(user), Out: msg.Content, MS: res.Elapsed.Milliseconds(),
250			Meta: "local snapshot, no web request"})
251	}
252
253	// What this conversation has already answered. A follow-up is where the loop
254	// goes wrong, since the results behind those answers are gone and the
255	// answers are not, so the model rewrites one instead of fetching anything.
256	var answered []string
257	for _, m := range history {
258		if m.Role == RoleAssistant && strings.TrimSpace(m.Content) != "" {
259			answered = append(answered, m.Content)
260		}
261	}
262
263	// news reads every feed on the list, so a second call re-reads all of them
264	// for a rundown the turn already has. One is the whole answer.
265	var usedNews bool
266	var stats Stats
267	schemas := e.reg.Schemas()
268
269	// A model that gets a thin or failed result will ask for the very same
270	// thing again, and again, until the round budget runs out. Nothing in the
271	// prompt reliably stops it, so the harness does: an identical call is
272	// answered from the ledger with a line telling it not to repeat, and after
273	// enough repeats the tools come off the table entirely.
274	seen := map[string]tools.Result{}
275	repeats := 0
276	gates := 0
277	// Set when the gate has sent the turn back, so the round that follows a
278	// nudge cannot answer with prose again. The nudge still carries the query,
279	// and this is what makes it an instruction rather than a request.
280	forceTools := false
281
282	for round := 0; round < maxToolRounds; round++ {
283		last := round == maxToolRounds-1
284		if last {
285			// Out of tool budget. Taking the tools away is what forces an
286			// answer; leaving them on lets a model spend every round calling
287			// something and hand back an empty turn.
288			msgs = append(msgs, Message{Role: RoleUser,
289				Content: "You have used your tool budget for this turn. Answer now with what you have, and say plainly if something is missing."})
290			break
291		}
292		offer := schemas
293		if usedNews {
294			offer = tools.Without(offer, tools.News.Name)
295		}
296		if repeats >= 2 {
297			// It is going in circles. Take the tools away and make it answer
298			// with what it has rather than spending the rest of the budget.
299			offer = nil
300		}
301		emit(Event{Kind: "status", Text: thinkingLabel(round)})
302		roundStart := time.Now()
303		var reply Message
304		var st Stats
305		var err error
306		if forceTools && len(offer) > 0 {
307			reply, st, err = e.llm.CompleteRequiringTool(ctx, msgs, offer, toolTurnTokens)
308		} else {
309			reply, st, err = e.llm.CompleteStats(ctx, msgs, offer, toolTurnTokens)
310		}
311		forcedThisRound := forceTools
312		forceTools = false
313		stats.merge(st)
314		on := "the conversation so far, plus " + itoa(len(offer)) + " tools on the table"
315		if forcedThisRound {
316			on += ", and it was made to call one"
317		}
318		tr.Add(Step{Kind: "model", Label: "round " + itoa(round+1) + ", decide",
319			In:  on,
320			Out: decision(reply), MS: time.Since(roundStart).Milliseconds(), Bad: err != nil,
321			Meta: itoa(st.Prompt) + " tokens in, " + itoa(st.Completion) + " out"})
322		// A tool call cut off by the token budget arrives as unparseable JSON
323		// and llama.cpp refuses the whole request, which would otherwise lose an
324		// answer the model was most of the way through. Asking again with the
325		// tools off is always answerable, since by then it has whatever the
326		// earlier rounds fetched.
327		if err != nil && isTruncatedToolCall(err) {
328			emit(Event{Kind: "status", Text: "answering"})
329			reply, st, err = e.llm.CompleteStats(ctx, msgs, nil, toolTurnTokens)
330			stats.merge(st)
331			if err == nil {
332				break
333			}
334		}
335		if err != nil {
336			return Message{}, used, nil, deps.Widgets.List(), stats, err
337		}
338		if len(reply.ToolCalls) == 0 {
339			// A model sometimes writes its tool call syntax as ordinary text,
340			// which llama.cpp cannot parse and hands back as content. Recover
341			// the call so the turn is not wasted, and strip the markup either
342			// way so it never reaches the page.
343			cleaned, salvaged := salvageCalls(reply.Content, func(n string) bool {
344				_, ok := e.reg.Get(n)
345				return ok
346			})
347			if len(salvaged) > 0 {
348				reply.Content = cleaned
349				reply.ToolCalls = salvaged
350			} else {
351				// It stopped calling tools, which is not the same as having
352				// answered. A reply that offers to go and check, or that
353				// asserts things nothing in this turn checked, goes back with
354				// the tools still on rather than becoming the answer.
355				if gates < maxGates && round < maxToolRounds-1 {
356					gateStart := time.Now()
357					nudge, gst := e.gate(ctx, user, reply.Content, answered, used, emit)
358					stats.merge(gst)
359					tr.Add(Step{Kind: "gate", Label: "checked the draft before sending it",
360						In: reply.Content, Out: gateOutcome(nudge),
361						MS: time.Since(gateStart).Milliseconds(), Bad: nudge != ""})
362					if nudge != "" {
363						gates++
364						// The draft itself is never appended. A model handed
365						// its own text back writes it again.
366						msgs = append(msgs, Message{Role: RoleUser, Content: nudge})
367						forceTools = true
368						continue
369					}
370				}
371				// It answered without tools. Stream it properly rather than
372				// handing back a block of text that appeared all at once.
373				break
374			}
375		}
376		msgs = append(msgs, reply)
377		for _, tc := range reply.ToolCalls {
378			key := tc.Function.Name + "\x00" + canonArgs(tc.Function.Arguments)
379			if prev, done := seen[key]; done {
380				repeats++
381				emit(Event{Kind: "tool", Tool: tc.Function.Name, Args: shortArgs(tc.Function.Arguments)})
382				emit(Event{Kind: "tool_done", Tool: prev.Name, MS: 0, OK: prev.Err == ""})
383				body, _ := json.Marshal(map[string]any{
384					"repeat": true,
385					"note": "You already called this tool with these exact arguments in this turn. " +
386						"The result is below and it will not change. Do not call it again. " +
387						"Use what you have, or try different arguments, or answer and say what is missing.",
388					"result": prev.Content,
389				})
390				id := tc.ID
391				if id == "" {
392					id = tc.Function.Name
393				}
394				msgs = append(msgs, Message{Role: RoleTool, ToolCallID: id, Name: prev.Name, Content: string(body)})
395				continue
396			}
397			emit(Event{Kind: "tool", Tool: tc.Function.Name, Args: shortArgs(tc.Function.Arguments)})
398			if tc.Function.Name == tools.News.Name {
399				usedNews = true
400			}
401			res := e.reg.Call(ctx, deps, tc.Function.Name, json.RawMessage(tc.Function.Arguments))
402			seen[key] = res
403			used = append(used, res)
404			tr.Add(Step{Kind: "tool", Label: res.Name, In: tc.Function.Arguments,
405				Out: resultText(res), MS: res.Elapsed.Milliseconds(), Bad: res.Err != "",
406				Meta: snapshotMeta(res.Content)})
407			emit(Event{Kind: "tool_done", Tool: res.Name, MS: res.Elapsed.Milliseconds(), OK: res.Err == ""})
408			// Straight after the tool that named it, so the chart is drawing
409			// while the answer is still being written rather than appearing
410			// under a finished one.
411			for _, wdg := range drained(deps.Widgets, sentWidgets) {
412				emit(Event{Kind: "widget", Widget: &wdg})
413			}
414			body, _ := json.Marshal(res.Content)
415			if len(body) > 14000 {
416				body = append(body[:14000], []byte(`","truncated":true}`)...)
417			}
418			id := tc.ID
419			if id == "" {
420				id = tc.Function.Name
421			}
422			msgs = append(msgs, Message{Role: RoleTool, ToolCallID: id, Name: res.Name, Content: string(body)})
423		}
424	}
425
426	// Everything the turn read, numbered. The model is handed the numbers and
427	// never an address, so a link under this answer is one a tool fetched.
428	srcs := collectSources(used)
429
430	// The answer is generated fresh here rather than reusing whatever the last
431	// tool round produced, because that one was written under a small budget
432	// with tools still on the table. Without saying so, a model writes the
433	// sentence it would have written before calling another tool, which reads
434	// as "Let me check that" and then stops.
435	msgs = append(msgs, Message{Role: RoleUser, Content: finalTurn + sourcePrompt(srcs)})
436
437	emit(Event{Kind: "status", Text: "writing"})
438	answerStart := time.Now()
439	var sb strings.Builder
440	w := &blockWriter{emit: emit, render: func(md string) string {
441		return linkCitations(e.Render(prepare(md, srcs)), srcs)
442	}}
443	text, st, err := e.llm.Stream(ctx, msgs, answerTokens, func(d string) {
444		sb.WriteString(d)
445		w.write(d)
446	})
447	w.flush()
448	stats.merge(st)
449	tr.Add(Step{Kind: "answer", Label: "wrote the reply", MS: time.Since(answerStart).Milliseconds(),
450		Out: sb.String(), Bad: err != nil && sb.Len() == 0,
451		Meta: itoa(st.Prompt) + " tokens in, " + itoa(st.Completion) + " out"})
452	if err != nil && sb.Len() == 0 {
453		return Message{}, used, nil, deps.Widgets.List(), stats, err
454	}
455	// Only on the whole answer, never on a streamed block. A block is finished
456	// when a blank line closes it and nothing knows yet whether another one is
457	// coming, so a paragraph mid answer would read as the end of it.
458	text = dropClosingOffer(prepare(text, srcs))
459	if strings.TrimSpace(text) == "" {
460		text = "I could not produce an answer for that. The model returned nothing."
461		emit(Event{Kind: "block", HTML: e.Render(text)})
462	}
463	return Message{Role: RoleAssistant, Content: text}, used, cited(text, srcs), deps.Widgets.List(), stats, nil
464}
465
466// prepare is everything done to the model's markdown before it is rendered or
467// stored: the address dump at the end goes, a schemeless address becomes a
468// link, and the citations are repaired. It runs on each finished block as it
469// streams and on the whole answer at the end, and agrees with itself because
470// every step works a line at a time.
471func prepare(md string, srcs []Source) string {
472	return attach(dropLabelMarks(linkBareAddresses(dropSourceList(md))), srcs)
473}
474
475// blockWriter turns a token stream into finished markdown blocks. It only ever
476// closes a block on a blank line that is not inside a fenced code block, since
477// a fence is full of blank lines and cutting one in half renders as garbage.
478type blockWriter struct {
479	emit     func(Event)
480	render   func(string) string
481	buf      strings.Builder
482	fenced   bool
483	lastTail string
484}
485
486func (w *blockWriter) write(d string) {
487	w.buf.WriteString(d)
488	for {
489		cut, ok := w.boundary(w.buf.String())
490		if !ok {
491			break
492		}
493		s := w.buf.String()
494		block := strings.TrimRight(s[:cut], "\n")
495		rest := s[cut:]
496		w.buf.Reset()
497		w.buf.WriteString(rest)
498		// An empty render is a block the cleanup took out, which is the
499		// address list the model still writes at the end sometimes.
500		if h := w.render(block); strings.TrimSpace(block) != "" && strings.TrimSpace(h) != "" {
501			w.emit(Event{Kind: "block", HTML: h})
502		}
503		w.lastTail = ""
504	}
505	tail := w.buf.String()
506	if looksLikeCall(tail) {
507		// Hold it back rather than showing markup that is about to be removed.
508		return
509	}
510	if tail != w.lastTail {
511		w.lastTail = tail
512		w.emit(Event{Kind: "tail", Text: tail})
513	}
514}
515
516// boundary finds the end of the first complete block in s, tracking fences so
517// a blank line inside one is not treated as the end of anything.
518func (w *blockWriter) boundary(s string) (int, bool) {
519	fenced := w.fenced
520	at := 0
521	lines := strings.SplitAfter(s, "\n")
522	for i, ln := range lines {
523		trimmed := strings.TrimSpace(ln)
524		if strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") {
525			fenced = !fenced
526		}
527		at += len(ln)
528		// A blank line outside a fence ends a block, and the last line is not
529		// a boundary because more of it may still be coming.
530		if !fenced && trimmed == "" && i < len(lines)-1 && at > 0 {
531			w.fenced = false
532			return at, true
533		}
534	}
535	return 0, false
536}
537
538func (w *blockWriter) flush() {
539	rest, _ := salvageCalls(w.buf.String(), func(string) bool { return false })
540	rest = strings.TrimSpace(rest)
541	w.buf.Reset()
542	if h := w.render(rest); rest != "" && strings.TrimSpace(h) != "" {
543		w.emit(Event{Kind: "block", HTML: h})
544	}
545	w.emit(Event{Kind: "tail", Text: ""})
546}
547
548// looksLikeCall reports whether a chunk is the start of a tool call written as
549// prose. The tail is held back once this is true, so half a tag is never shown
550// on its way to being stripped.
551func looksLikeCall(s string) bool {
552	return strings.Contains(s, "<tool_call") || strings.Contains(s, "<function=")
553}
554
555const finalTurn = `Write your reply now. You have no tools left for this turn, so do not say you are about to look something up, do not offer to check anything, and do not describe what you would do next. There is nobody to answer an offer.
556
557Use the tool results above and what this conversation has already established. Every name, title, date, number and price in your answer has to come from one of those two, and a fact you cannot point at is one to leave out. Do not quote a page you read in an earlier turn, since it is not in front of you now. Do not attach a title to the wrong person or a place to the wrong country, which is the mistake to check for before you write a name.
558
559Give the whole answer in one go, with the specifics. If a part is still missing, say which part in one line and answer the rest. If the last message was not a question, just reply to it, a line or two is the whole job.`
560
561func thinkingLabel(round int) string {
562	if round == 0 {
563		return "thinking"
564	}
565	return "checking"
566}
567
568// sizeArgs say how much to return rather than what to fetch. They are left out
569// of the repeat key, because asking for the same page again with a bigger limit
570// is the same call and it is exactly how a model talks itself into a loop.
571var sizeArgs = map[string]bool{"n": true, "max_chars": true, "top": true, "limit": true, "days": true}
572
573// canonArgs is a stable key for a set of arguments, so the same call written
574// with the keys in a different order is still recognised as the same call.
575func canonArgs(raw string) string {
576	var m map[string]any
577	if json.Unmarshal([]byte(raw), &m) != nil {
578		return strings.TrimSpace(raw)
579	}
580	keys := make([]string, 0, len(m))
581	for k := range m {
582		if sizeArgs[k] {
583			continue
584		}
585		keys = append(keys, k)
586	}
587	sort.Strings(keys)
588	var b strings.Builder
589	for _, k := range keys {
590		fmt.Fprintf(&b, "%s=%v;", k, m[k])
591	}
592	return strings.ToLower(b.String())
593}
594
595// shortArgs is what the UI shows beside a tool chip. The whole argument object
596// is noise in a status line.
597func shortArgs(raw string) string {
598	var m map[string]any
599	if json.Unmarshal([]byte(raw), &m) != nil {
600		return ""
601	}
602	for _, k := range []string{"query", "location", "symbols", "url", "expression", "artist", "league"} {
603		if v, ok := m[k]; ok {
604			s := fmt.Sprint(v)
605			if len(s) > 60 {
606				s = s[:57] + "..."
607			}
608			return s
609		}
610	}
611	return ""
612}
613
614// isTruncatedToolCall spots the refusal llama.cpp returns when the arguments of
615// a tool call did not parse. It is matched on the message because the status is
616// a plain 500 that says nothing else.
617func isTruncatedToolCall(err error) bool {
618	if err == nil {
619		return false
620	}
621	m := strings.ToLower(err.Error())
622	return strings.Contains(m, "tool call") && strings.Contains(m, "parse")
623}
624
625// drained returns the widgets a sink has gained since it was last read. The
626// sink keeps the whole turn's list because that is what gets stored on the
627// message, so emitting has to track what it already sent.
628func drained(sink *tools.Sink, sent map[string]bool) []tools.Widget {
629	var out []tools.Widget
630	for _, w := range sink.List() {
631		k := w.Kind + "\x00" + w.Symbol + "\x00" + w.Place
632		if sent[k] {
633			continue
634		}
635		sent[k] = true
636		out = append(out, w)
637	}
638	return out
639}
640
641// decision says what a round chose, which is the part of a reply worth showing
642// when the reply itself is a tool call rather than prose.
643func decision(m Message) string {
644	if len(m.ToolCalls) == 0 {
645		return "answered without calling anything"
646	}
647	var names []string
648	for _, tc := range m.ToolCalls {
649		names = append(names, tc.Function.Name+"("+shortArgs(tc.Function.Arguments)+")")
650	}
651	return "called " + strings.Join(names, ", ")
652}
653
654func gateOutcome(nudge string) string {
655	if nudge == "" {
656		return "let it through"
657	}
658	return "sent it back: " + nudge
659}
660
661func resultText(r tools.Result) string {
662	if r.Err != "" {
663		return r.Err
664	}
665	b, err := json.Marshal(r.Content)
666	if err != nil {
667		return ""
668	}
669	return string(b)
670}
671
672// snapshotMeta says how old a tool's data is, for the tools that read something
673// dated rather than the live thing.
674func snapshotMeta(content any) string {
675	if d := snapshotAge(content); d != "" {
676		return "snapshot taken " + d
677	}
678	return ""
679}