orchard

  1// Compaction. A small model degrades on a long conversation, so the old turns
  2// become a summary and the recent ones stay verbatim.
  3//
  4// The rule that shapes this: llama.cpp caches the prompt prefix it has already
  5// processed, and a rewrite of the history throws that away. Measured on this
  6// stack a repeated 7,661 token prefix reprocessed 4 tokens instead of all of
  7// them, 68ms against 2,535ms. So compaction happens at a threshold and not
  8// every turn, and the summary sits at the front where it stays put between
  9// compactions rather than moving every message.
 10package main
 11
 12import (
 13	"context"
 14	"fmt"
 15	"strings"
 16	"unicode"
 17	"unicode/utf8"
 18)
 19
 20const (
 21	// Roughly four characters to a token, which is close enough to decide when
 22	// to compact and cheaper than asking the server to count.
 23	charsPerToken = 4
 24
 25	// Compact when the conversation would fill more than this share of the
 26	// window. Well below full, because the answer still needs room to be
 27	// generated into.
 28	compactAtFraction = 0.55
 29
 30	// Never summarise the last few exchanges. A follow-up like "why?" refers
 31	// to them and a summary cannot carry that.
 32	keepVerbatim = 6
 33)
 34
 35type Compactor struct {
 36	llm       *LLM
 37	ctxTokens int
 38}
 39
 40func NewCompactor(llm *LLM, ctxTokens int) *Compactor {
 41	if ctxTokens <= 0 {
 42		ctxTokens = 32768
 43	}
 44	return &Compactor{llm: llm, ctxTokens: ctxTokens}
 45}
 46
 47func (c *Compactor) budgetChars() int {
 48	return int(float64(c.ctxTokens) * compactAtFraction * charsPerToken)
 49}
 50
 51// Window turns a stored conversation into what the model is handed. It returns
 52// the messages plus whether it compacted, so the caller can persist a new
 53// summary rather than recomputing one every turn.
 54func (c *Compactor) Window(ctx context.Context, conv Conversation, stored []Stored) (msgs []Message, summary string, covered int, changed bool) {
 55	summary, covered = conv.Summary, conv.Summarize
 56	if covered > len(stored) {
 57		covered = 0
 58		summary = ""
 59	}
 60
 61	size := 0
 62	for _, m := range stored[covered:] {
 63		size += len(m.Content)
 64	}
 65	size += len(summary)
 66
 67	if size > c.budgetChars() && len(stored)-covered > keepVerbatim {
 68		cut := len(stored) - keepVerbatim
 69		if cut > covered {
 70			if s, err := c.summarise(ctx, summary, stored[covered:cut]); err == nil && strings.TrimSpace(s) != "" {
 71				summary, covered, changed = s, cut, true
 72			}
 73		}
 74	}
 75
 76	if strings.TrimSpace(summary) != "" {
 77		msgs = append(msgs, Message{Role: RoleUser,
 78			Content: "Earlier in this conversation:\n" + summary})
 79		msgs = append(msgs, Message{Role: RoleAssistant,
 80			Content: "Understood, I have that context."})
 81	}
 82	for _, m := range stored[covered:] {
 83		if m.Role != RoleUser && m.Role != RoleAssistant {
 84			continue
 85		}
 86		msgs = append(msgs, Message{Role: m.Role, Content: m.Content})
 87	}
 88	return msgs, summary, covered, changed
 89}
 90
 91// summarise rewrites the whole summary rather than appending to it, so it stops
 92// growing without bound. What it is told to keep is what a follow-up actually
 93// needs: decisions, facts established, and anything the user asked for that has
 94// not been delivered yet.
 95func (c *Compactor) summarise(ctx context.Context, prev string, older []Stored) (string, error) {
 96	var b strings.Builder
 97	if strings.TrimSpace(prev) != "" {
 98		b.WriteString("Existing summary of even earlier turns:\n")
 99		b.WriteString(prev)
100		b.WriteString("\n\n")
101	}
102	b.WriteString("Conversation to fold in:\n")
103	for _, m := range older {
104		who := "User"
105		if m.Role == RoleAssistant {
106			who = "Assistant"
107		}
108		body := m.Content
109		if len(body) > 3000 {
110			body = body[:3000] + " ..."
111		}
112		fmt.Fprintf(&b, "%s: %s\n\n", who, body)
113	}
114
115	msgs := []Message{
116		{Role: RoleSystem, Content: "You compress a conversation so it can continue in a smaller window. " +
117			"Write one replacement summary covering everything given, not a summary of the summary. Keep: what the user " +
118			"is trying to do, decisions made, facts and numbers established, names and urls that were settled on, their " +
119			"stated preferences, and anything they asked for that is still outstanding. Drop pleasantries and anything " +
120			"superseded later. Write plain sentences under bold labels, no more than 200 words, and never invent detail " +
121			"that is not in the text."},
122		{Role: RoleUser, Content: b.String()},
123	}
124	out, err := c.llm.Complete(ctx, msgs, nil, 600)
125	if err != nil {
126		return "", err
127	}
128	return strings.TrimSpace(out.Content), nil
129}
130
131// Title asks for a short name for a conversation, once, off the first exchange.
132//
133// The answer is passed as well as the question because a question is often too
134// short to name on its own. "VXUS" alone was titled "Video game streaming
135// service", when the reply beside it said plainly that it is a Vanguard ETF.
136func (c *Compactor) Title(ctx context.Context, first, answer string) string {
137	var b strings.Builder
138	b.WriteString("Name the conversation below. Do not answer it.\n\nMessage:\n<<<\n")
139	b.WriteString(trim(first, 500))
140	b.WriteString("\n>>>")
141	if a := strings.TrimSpace(answer); a != "" {
142		b.WriteString("\n\nThe reply it got, which is what the message turned out to be about:\n<<<\n")
143		b.WriteString(trim(a, 700))
144		b.WriteString("\n>>>")
145	}
146	msgs := []Message{
147		{Role: RoleSystem, Content: "You name conversations. You never answer the message you are given. " +
148			"Reply with a noun phrase of three to six words naming the subject, no quotes, no trailing " +
149			"period, and none of the words chat, conversation, question or help. " +
150			"Where the message is short or is an abbreviation, a ticker or a name, take what it refers to " +
151			"from the reply rather than guessing at it."},
152		{Role: RoleUser, Content: b.String()},
153	}
154	out, err := c.llm.Complete(ctx, msgs, nil, 30)
155	if err != nil {
156		return ""
157	}
158	// A model that answers with a sentence and then explains itself still gave
159	// a usable name on its first line, so take that rather than throwing the
160	// whole thing away and falling back to the raw question.
161	t := strings.TrimSpace(out.Content)
162	if i := strings.IndexByte(t, '\n'); i >= 0 {
163		t = t[:i]
164	}
165	// A model asked for a title sometimes answers with a markdown heading.
166	t = strings.Trim(strings.TrimSpace(t), "\"'.:*#- ")
167	if t == "" {
168		return ""
169	}
170	return trim(sentenceCase(titleWords(t)), 48)
171}
172
173// sentenceCase lifts the first letter and leaves every other one alone. The
174// model answers in whatever case it feels like, so the sidebar held "Biopharma
175// Selloff" next to "nix config file sharing", and a list of conversations reads
176// as a list when they agree. Only the first letter moves, because VXUS, Lp(a)
177// and iPhone are all cased the way they are for a reason.
178func sentenceCase(t string) string {
179	first := t
180	if i := strings.IndexByte(t, ' '); i > 0 {
181		first = t[:i]
182	}
183	// iPhone and eBay carry their capital in the middle, so a word holding one
184	// anywhere is already cased the way somebody meant it.
185	if strings.IndexFunc(first, unicode.IsUpper) >= 0 {
186		return t
187	}
188	for i, r := range t {
189		if !unicode.IsLetter(r) {
190			continue
191		}
192		return t[:i] + string(unicode.ToUpper(r)) + t[i+utf8.RuneLen(r):]
193	}
194	return t
195}
196
197// titleWords caps a title at six words. A model that answered the question
198// instead of naming it still opens on the subject, so the first six words are a
199// usable name where the whole sentence is not.
200func titleWords(t string) string {
201	f := strings.Fields(t)
202	if len(f) <= 6 {
203		return strings.Join(f, " ")
204	}
205	return strings.Join(f[:6], " ")
206}