orchard

  1package main
  2
  3import "strings"
  4
  5// Shape is what kind of answer a question wants. The prototype wrote every
  6// answer as four to eight sentences of prose, which is right for a factual
  7// question and wrong for a recipe, where the ingredients and the order of the
  8// steps are the answer.
  9type Shape string
 10
 11const (
 12	ShapeFactual    Shape = "factual"
 13	ShapeRecipe     Shape = "recipe"
 14	ShapeHowTo      Shape = "howto"
 15	ShapeComparison Shape = "comparison"
 16	ShapeNews       Shape = "news"
 17	ShapeStatus     Shape = "status"
 18	ShapeCode       Shape = "code"
 19	ShapeUpcoming   Shape = "upcoming"
 20	ShapeSummary    Shape = "summary"
 21)
 22
 23// shapeEnum is what the plan step offers the model, and every entry has a
 24// contract below. A shape the planner can pick and the contract map does not
 25// hold falls back to prose, which is the wrong format quietly rather than
 26// loudly, so a test walks this list.
 27var shapeEnum = []Shape{
 28	ShapeFactual, ShapeRecipe, ShapeHowTo, ShapeComparison,
 29	ShapeNews, ShapeStatus, ShapeCode, ShapeUpcoming,
 30}
 31
 32// offEnum are the shapes the planner is never offered, because they are not a
 33// reading of the question. A summary happens when the question carried the
 34// address of the page to read, which is known before any planning.
 35var offEnum = []Shape{ShapeSummary}
 36
 37// Contract is the format instruction handed to the synthesis step, and the
 38// passage budget the shape needs. A recipe needs more of the page than a
 39// factual answer does, because ingredients and method are usually far apart.
 40type Contract struct {
 41	Shape       Shape
 42	Instruction string
 43	MaxPassages int
 44	PerSource   int
 45	MaxTokens   int
 46
 47	// Reminder is the one rule that has to survive to the end, repeated after
 48	// the passages rather than left in the system prompt. A model reads the
 49	// beginning and the end of what it was given and is worst in the middle,
 50	// and by the time this one is generating, the instruction it needs most is
 51	// several thousand tokens behind it.
 52	Reminder string
 53}
 54
 55// answerFirst is prepended to every shape. The rule Isaac stated: answer the
 56// question, then the few facts worth skimming, and only then the detail, which
 57// is a bonus rather than something forced on a reader who wanted one number.
 58// The closing section used to be "the fuller explanation", and a 4B given that
 59// instruction after it has already stated the facts writes the list again in
 60// prose. That restatement is generated from a weaker signal than the bullets
 61// were, so it contradicted them often enough to be the worst thing on the page:
 62// a wrong sentence sitting directly under the right one, in a site whose whole
 63// claim is that every sentence is checked.
 64//
 65// So the rule is the same one dash's weather alert landed on. Say a thing once,
 66// and a section that has nothing new to add does not get written.
 67// Two rules every shape needs, learned off a recipe that came back with a
 68// citation after every noun and a line reading "Total Time: Not specified in
 69// passages". The reader is not in the room with the pipeline and does not know
 70// what a passage is.
 71const houseStyle = "Put at most one citation at the end of a sentence or a list item, never in the middle of one and never more than one. " +
 72	"Never refer to the answer itself, so no \"this response\", \"this answer\" or \"below you will find\". Write the thing rather than describing it. " +
 73	"Write for someone who cannot see your sources and does not know how you work. " +
 74	"Never mention passages, sources, context, or what you were given, and never write that something was not specified. " +
 75	"If a detail is missing, leave the line out entirely rather than writing a line saying it is missing. "
 76
 77// Passages describe the future as of when they were written, and a model
 78// repeating their tense says a thing is "targeted for April" in September. The
 79// date is in the ambient context and the model still did it, so both
 80// time-sensitive shapes have to be told to compare.
 81const datesArePast = "Today's date is given above, so check every date you write against it. " +
 82	"A date that has already passed is not a target, a plan or something upcoming, whatever tense the passage uses, because the passage was written before it. " +
 83	"If a passage says something was scheduled for a date that has now passed and nothing tells you what happened, say plainly that you do not know how it went rather than repeating the plan. " +
 84	"Never describe a past date as a future one. " +
 85	"The same goes for tense: a passage saying something is happening now, is currently underway, or is in progress means when that passage was written, not today. " +
 86	"If that was weeks or months ago, say what it was doing then and give the date, rather than saying it is doing it now. "
 87
 88// The mirror of datesArePast, and the half that was missing. Every shape above
 89// describes the past or a standing fact, so "when is the next Liverpool game"
 90// was classified news, whose contract opens by saying the question is about
 91// what already happened, and the answer was four fixtures that had been played
 92// followed by a line admitting it did not have the next one.
 93const datesAreFuture = "The question asks about something that has not happened yet, so only a date that is today or later can answer it. " +
 94	"Today's date is given above. Check every date you are about to write against it. " +
 95	"A match, launch, release or meeting dated before today has already happened and is not the answer, whatever tense the passage uses. " +
 96	"If nothing in the passages is dated today or later, say in the first sentence that you do not have it, and stop. " +
 97	"Listing what has already happened does not answer a question about what is next, so do not do it. "
 98
 99const answerFirst = houseStyle +
100	"Answer the question directly in the first sentence, before anything else. " +
101	"Then give the few facts that matter as a short markdown bullet list, bolding the value in each one. " +
102	"Keep that part tight, a reader should get what they asked for without scrolling. " +
103	"Stop there unless you have something the bullets do not already carry. " +
104	"Never restate, summarise or conclude, and never write a closing paragraph that repeats the list in prose. " +
105	"If a caveat, a disagreement between sources, or a piece of background genuinely adds something, write it as one or two sentences that mention no fact already in a bullet. " +
106	"Never open with background, a definition, or a restatement of the question, and never make someone read to the second paragraph to find what they asked for."
107
108var contracts = map[Shape]Contract{
109	ShapeFactual: {
110		Shape: ShapeFactual,
111		Instruction: strings.Join([]string{
112			answerFirst,
113			"For a question with one answer, say it plainly and then list the numbers, comparisons and where it is.",
114			"Three to six bullets is usually right.",
115		}, " "),
116		MaxPassages: 12, PerSource: 3, MaxTokens: 700,
117	},
118	ShapeRecipe: {
119		Shape: ShapeRecipe,
120		Instruction: strings.Join([]string{
121			houseStyle,
122			"Write an actual recipe, not a description of one.",
123			"Start with one sentence saying what it makes, then a line for yield and a line for total time, each only if you actually have it. Omit the line otherwise, and never write that it was not given.",
124			"Then a `## Ingredients` heading with a markdown bullet list, one ingredient per line.",
125			"Copy each ingredient line exactly as it is written, keeping its quantity, like `2 tbsp butter` or `8 large eggs`.",
126			"Never invent, guess or carry over a quantity. If a line gives no amount, write the ingredient on its own.",
127			"The yield is not a quantity for anything. A recipe making 15 burritos does not use 15 of each ingredient.",
128			"Then a `## Method` heading with a numbered list, one step per line, in order.",
129			"Bold ingredient names in the ingredient list and bold temperatures and times in the steps.",
130			"Follow one recipe rather than blending several, and prefer the one that comes with quantities.",
131			"The method may only use ingredients that are in your ingredient list. If a step needs something the list does not have, either add it to the list with its quantity or leave the step out.",
132			"Notes, substitutions and storage come after the method, only when the passages give them, and they never repeat an ingredient or a step already listed.",
133			"Cite once at the end of each step, not after each ingredient in it.",
134		}, " "),
135		MaxPassages: 18, PerSource: 6, MaxTokens: 1400,
136	},
137	ShapeHowTo: {
138		Shape: ShapeHowTo,
139		Instruction: strings.Join([]string{
140			"Open with one sentence saying what the steps achieve.",
141			"Then a numbered list of steps in the order they must be done, one action per step, written as an instruction.",
142			"Bold any command, file name, or exact value the reader has to type.",
143			"Cite the passage each step came from, once at the end of it.",
144			"Caveats and alternatives come after the steps and only when they add something, never a summary of the steps just given.",
145		}, " "),
146		MaxPassages: 14, PerSource: 4, MaxTokens: 1000,
147	},
148	ShapeComparison: {
149		Shape: ShapeComparison,
150		Instruction: strings.Join([]string{
151			"Open with one sentence naming which option suits which case.",
152			"Then a markdown bullet list with one line per point of difference, each naming both sides, bolding the option name at the start of the line.",
153			"Cite the passage each point of difference came from, once at the end of its line.",
154			"Then one sentence on the tradeoff that actually decides it, and stop.",
155			"Do not close by restating the differences already listed.",
156		}, " "),
157		MaxPassages: 14, PerSource: 4, MaxTokens: 900,
158	},
159	// "What is the status of the Lindsay Clancy case" is not the same question
160	// as "what happened". It asks where something stands now, which means the
161	// newest source is the one that matters and an old one is misleading rather
162	// than merely incomplete.
163	ShapeStatus: {
164		Shape: ShapeStatus,
165		Instruction: strings.Join([]string{
166			houseStyle,
167			datesArePast,
168			"Open with one sentence saying where this stands and the date it is current to.",
169			"Then a markdown bullet list of what has happened, oldest first, each with its date.",
170			"Bold the dates and the outcomes.",
171			"If the newest thing you have is more than a few weeks older than today, say so plainly in the first sentence and say that anything since is not covered, because a stale answer to a status question reads as current and is worse than no answer.",
172			"Do not present a scheduled or expected step as though it has happened, and do not present something that was scheduled for a date now past as though it is still ahead.",
173			"Say what is outstanding or next if the passages give it.",
174		}, " "),
175		MaxPassages: 16, PerSource: 4, MaxTokens: 1100,
176	},
177
178	// A code answer is the one shape where the reader does not read the
179	// answer, they paste it. So the prose is the part that gets cut and the
180	// file is the part that has to be whole: a snippet with a comment saying
181	// the rest goes here is worth nothing to someone who wanted a working
182	// thing, and it is the failure a small model reaches for when the passages
183	// only show fragments.
184	ShapeCode: {
185		Shape: ShapeCode,
186		Instruction: strings.Join([]string{
187			houseStyle,
188			"Open with one sentence saying what this does and what it needs installed, and cite it. Then the code.",
189			"Put every file in its own fenced code block, with the language after the opening fence, and the file name in bold on the line above it.",
190			"Name each file for what it does rather than app.py or script.py, and use the name the tool requires when there is one, like Dockerfile or docker-compose.yml.",
191			"Give whole files that run as they are. Every import, every function it calls, and the entry point.",
192			"Never write an ellipsis, a comment saying the rest of the code goes here, or a placeholder for something you did not write.",
193			"A value the reader has to supply is a named constant at the top with a real default, not a gap in the middle.",
194			"Never put a citation inside a code block, since it is pasted into a file and it would break it. Cite on the sentences around the code.",
195			"Comment only what is not obvious from the code, and never annotate a line with what it plainly does.",
196			"Never write a comment about what you were reading, what it did or did not say, or what you could not find. A comment reasoning about that leaves working looking code that does nothing.",
197			"If you cannot find how a part of it is done, write the simplest version that really works, using the plainest tool available, and say what is left under Watch out for.",
198			"After the code, a `## Run it` heading with the exact commands in order, one per line in a single shell block, starting from a clean machine: what to install, then how it is actually started.",
199			"That is the only place an install or a run command appears. Never put one between the files, and never write the same command twice.",
200			"End that block with the way the question said it would be used. A question about cron ends on the crontab line, one about Docker ends on the docker command, one about a web page ends on the URL to open.",
201			"Then `## Watch out for` with at most three lines, each citing where it came from, and only for something that will actually bite: a version that matters, a rate limit, a platform difference. Leave the heading out when there is nothing.",
202			"Never explain the code line by line and never restate what the file already says.",
203			"If the answer is one command rather than a program, give the command on its own in a shell block and stop.",
204			"Never wrap a command in a program that only prints it, and never write a file whose comments say it is conceptual, illustrative, or what would happen in a real environment. Everything you write has to actually run.",
205		}, " "),
206		// A single page app is one file and one file is the whole answer, so
207		// this is nearly double the next largest shape. Under it the model
208		// stops mid-tag, which is worth nothing to anybody.
209		MaxPassages: 16, PerSource: 4, MaxTokens: 3400,
210		Reminder: "Whole files that run as they are. No ellipsis, no placeholder, no comment standing in for code you did not write.",
211	},
212
213	// "When is the next Liverpool game" and "who did Liverpool play last night"
214	// want opposite halves of the same fixture list, and the plan step had six
215	// pasts and a present to choose between until this existed.
216	ShapeUpcoming: {
217		Shape: ShapeUpcoming,
218		Instruction: strings.Join([]string{
219			houseStyle,
220			datesAreFuture,
221			"Open with one sentence giving the next one: what it is, its date, and the time, the place and the competition when the passages carry them. Bold the date and the time.",
222			"The next one is the soonest date, whichever passage it is in. A page listing one competition calls its own first match the next match and it is not, so read every date across every passage before choosing.",
223			"A question asking when the next one is wants that one. Give it, cite it like any other sentence, and stop.",
224			"Only when the question asks for a list of what is coming up, give at most five, soonest first, as markdown bullets with their dates.",
225			"Every line of that list is a fixture a passage states. Never continue a pattern, never work out where a season goes next, and never write a date, an opponent or a time you did not read.",
226			"Every date and time comes from a passage. Never work one out yourself, never write a day of the week a passage did not give, and never convert a kick off time into another time zone.",
227			"Never comment on how old the passages are or when they were written. Whether the schedule has moved since is worked out elsewhere.",
228		}, " "),
229		MaxPassages: 14, PerSource: 3, MaxTokens: 800,
230		Reminder: "Only a date today or later can answer this. Anything earlier has already happened.",
231	},
232
233	// The reader picked the source, so this answers from one page rather than
234	// from whatever a search turned up, and it is the one shape whose evidence
235	// is not chosen by the pipeline.
236	ShapeSummary: {
237		Shape: ShapeSummary,
238		Instruction: strings.Join([]string{
239			houseStyle,
240			"The reader handed you this page and wants it read, so use the passages from it and nothing you know about the subject from elsewhere.",
241			"If the question asks something specific about the page, answer that in the first sentence and then give the lines that bear on it.",
242			"If it only asks what the page says, open with one sentence naming what the page is and who published it, then three to six markdown bullets carrying the points it makes, in the order it makes them.",
243			"Bold the names, numbers and dates, and keep every price, date and version exactly as it is written.",
244			"A page announcing or selling something is stating its own claim, so write that it says so rather than repeating it as fact.",
245			"If the page does not cover what was asked, say that in the first sentence and then say what it does cover.",
246			"Stop there, and never close with what it all means.",
247		}, " "),
248		MaxPassages: 16, PerSource: 16, MaxTokens: 900,
249	},
250
251	ShapeNews: {
252		Shape: ShapeNews,
253		Instruction: strings.Join([]string{
254			houseStyle,
255			datesArePast,
256			"Lead with what happened and when, in one sentence, naming the date.",
257			"Then three to five markdown bullets, each a specific development with its date if the passages give one.",
258			"Background comes after the bullets, only when it is not already in one, and never as a summary of them.",
259			"Bold names, numbers and dates.",
260			"The question asks about what already happened.",
261			"Passages about scheduled, upcoming or future events do not answer it, so do not use them as if they did.",
262			"If every passage is about something upcoming rather than something that happened, say exactly that and name the most recent thing the passages do cover.",
263			"Say plainly if the passages disagree or if the newest one is older than the question implies.",
264		}, " "),
265		MaxPassages: 14, PerSource: 3, MaxTokens: 900,
266		Reminder: "This asks what already happened. A passage about something upcoming does not answer it.",
267	},
268}
269
270// shapeNames is the enum handed to the plan step's grammar.
271func shapeNames() []string {
272	out := make([]string, len(shapeEnum))
273	for i, s := range shapeEnum {
274		out[i] = string(s)
275	}
276	return out
277}
278
279func contractFor(s Shape) Contract {
280	if c, ok := contracts[s]; ok {
281		return c
282	}
283	return contracts[ShapeFactual]
284}
285
286// guessShape is the fallback when the model's classification fails or is
287// missing. Cheap keyword matching, and it only has to beat always answering
288// with prose.
289func guessShape(q string) Shape {
290	l := strings.ToLower(q)
291	switch {
292	case containsAny(l, "recipe", "how do i make", "how to make", "ingredients for", "cook", "bake"):
293		return ShapeRecipe
294	case containsAny(l, " vs ", "versus", "compare", "difference between", "better than"):
295		return ShapeComparison
296	case containsAny(l, "write me a", "write a script", "build me a", "give me a", "example code",
297		"code for", "dockerfile", "docker compose", "regex for", "sql query", "bash script",
298		"python script", "shell script", "one liner", "snippet"):
299		return ShapeCode
300	case containsAny(l, "how do i", "how to", "how can i", "steps to", "set up", "install", "configure"):
301		return ShapeHowTo
302	case containsAny(l, "when is the next", "when's the next", "when do ", "when does ",
303		"when are ", "when will", "next game", "next match", "next fixture", "next launch",
304		"upcoming", "fixtures", "schedule for", "release date", "kick off", "kickoff",
305		"coming up", "who do they play next"):
306		return ShapeUpcoming
307	case containsAny(l, "status of", "what happened to", "where does", "latest on",
308		"any update", "how did it end", "is it over", "still going on", "outcome of",
309		"verdict", "trial", "case against", "investigation into"):
310		return ShapeStatus
311	case containsAny(l, "latest", "news", "happened", "announced", "released", "update on",
312		"most recent", "last ", "who won", "score", "result of", "this week", "yesterday"):
313		return ShapeNews
314	}
315	return ShapeFactual
316}
317
318func containsAny(s string, subs ...string) bool {
319	for _, sub := range subs {
320		if strings.Contains(s, sub) {
321			return true
322		}
323	}
324	return false
325}