orchard

  1package main
  2
  3import (
  4	"context"
  5	"os"
  6	"strings"
  7	"testing"
  8	"time"
  9)
 10
 11// TestPipelineLive runs the real thing against the real web and the real model.
 12// It skips when the model is not up, so `make test` on a machine without a GPU
 13// still passes.
 14func TestPipelineLive(t *testing.T) {
 15	url := os.Getenv("LLM_URL")
 16	if url == "" {
 17		url = "http://search-llm:8091"
 18	}
 19	if os.Getenv("SEARCH_LIVE") == "" {
 20		t.Skip("set SEARCH_LIVE=1 to spend real searches")
 21	}
 22	llm := NewLLM(url, "")
 23	if !llm.Healthy(context.Background()) {
 24		t.Skip("model not up")
 25	}
 26
 27	dir, _ := os.MkdirTemp("", "search-e2e")
 28	defer os.RemoveAll(dir)
 29	store, err := OpenStore(dir)
 30	if err != nil {
 31		t.Fatal(err)
 32	}
 33	defer store.Close()
 34
 35	e := NewEngine(store, llm, NewBudget())
 36	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 37	defer cancel()
 38
 39	var steps []string
 40	pr := Progress(func(step, detail string) { steps = append(steps, step+": "+detail) })
 41
 42	ans, err := e.Run(ctx, "What is SQLite FTS5 and what is it used for?", nil, pr)
 43	if err != nil {
 44		t.Fatalf("run: %v", err)
 45	}
 46	for _, s := range steps {
 47		t.Logf("  %s", s)
 48	}
 49	t.Logf("shape=%s elapsed=%s support=%.0f%%", ans.Shape, ans.Elapsed, ans.Support*100)
 50	t.Logf("\n%s", ans.Text)
 51	if ans.Text == "" {
 52		t.Fatal("empty answer")
 53	}
 54	if len(ans.Sources) == 0 {
 55		t.Fatal("no sources")
 56	}
 57	if !strings.Contains(ans.HTML, "<") {
 58		t.Error("html was not rendered")
 59	}
 60}
 61
 62// TestGivenPageLive is the other half of the pipeline: a question that names
 63// the page to read never plans a search, and the answer comes off that page
 64// and nothing else.
 65func TestGivenPageLive(t *testing.T) {
 66	url := os.Getenv("LLM_URL")
 67	if url == "" {
 68		url = "http://search-llm:8091"
 69	}
 70	if os.Getenv("SEARCH_LIVE") == "" {
 71		t.Skip("set SEARCH_LIVE=1 to fetch a real page")
 72	}
 73	llm := NewLLM(url, "")
 74	if !llm.Healthy(context.Background()) {
 75		t.Skip("model not up")
 76	}
 77
 78	dir, _ := os.MkdirTemp("", "search-page")
 79	defer os.RemoveAll(dir)
 80	store, err := OpenStore(dir)
 81	if err != nil {
 82		t.Fatal(err)
 83	}
 84	defer store.Close()
 85
 86	e := NewEngine(store, llm, NewBudget())
 87	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
 88	defer cancel()
 89
 90	const page = "https://go.dev/blog/go1.24"
 91	var steps []string
 92	pr := Progress(func(step, detail string) { steps = append(steps, step+": "+detail) })
 93
 94	ans, err := e.Run(ctx, "summary of this "+page, nil, pr)
 95	if err != nil {
 96		t.Fatalf("run: %v", err)
 97	}
 98	for _, s := range steps {
 99		t.Logf("  %s", s)
100	}
101	t.Logf("shape=%s skill=%s elapsed=%s support=%.0f%%", ans.Shape, ans.Skill, ans.Elapsed, ans.Support*100)
102	t.Logf("\n%s", ans.Text)
103
104	if ans.Skill != "page" {
105		t.Errorf("skill is %q, want page", ans.Skill)
106	}
107	if ans.Shape != ShapeSummary {
108		t.Errorf("shape is %s, want summary", ans.Shape)
109	}
110	if len(ans.Queries) != 0 {
111		t.Errorf("it searched anyway: %v", ans.Queries)
112	}
113	if len(ans.Sources) != 1 || ans.Sources[0].URL != page {
114		t.Errorf("sources are %+v, want only the page it was given", ans.Sources)
115	}
116	if ans.Text == "" {
117		t.Fatal("empty answer")
118	}
119}
120
121func TestSplitClaims(t *testing.T) {
122	in := strings.Join([]string{
123		"Here is the intro sentence about the thing [1].",
124		"",
125		"- **flour** [2]",
126		"- **sugar** [2]",
127		"- **eggs** [2]",
128		"",
129		"1. Mix the flour and sugar together in a bowl [3].",
130		"2. Bake at 350F for twenty minutes [3].",
131	}, "\n")
132
133	got := splitClaims(in)
134	// One prose sentence, one grouped bullet run, two separate numbered steps.
135	if len(got) != 4 {
136		t.Fatalf("want 4 claims, got %d: %q", len(got), got)
137	}
138	if !strings.Contains(got[1], "flour") || !strings.Contains(got[1], "eggs") {
139		t.Errorf("bullets should group into one claim, got %q", got[1])
140	}
141	if strings.Contains(got[2], "Bake") {
142		t.Errorf("numbered steps must not group, got %q", got[2])
143	}
144}
145
146func TestCitedIDsAndStrip(t *testing.T) {
147	s := "The thing is true [3] and also this [12][3]."
148	ids := citedIDs(s)
149	if len(ids) != 2 || ids[0] != 3 || ids[1] != 12 {
150		t.Errorf("citedIDs = %v, want [3 12]", ids)
151	}
152	if got := stripCitations(s); got != "The thing is true and also this ." {
153		t.Errorf("stripCitations = %q", got)
154	}
155}
156
157func TestLinkCitationsSkipsTags(t *testing.T) {
158	in := `<a href="/x[9]">text [3]</a>`
159	got := linkCitations(in)
160	if strings.Contains(got, `/x<a class="cite"`) {
161		t.Error("substituted inside a tag")
162	}
163	if !strings.Contains(got, `href="#p3"`) {
164		t.Errorf("did not link the citation: %s", got)
165	}
166}
167
168func TestAmbientFactsHasNoInstructions(t *testing.T) {
169	f := AmbientFacts()
170	if strings.Contains(f, "Never state it") {
171		t.Error("instructions leaked into the facts shown to a person")
172	}
173	if !strings.Contains(f, "Today is") {
174		t.Error("no date in the ambient facts")
175	}
176}
177
178func TestBudgetWindow(t *testing.T) {
179	b := NewBudget()
180	if st := b.State(); st.Left != budgetMax || st.Cooling {
181		t.Fatalf("fresh budget: %+v", st)
182	}
183	for i := 0; i < budgetMax; i++ {
184		b.Spend()
185	}
186	st := b.State()
187	if st.Left != 0 || st.Questions != 0 {
188		t.Errorf("spent budget should be empty: %+v", st)
189	}
190	if st.ResetIn <= 0 {
191		t.Error("a spent budget should say when room opens")
192	}
193
194	b2 := NewBudget()
195	b2.Limited()
196	cooling, left := b2.Cooling()
197	if !cooling || left <= 0 {
198		t.Error("a 202 should start a cooldown")
199	}
200	if st := b2.State(); !st.Cooling || st.Note == "" {
201		t.Errorf("cooling state should explain itself: %+v", st)
202	}
203}
204
205// TestDevBypassCannotShipEnabled is the guard on the auth bypass. The release
206// image is built with -tags embed, where Reloaded is false, so this asserts the
207// only thing that matters: the environment variable cannot turn it on there.
208func TestDevBypassCannotShipEnabled(t *testing.T) {
209	t.Setenv("SEARCH_DEV_NOAUTH", "1")
210	if !Reloaded && devOpen() {
211		t.Fatal("the auth bypass is reachable in an embed build")
212	}
213	if Reloaded && !devOpen() {
214		t.Fatal("the bypass should work in a development build")
215	}
216}
217
218// liveEngine is the real thing against the real model, with a throwaway store.
219func liveEngine(t *testing.T) *Engine {
220	t.Helper()
221	if os.Getenv("SEARCH_LIVE") == "" {
222		t.Skip("set SEARCH_LIVE=1 to spend real searches")
223	}
224	url := os.Getenv("LLM_URL")
225	if url == "" {
226		url = "http://search-llm:8091"
227	}
228	llm := NewLLM(url, "")
229	if !llm.Healthy(context.Background()) {
230		t.Skip("model not up")
231	}
232	dir := os.Getenv("SEARCH_STORE")
233	if dir == "" {
234		var err error
235		if dir, err = os.MkdirTemp("", "search-e2e"); err != nil {
236			t.Fatal(err)
237		}
238		t.Cleanup(func() { os.RemoveAll(dir) })
239	}
240	store, err := OpenStore(dir)
241	if err != nil {
242		t.Fatal(err)
243	}
244	t.Cleanup(func() { store.Close() })
245	return NewEngine(store, llm, NewBudget())
246}
247
248// The shape decides the contract, so a coding question read as a how-to gets
249// prose with a snippet in it rather than a file. This costs one model call per
250// case and no searches.
251func TestShapeEval(t *testing.T) {
252	e := liveEngine(t)
253	cases := []struct {
254		q    string
255		want Shape
256	}{
257		{"build me a dockerfile that will run ollama on a windows system running docker desktop", ShapeCode},
258		{"give me a mini flask site that serves a cached frontend api for yahoo finance", ShapeCode},
259		{"write a simple html page with a map of the world showing how hot it is in every country", ShapeCode},
260		{"write me a python script i can add to my daily cron to do regular restic backups", ShapeCode},
261		{"what is the best way to export a database to a csv file on an as/400", ShapeCode},
262		{"regex to match an email address in javascript", ShapeCode},
263		{"how do i set up wireguard on debian", ShapeHowTo},
264		{"how do i make a breakfast burrito", ShapeRecipe},
265		{"sqlite vs postgres for a small site", ShapeComparison},
266		{"what is the status of the artemis program", ShapeStatus},
267		{"what is sqlite fts5", ShapeFactual},
268		{"when is the next liverpool game", ShapeUpcoming},
269		{"when do the panthers play next", ShapeUpcoming},
270		{"when is the next spacex launch", ShapeUpcoming},
271		{"what are the upcoming premier league fixtures this weekend", ShapeUpcoming},
272		{"when does the new elder scrolls come out", ShapeUpcoming},
273		// The other half of the same fixture list, which has to stay news.
274		{"did liverpool win their last match", ShapeNews},
275	}
276	only := os.Getenv("SEARCH_EVAL_ONLY")
277	var wrong, ran int
278	for _, c := range cases {
279		if only != "" && !strings.Contains(c.q, only) {
280			continue
281		}
282		ran++
283		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
284		got := e.plan(ctx, c.q, "")
285		cancel()
286		if got.Shape != c.want {
287			wrong++
288			t.Errorf("%-72q got %-10s want %s", c.q, got.Shape, c.want)
289		}
290	}
291	t.Logf("%d of %d shaped correctly", ran-wrong, ran)
292}
293
294// TestCodeAnswerLive runs one real coding question end to end and prints what
295// came back, which is the only way to judge whether the answer is pasteable.
296// SEARCH_Q overrides the question while iterating.
297func TestCodeAnswerLive(t *testing.T) {
298	e := liveEngine(t)
299	q := os.Getenv("SEARCH_Q")
300	if q == "" {
301		q = "write me a python script i can add to my daily cron to do regular restic backups"
302	}
303	ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute)
304	defer cancel()
305
306	var steps []string
307	ans, err := e.Run(ctx, q, nil, Progress(func(step, detail string) {
308		steps = append(steps, step+": "+detail)
309	}))
310	if err != nil {
311		t.Fatalf("run: %v", err)
312	}
313	for _, s := range steps {
314		t.Logf("  %s", s)
315	}
316	t.Logf("shape=%s elapsed=%s support=%.0f%%", ans.Shape, ans.Elapsed, ans.Support*100)
317	for _, c := range ans.Checks {
318		t.Logf("  check %-24s %-10s %d lines  ok=%v  %s", c.File, c.Lang, c.Lines, c.OK, c.Note)
319	}
320	for _, d := range ans.Deps {
321		t.Logf("  dep   %-30s %-8s checked=%v found=%v", d.Name, d.Eco, d.Checked, d.Found)
322	}
323	for _, w := range ans.Warnings {
324		t.Logf("  warn  %s", w)
325	}
326	for _, s := range ans.Sources {
327		t.Logf("  src   [%d] %s", s.N, s.URL)
328	}
329	t.Logf("\n%s", ans.Text)
330
331	if ans.Shape != ShapeCode {
332		t.Errorf("shape was %s, so the code contract never ran", ans.Shape)
333	}
334	if len(codeBlocks(ans.Text)) == 0 {
335		t.Error("a code question came back with no code in it")
336	}
337	if strings.Contains(ans.HTML, "<pre") && !strings.Contains(ans.HTML, "<code") {
338		t.Error("code did not render as a code block")
339	}
340}
341
342// A follow-up names nothing a router can match. "odds on the match" carries no
343// team, no competition and no date, so routing it before the rewrite meant a
344// follow-up could never reach a skill, whatever it asked for.
345func TestFollowupReachesASkill(t *testing.T) {
346	e := liveEngine(t)
347	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
348	defer cancel()
349
350	history := []Turn{{
351		Question: "what is the s&p 500 at",
352		Answer:   "The S&P 500 is at 6,502, up 0.4% on the day.",
353	}}
354	ans, err := e.Run(ctx, "what about the nasdaq", history, nil)
355	if err != nil {
356		t.Fatalf("run: %v", err)
357	}
358	t.Logf("skill=%q standalone=%q", ans.Skill, ans.Standalone)
359	t.Logf("\n%s", ans.Text)
360	if ans.Skill != "markets" {
361		t.Errorf("a follow-up asking for a quote answered from %q, not the markets skill", ans.Skill)
362	}
363	if !strings.Contains(strings.ToLower(ans.Standalone), "nasdaq") {
364		t.Errorf("the router should have seen the resolved question, got %q", ans.Standalone)
365	}
366}
367
368// The question that started this, end to end. It spends real searches, so it is
369// behind SEARCH_LIVE like the rest, and what it checks is the one thing the
370// support rate cannot see: whether the answer names a date that has not passed.
371func TestUpcomingAnswerLive(t *testing.T) {
372	e := liveEngine(t)
373	q := os.Getenv("SEARCH_Q")
374	if q == "" {
375		q = "when is the next liverpool game"
376	}
377	ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute)
378	defer cancel()
379
380	var steps []string
381	ans, err := e.Run(ctx, q, nil, Progress(func(step, detail string) {
382		steps = append(steps, step+": "+detail)
383	}))
384	if err != nil {
385		t.Fatalf("run: %v", err)
386	}
387	for _, s := range steps {
388		t.Logf("  %s", s)
389	}
390	t.Logf("shape=%s elapsed=%s support=%.0f%% retried=%v", ans.Shape, ans.Elapsed, ans.Support*100, ans.Retried)
391	for _, w := range ans.Warnings {
392		t.Logf("  warn  %s", w)
393	}
394	for _, c := range ans.Citations {
395		t.Logf("  cite  checked=%v supported=%v [%d] %s", c.Checked, c.Supported, c.PassageID, truncate(c.Sentence, 90))
396	}
397	for _, s := range ans.Sources {
398		t.Logf("  src   [%d] %s %s", s.N, s.Published, s.URL)
399	}
400	t.Logf("\n%s", ans.Text)
401
402	if ans.Shape != ShapeUpcoming {
403		t.Errorf("shape was %s, so the upcoming contract never ran", ans.Shape)
404	}
405	if noFutureDate(ans.Text, localNow()) {
406		t.Error("the answer names no date today or later, which is the whole question")
407	}
408}