orchard

  1package main
  2
  3import (
  4	"context"
  5	"encoding/json"
  6	"fmt"
  7	"net/http"
  8	"net/http/httptest"
  9	"strings"
 10	"testing"
 11
 12	"chat.bythewood.me/tools"
 13)
 14
 15// The shapes a question actually arrives in, and what has to come out of them
 16// for the snapshot to be asked about the right thing.
 17func TestSubjectOf(t *testing.T) {
 18	cases := []struct{ question, want string }{
 19		{"what is a goodyear welt", "goodyear welt"},
 20		{"what is postgresql", "postgresql"},
 21		{"what is kubernetes for", "kubernetes"},
 22		{"tell me about photosynthesis", "photosynthesis"},
 23		{"who is kim jong un", "kim jong un"},
 24		{"what's a b-tree", "b-tree"},
 25		{"what is a b-tree and why do databases use them", "b-tree"},
 26		{"explain the calvin cycle", "calvin cycle"},
 27		{"where is yadkin valley", "yadkin valley"},
 28		{"define entropy", "entropy"},
 29
 30		// Nothing worth looking up, so the gate is left as it was.
 31		{"", ""},
 32		// The snapshot has an article on each of these and none of them answers
 33		// what is being asked, which is today's value.
 34		{"what's the weather like", ""},
 35		{"what is the weather", ""},
 36		{"what time is it", ""},
 37		{"what's the price", ""},
 38		{"why", ""},
 39		{"can you write me a bash script that renames every file in a directory", ""},
 40		{"what do you think about the way i structured the makefile in that repo", ""},
 41	}
 42	for _, c := range cases {
 43		if got := subjectOf(c.question); got != c.want {
 44			t.Errorf("subjectOf(%q) = %q, want %q", c.question, got, c.want)
 45		}
 46	}
 47}
 48
 49// fakeWiki stands in for kiwix so the engine can be driven without one.
 50func fakeWiki(t *testing.T, title, lead string) string {
 51	t.Helper()
 52	mux := http.NewServeMux()
 53	mux.HandleFunc("/content/wikipedia/", func(w http.ResponseWriter, r *http.Request) {
 54		want := "/content/wikipedia/" + strings.ReplaceAll(title, " ", "_")
 55		if r.URL.Path != want {
 56			http.NotFound(w, r)
 57			return
 58		}
 59		fmt.Fprintf(w, "<p>%s</p>", lead)
 60	})
 61	mux.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
 62		fmt.Fprint(w, `<?xml version="1.0"?><rss><channel></channel></rss>`)
 63	})
 64	mux.HandleFunc("/catalog/v2/entries", func(w http.ResponseWriter, r *http.Request) {
 65		fmt.Fprint(w, `<feed><entry><updated>2026-06-11T00:00:00Z</updated></entry></feed>`)
 66	})
 67	srv := httptest.NewServer(mux)
 68	t.Cleanup(srv.Close)
 69	return srv.URL
 70}
 71
 72func groundEngine(t *testing.T, base string) *Engine {
 73	t.Helper()
 74	tools.WikiBase = base
 75	t.Cleanup(func() { tools.WikiBase = "http://orchard-wiki:8000" })
 76	return NewEngine(NewLLM("http://127.0.0.1:1", "local", ""), "test model")
 77}
 78
 79// The case this was written for: the model answered from memory, so the gate
 80// has no tool result to check the draft against, and the snapshot has one.
 81func TestBackgroundIsFoundForAnAnsweredFromMemoryQuestion(t *testing.T) {
 82	eng := groundEngine(t, fakeWiki(t, "Goodyear welt",
 83		"A Goodyear welt is a strip of leather. The machine was invented by Auguste Destouy and improved for Charles Goodyear Jr."))
 84
 85	got := eng.background(context.Background(), "what is a goodyear welt")
 86	if !strings.Contains(got, "Charles Goodyear Jr") {
 87		t.Errorf("background did not carry the article: %q", got)
 88	}
 89	if !strings.Contains(got, "looked up here rather than by the model") {
 90		t.Errorf("background does not say where it came from: %q", got)
 91	}
 92	// Without the age the gate cannot tell a stale article from a wrong draft.
 93	if !strings.Contains(got, "June 2026") {
 94		t.Errorf("background does not carry the snapshot date: %q", got)
 95	}
 96}
 97
 98// The gate is told how old the background is, so a draft that is right about
 99// something recent is not sent back to be corrected against an older article.
100func TestTheGatePromptWeighsTheSnapshotAge(t *testing.T) {
101	if !strings.Contains(gateSystem, "background is old rather than the draft wrong") {
102		t.Error("the gate prompt does not tell it a stale background is not a contradiction")
103	}
104	if !strings.Contains(gateSystem, "on the date it states") {
105		t.Error("the gate prompt does not say the background carries its own date")
106	}
107}
108
109// A question with no article behind it must add nothing, since an empty
110// background has to leave the gate exactly as it was.
111func TestBackgroundIsEmptyWhenTheSnapshotHasNothing(t *testing.T) {
112	eng := groundEngine(t, fakeWiki(t, "Something Else", "Unrelated."))
113
114	if got := eng.background(context.Background(), "what is a zzzznotathingxyz"); got != "" {
115		t.Errorf("background = %q, want empty", got)
116	}
117}
118
119// A question that is not about a lookupable thing must not cost a call at all.
120func TestBackgroundSkipsAQuestionWithNoSubject(t *testing.T) {
121	calls := 0
122	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
123		calls++
124		http.NotFound(w, r)
125	}))
126	defer srv.Close()
127	eng := groundEngine(t, srv.URL)
128
129	got := eng.background(context.Background(), "can you write me a bash script that renames every file in a directory")
130	if got != "" {
131		t.Errorf("background = %q, want empty", got)
132	}
133	if calls != 0 {
134		t.Errorf("the snapshot was asked %d times for a question with no subject", calls)
135	}
136}
137
138// A snapshot that is down must not change a verdict, since this only ever adds
139// evidence and the turn has to survive without it.
140func TestBackgroundSurvivesTheSnapshotBeingDown(t *testing.T) {
141	eng := groundEngine(t, "http://127.0.0.1:1")
142
143	if got := eng.background(context.Background(), "what is postgresql"); got != "" {
144		t.Errorf("background = %q, want empty when the snapshot is unreachable", got)
145	}
146}
147
148// The nudge has to name the tool and the subject, or a model sent back goes and
149// searches the web for what is already on this machine.
150func TestWikiNudgeNamesTheToolAndTheSubject(t *testing.T) {
151	n := wikiNudge("goodyear welt")
152	for _, want := range []string{"wikipedia", "goodyear welt", "correct"} {
153		if !strings.Contains(n, want) {
154			t.Errorf("nudge does not mention %q: %s", want, n)
155		}
156	}
157	if strings.Contains(strings.ToLower(n), "search the web") {
158		t.Errorf("nudge sends it to the web: %s", n)
159	}
160}
161
162// The gate is handed the background as its own block, so "no tool was called"
163// stays true and the model is not told something was fetched when nothing was.
164func TestBackgroundDoesNotMasqueradeAsAToolResult(t *testing.T) {
165	var sent string
166	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
167		b, _ := json.Marshal(map[string]any{
168			"choices": []map[string]any{{"message": map[string]any{
169				"content": `{"verdict":"answered","query":""}`}}},
170		})
171		body := make([]byte, r.ContentLength)
172		r.Body.Read(body)
173		sent = string(body)
174		w.Header().Set("Content-Type", "application/json")
175		w.Write(b)
176	}))
177	defer srv.Close()
178
179	eng := NewEngine(NewLLM(srv.URL, "local", ""), "test model")
180	eng.enough(context.Background(), "what is a goodyear welt", "A draft.",
181		"wikipedia on Goodyear welt, from an offline snapshot taken June 2026, looked up here rather than by the model: a strip of leather", nil, nil)
182
183	if !strings.Contains(sent, "no tool was called") {
184		t.Errorf("the gate was not told that no tool ran: %s", trimLine(sent, 400))
185	}
186	if !strings.Contains(sent, "Background, looked up locally") {
187		t.Errorf("the background was not sent as its own block: %s", trimLine(sent, 400))
188	}
189}
190
191// A gate that answers "research" whatever it is asked, so the path through it
192// is what the test is measuring rather than the model's judgment.
193func fakeGate(t *testing.T, verdict string) string {
194	t.Helper()
195	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
196		b, _ := json.Marshal(map[string]any{
197			"choices": []map[string]any{{"message": map[string]any{
198				"content": `{"verdict":"` + verdict + `","query":"something"}`}}},
199		})
200		w.Header().Set("Content-Type", "application/json")
201		w.Write(b)
202	}))
203	t.Cleanup(srv.Close)
204	return srv.URL
205}
206
207// The whole reason for a local corpus is that it works while a search host is
208// refusing us, so the check it makes possible has to work then too.
209func TestTheLocalCheckStillRunsWhenSearchIsBoxed(t *testing.T) {
210	tools.WikiBase = fakeWiki(t, "Goodyear welt", "A Goodyear welt is a strip of leather.")
211	t.Cleanup(func() { tools.WikiBase = "http://orchard-wiki:8000" })
212
213	eng := NewEngine(NewLLM(fakeGate(t, "research"), "local", ""), "test model")
214	eng.Deps().Guard.Trip(tools.SearchHost)
215	if _, down := eng.SearchDown(); !down {
216		t.Fatal("the search host was not boxed, so this proves nothing")
217	}
218
219	nudge, _ := eng.gate(context.Background(), "what is a goodyear welt",
220		"A goodyear welt is made by the Goodyear tyre company.", nil, nil, func(Event) {})
221	if nudge == "" {
222		t.Fatal("a draft the snapshot disagrees with was let through while search was boxed")
223	}
224	if !strings.Contains(nudge, "wikipedia") {
225		t.Errorf("the nudge does not name the local tool: %s", nudge)
226	}
227	if strings.Contains(strings.ToLower(nudge), "search for") {
228		t.Errorf("the nudge sends it to a search that is refusing us: %s", nudge)
229	}
230}
231
232// With search boxed and nothing local to check against, the draft has to stand.
233// Sending it anywhere is a loop, since every road out of here needs that host.
234func TestABoxedSearchWithNoBackgroundLetsTheDraftStand(t *testing.T) {
235	tools.WikiBase = fakeWiki(t, "Something Else", "Unrelated.")
236	t.Cleanup(func() { tools.WikiBase = "http://orchard-wiki:8000" })
237
238	eng := NewEngine(NewLLM(fakeGate(t, "research"), "local", ""), "test model")
239	eng.Deps().Guard.Trip(tools.SearchHost)
240
241	nudge, _ := eng.gate(context.Background(), "what is a zzzznotathingxyz",
242		"I think it is a kind of bird.", nil, nil, func(Event) {})
243	if nudge != "" {
244		t.Errorf("the turn was sent back with nowhere to go: %s", nudge)
245	}
246}
247
248// The chip has to say how old a snapshot answer is, since it otherwise looks
249// exactly like one read off the live web.
250func TestSnapshotAgeReachesTheToolSummary(t *testing.T) {
251	got := snapshotAge(map[string]any{"found": true, "snapshot_date": "June 2026"})
252	if got != "June 2026" {
253		t.Errorf("snapshotAge = %q, want %q", got, "June 2026")
254	}
255	// A tool that reads the live thing has no age and must not grow one.
256	if got := snapshotAge(map[string]any{"temperature": 71}); got != "" {
257		t.Errorf("snapshotAge = %q, want empty for a live tool", got)
258	}
259	if got := snapshotAge("not a map"); got != "" {
260		t.Errorf("snapshotAge = %q, want empty", got)
261	}
262}
263
264// "Big news today" reduces to "Big news", which was in neither map, and the
265// snapshot answered it with the founding date of Universe Today. The head noun
266// is what a phrase is really about.
267func TestSubjectOfSkipsALiveHeadNoun(t *testing.T) {
268	for _, q := range []string{
269		"Big news today",
270		"any big news today",
271		"what are the latest headlines",
272		"whats the current gas price",
273	} {
274		if got := subjectOf(q); got != "" {
275			t.Errorf("subjectOf(%q) = %q, want it skipped", q, got)
276		}
277	}
278	// And a real subject whose name merely contains one of those words survives.
279	for _, q := range []string{
280		"what is News Corporation",
281		"who is Rick Springfield",
282	} {
283		if subjectOf(q) == "" {
284			t.Errorf("subjectOf(%q) was skipped, want it looked up", q)
285		}
286	}
287}