orchard

  1package tools
  2
  3import (
  4	"context"
  5	"fmt"
  6	"net/http"
  7	"net/http/httptest"
  8	"net/url"
  9	"strings"
 10	"testing"
 11	"time"
 12)
 13
 14// fakeKiwix answers the two endpoints the tool uses. articles is keyed by the
 15// title as it appears in a path.
 16func fakeKiwix(t *testing.T, articles map[string]string, hits []string) (*httptest.Server, *int) {
 17	t.Helper()
 18	searches := 0
 19	mux := http.NewServeMux()
 20	mux.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
 21		searches++
 22		var items strings.Builder
 23		for _, h := range hits {
 24			fmt.Fprintf(&items, "<item><title>%s</title><link>/content/wikipedia/%s</link></item>",
 25				h, strings.ReplaceAll(h, " ", "_"))
 26		}
 27		w.Header().Set("Content-Type", "application/xml")
 28		fmt.Fprintf(w, `<?xml version="1.0" encoding="UTF-8"?><rss><channel>%s</channel></rss>`, items.String())
 29	})
 30	mux.HandleFunc("/content/wikipedia/", func(w http.ResponseWriter, r *http.Request) {
 31		title := strings.TrimPrefix(r.URL.Path, "/content/wikipedia/")
 32		body, ok := articles[title]
 33		if !ok {
 34			http.NotFound(w, r)
 35			return
 36		}
 37		fmt.Fprint(w, body)
 38	})
 39	mux.HandleFunc("/catalog/v2/entries", func(w http.ResponseWriter, r *http.Request) {
 40		fmt.Fprint(w, `<feed><entry><updated>2026-06-29T00:00:00Z</updated></entry></feed>`)
 41	})
 42	srv := httptest.NewServer(mux)
 43	t.Cleanup(srv.Close)
 44	return srv, &searches
 45}
 46
 47func wikiDeps() *Deps {
 48	return &Deps{HTTP: &http.Client{Timeout: 5 * time.Second}, Now: time.Now, Guard: NewGuard(time.Minute)}
 49}
 50
 51func lookup(t *testing.T, base, q string) map[string]any {
 52	t.Helper()
 53	out, err := wikiLookup(context.Background(), wikiDeps(), base, q)
 54	if err != nil {
 55		t.Fatalf("lookup %q: %v", q, err)
 56	}
 57	m, ok := out.(map[string]any)
 58	if !ok {
 59		t.Fatalf("lookup %q returned %T", q, out)
 60	}
 61	return m
 62}
 63
 64// A one word lookup names an article, and going through the ranker instead can
 65// return a page that merely mentions the word.
 66func TestExactTitleBeatsTheRanker(t *testing.T) {
 67	srv, searches := fakeKiwix(t,
 68		map[string]string{"PostgreSQL": "<p>PostgreSQL is a database.</p>"},
 69		[]string{"Misskey"})
 70
 71	got := lookup(t, srv.URL, "PostgreSQL")
 72	if got["found"] != true {
 73		t.Fatalf("found = %v, want true", got["found"])
 74	}
 75	if got["title"] != "PostgreSQL" {
 76		t.Errorf("title = %v, want PostgreSQL", got["title"])
 77	}
 78	if *searches != 0 {
 79		t.Errorf("the ranker was asked %d times when the title was exact", *searches)
 80	}
 81}
 82
 83// Article titles capitalise their first letter, so a lowercase query has to be
 84// tried both ways before falling through to search.
 85func TestExactTitleTriesTheCapital(t *testing.T) {
 86	srv, _ := fakeKiwix(t,
 87		map[string]string{"Photosynthesis": "<p>Photosynthesis is a process.</p>"},
 88		nil)
 89
 90	got := lookup(t, srv.URL, "photosynthesis")
 91	if got["title"] != "Photosynthesis" {
 92		t.Errorf("title = %v, want Photosynthesis", got["title"])
 93	}
 94}
 95
 96// The failure this guard exists for: a short query with no article of its own,
 97// where the best ranked hit shares nothing with it and reads as an answer.
 98func TestShortQueryWithNoRealMatchIsAMiss(t *testing.T) {
 99	srv, _ := fakeKiwix(t,
100		map[string]string{"Abu_Simbel_temples": "<p>Abu Simbel is in Egypt.</p>"},
101		[]string{"Abu Simbel temples"})
102
103	got := lookup(t, srv.URL, "goodyear welt")
104	if got["found"] != false {
105		t.Fatalf("found = %v, want false for a match sharing no word", got["found"])
106	}
107	if got["summary"] != nil {
108		t.Error("a rejected match still carried a summary")
109	}
110	near, _ := got["near_titles"].([]string)
111	if len(near) == 0 || near[0] != "Abu Simbel temples" {
112		t.Errorf("near_titles = %v, want the rejected title first", got["near_titles"])
113	}
114}
115
116// A question is not what this takes. Ranking lead sections across 19 million
117// articles answered "who is the leader of north korea" with a military history
118// article, so a question is refused and the model is told to pass the subject.
119func TestAQuestionIsRefusedRatherThanAnsweredWrong(t *testing.T) {
120	srv, _ := fakeKiwix(t,
121		map[string]string{"Military_history_of_Korea": "<p>Korea's military history.</p>"},
122		[]string{"Military history of Korea"})
123
124	got := lookup(t, srv.URL, "who is the leader of north korea")
125	if got["found"] != false {
126		t.Fatalf("found = %v, want false for a question", got["found"])
127	}
128	note, _ := got["note"].(string)
129	if !strings.Contains(note, "name of the thing") {
130		t.Errorf("note does not say to pass the subject: %q", note)
131	}
132}
133
134// The same question asked as a subject has to work, or the advice in the miss
135// above goes nowhere.
136func TestTheSubjectBehindAQuestionResolves(t *testing.T) {
137	srv, _ := fakeKiwix(t,
138		map[string]string{"North_Korea": "<p>North Korea is a country in East Asia.</p>"},
139		[]string{"North Korea"})
140
141	got := lookup(t, srv.URL, "North Korea")
142	if got["title"] != "North Korea" {
143		t.Errorf("title = %v, want North Korea", got["title"])
144	}
145}
146
147// A title carrying more than was asked for still matches when the extra is a
148// qualifier, which is how a place resolves from the name people use for it.
149func TestAQualifiedTitleStillMatches(t *testing.T) {
150	srv, _ := fakeKiwix(t,
151		map[string]string{"Yadkin_Valley,_North_Carolina": "<p>The Yadkin Valley is a region.</p>"},
152		[]string{"Yadkin Valley, North Carolina"})
153
154	got := lookup(t, srv.URL, "yadkin valley")
155	if got["found"] != true {
156		t.Fatalf("found = %v, want true", got["found"])
157	}
158}
159
160// The licence notice kiwix appends to every article would otherwise end every
161// summary on the same two sentences.
162func TestTheLicenceFooterIsStripped(t *testing.T) {
163	article := `<p>A thing exists.</p><div class="zim-footer">This article is issued from Wikipedia.</div>`
164	srv, _ := fakeKiwix(t, map[string]string{"Thing": article}, nil)
165
166	got := lookup(t, srv.URL, "Thing")
167	sum, _ := got["summary"].(string)
168	if strings.Contains(sum, "issued from Wikipedia") {
169		t.Errorf("the licence footer survived: %s", sum)
170	}
171	if !strings.Contains(sum, "A thing exists") {
172		t.Errorf("the prose was lost: %s", sum)
173	}
174}
175
176// The infobox is kept as labelled lines rather than flattened into the prose.
177// Throwing it away read better and lost the row that answers who holds an
178// office, which is the only place a lead names an incumbent.
179func TestTheInfoboxBecomesLabelledLines(t *testing.T) {
180	article := `<h1>Sourdough</h1>
181	<table class="infobox"><tr><th>Type</th><td>Bread</td></tr>
182	<tr><td>Emblem of the loaf</td></tr>
183	<tr><td><table><tr><td>Nested junk</td></tr></table></td></tr></table>
184	<p>Sourdough bread is made by fermentation.</p>`
185	srv, _ := fakeKiwix(t, map[string]string{"Sourdough": article}, nil)
186
187	got := lookup(t, srv.URL, "Sourdough")
188	sum, _ := got["summary"].(string)
189	if !strings.Contains(sum, "Type: Bread") {
190		t.Errorf("the labelled row was lost: %s", sum)
191	}
192	if strings.Contains(sum, "Emblem of the loaf") {
193		t.Errorf("a picture caption was kept as a fact: %s", sum)
194	}
195	if !strings.Contains(sum, "fermentation") {
196		t.Errorf("summary lost the prose: %s", sum)
197	}
198}
199
200// The failure this was written for. "Prime Minister of Japan" describes the
201// office in its lead and names the current holder only in the infobox, so
202// dropping the table meant the snapshot could not answer who it is.
203func TestTheIncumbentRowSurvives(t *testing.T) {
204	article := `<table class="infobox">
205	<tr><td>Standard of the prime minister</td></tr>
206	<tr><td>Incumbent Sanae Takaichi since 21 October 2025</td></tr>
207	<tr><th>Seat</th><td>Tokyo</td></tr></table>
208	<p>The prime minister of Japan is the head of government.</p>`
209	srv, _ := fakeKiwix(t, map[string]string{"Prime_Minister_of_Japan": article}, nil)
210
211	got := lookup(t, srv.URL, "Prime Minister of Japan")
212	sum, _ := got["summary"].(string)
213	if !strings.Contains(sum, "Sanae Takaichi") {
214		t.Errorf("the incumbent is missing, which is the whole point: %s", sum)
215	}
216	if !strings.Contains(sum, "head of government") {
217		t.Errorf("the prose was lost: %s", sum)
218	}
219	if strings.Contains(sum, "Standard of the prime minister") {
220		t.Errorf("a picture caption was kept as a fact: %s", sum)
221	}
222}
223
224// An infobox carries its own stylesheet in a cell, which flattens into a run of
225// class rules and is never a fact.
226func TestInfoboxStylesheetsAreNotFacts(t *testing.T) {
227	article := `<table class="infobox">
228	<tr><th>Type</th><td>.mw-parser-output .plainlist ol{margin:0}</td></tr>
229	<tr><th>Seat</th><td>Tokyo</td></tr></table><p>Prose.</p>`
230	srv, _ := fakeKiwix(t, map[string]string{"Thing": article}, nil)
231
232	got := lookup(t, srv.URL, "Thing")
233	sum, _ := got["summary"].(string)
234	if strings.Contains(sum, "mw-parser-output") {
235		t.Errorf("a stylesheet came back as a fact: %s", sum)
236	}
237	if !strings.Contains(sum, "Seat: Tokyo") {
238		t.Errorf("the real row was lost with it: %s", sum)
239	}
240}
241
242// A long infobox must not crowd out the prose the reader came for.
243func TestTheInfoboxIsCapped(t *testing.T) {
244	var rows strings.Builder
245	for i := 0; i < 40; i++ {
246		fmt.Fprintf(&rows, "<tr><th>Label%d</th><td>Value%d</td></tr>", i, i)
247	}
248	article := "<table class=\"infobox\">" + rows.String() + "</table><p>Prose.</p>"
249	srv, _ := fakeKiwix(t, map[string]string{"Thing": article}, nil)
250
251	got := lookup(t, srv.URL, "Thing")
252	sum, _ := got["summary"].(string)
253	if strings.Count(sum, "Label") > wikiMaxFacts {
254		t.Errorf("more than %d rows came back: %s", wikiMaxFacts, sum)
255	}
256	if !strings.Contains(sum, "Prose.") {
257		t.Errorf("the prose was crowded out: %s", sum)
258	}
259}
260
261// Cutting at the first heading is what keeps this a lead section if the ZIM is
262// ever swapped for a flavour that carries whole articles.
263func TestOnlyTheLeadSectionIsReturned(t *testing.T) {
264	article := `<p>The lead sentence.</p><h2 id="History">History</h2><p>Everything after.</p>`
265	srv, _ := fakeKiwix(t, map[string]string{"Thing": article}, nil)
266
267	got := lookup(t, srv.URL, "Thing")
268	sum, _ := got["summary"].(string)
269	if strings.Contains(sum, "Everything after") {
270		t.Errorf("the body below the first heading came back: %s", sum)
271	}
272	if !strings.Contains(sum, "lead sentence") {
273		t.Errorf("the lead is missing: %s", sum)
274	}
275}
276
277// Nothing found has to be a miss the model is told to search past, since the
278// snapshot is large enough that an empty result reads as proof of absence.
279func TestNothingFoundSaysSoAndSendsItToSearch(t *testing.T) {
280	srv, _ := fakeKiwix(t, nil, nil)
281
282	got := lookup(t, srv.URL, "zzzznotathing")
283	if got["found"] != false {
284		t.Fatalf("found = %v, want false", got["found"])
285	}
286	note, _ := got["note"].(string)
287	if !strings.Contains(note, "web_search") {
288		t.Errorf("note does not send it to search: %q", note)
289	}
290}
291
292// The date is the difference between background and a stale fact presented as
293// current, so it has to reach the model with the answer.
294func TestTheSnapshotDateIsInTheResult(t *testing.T) {
295	srv, _ := fakeKiwix(t, map[string]string{"Thing": "<p>A thing.</p>"}, nil)
296
297	got := lookup(t, srv.URL, "Thing")
298	note, _ := got["note"].(string)
299	if !strings.Contains(note, "June 2026") {
300		t.Errorf("note does not carry the snapshot date: %q", note)
301	}
302}
303
304// A title with a space or a bracket has to survive into a url the model can
305// hand to web_fetch.
306func TestTheWikipediaURLIsUsable(t *testing.T) {
307	srv, _ := fakeKiwix(t,
308		map[string]string{"Supreme_Leader_(North_Korea)": "<p>The supreme leader.</p>"},
309		[]string{"Supreme Leader (North Korea)"})
310
311	got := lookup(t, srv.URL, "who is the leader of north korea")
312	want := "https://en.wikipedia.org/wiki/Supreme_Leader_(North_Korea)"
313	if got["url"] != want {
314		t.Errorf("url = %v, want %v", got["url"], want)
315	}
316	if _, err := url.Parse(got["url"].(string)); err != nil {
317		t.Errorf("url does not parse: %v", err)
318	}
319}
320
321// The snapshot is on the bridge, so it must not be reachable through the fence
322// that exists for third party hosts, and must not put a container in the
323// penalty box when it blips.
324func TestTheSnapshotIsNotRateLimitedLikeAThirdParty(t *testing.T) {
325	srv, _ := fakeKiwix(t, map[string]string{"Thing": "<p>A thing.</p>"}, nil)
326	d := wikiDeps()
327
328	for i := 0; i < 5; i++ {
329		if _, err := wikiLookup(context.Background(), d, srv.URL, "Thing"); err != nil {
330			t.Fatalf("call %d: %v", i, err)
331		}
332	}
333	if down := d.Guard.Down(); len(down) > 0 {
334		t.Errorf("the snapshot host was put in the penalty box: %v", down)
335	}
336}