orchard

  1package skills
  2
  3import (
  4	"context"
  5	"os"
  6	"strings"
  7	"testing"
  8	"time"
  9)
 10
 11// The eval set. Every case is a question and the skill that should claim it,
 12// and the negative half matters more than the positive half: the routes that
 13// go wrong are the ones sitting just outside a skill rather than far from every
 14// skill, so "who won the us open" and "what are the odds on the us open" are
 15// both here on purpose.
 16var routes = []struct {
 17	q    string
 18	want string
 19}{
 20	// maths
 21	{"30 * 27", "maths"},
 22	{"what is 15% of 240", "maths"},
 23	{"(1200 + 450) / 3", "maths"},
 24	{"how does compound interest work", None},
 25	{"how many calories in a banana", None},
 26
 27	// convert
 28	{"30 celsius to fahrenheit", "convert"},
 29	{"how many km in 5 miles", "convert"},
 30	{"100 usd in eur", "convert"},
 31	{"convert 180 pounds to kg", "convert"},
 32	{"why is the euro weak against the dollar", None},
 33	{"what currency does japan use", None},
 34
 35	// time
 36	{"what time is it in tokyo", "time"},
 37	{"how many days until christmas", "time"},
 38	{"what is the date today", "time"},
 39	{"why do we have time zones", None},
 40	{"when was the declaration of independence signed", None},
 41
 42	// weather
 43	{"what is the weather this weekend", "weather"},
 44	{"is it going to rain tomorrow", "weather"},
 45	{"do i need a jacket today", "weather"},
 46	{"how do hurricanes form", None},
 47	{"what was the hottest day ever recorded", None},
 48
 49	// markets
 50	{"what is the S&P 500 right now", "markets"},
 51	{"bitcoin price", "markets"},
 52	{"is the dow up or down today", "markets"},
 53	{"why did the market drop", None},
 54	{"should i buy bitcoin", None},
 55
 56	// page, which is claimed before the model is asked
 57	{"summary of this https://cloudinabottle.org/blog/launch-post", "page"},
 58	{"what does https://go.dev/blog/go1.24 say about generics", "page"},
 59
 60	// odds
 61	{"what are the odds on the us open", "odds"},
 62	{"who is favoured to win the election", "odds"},
 63	{"what are the chances the fed cuts rates", "odds"},
 64	{"who won the us open", None},
 65	{"how is the us open going", None},
 66	{"how do betting odds work", None},
 67
 68	// sports
 69	{"did Liverpool win their last match", "sports"},
 70	{"what was the score in the chiefs game", "sports"},
 71	{"when do liverpool play next", None},
 72	{"explain the offside rule", None},
 73
 74	// code, which is the web's job and reaches for two skills on the way past:
 75	// a script that converts something is not a conversion, and a query that
 76	// counts rows is not arithmetic.
 77	{"write me a python script for restic backups i can run from cron", None},
 78	{"build me a dockerfile that runs ollama on docker desktop", None},
 79	{"write a script to convert a csv to json", None},
 80	{"sql query to sum the order totals per customer", None},
 81	{"what is the best way to export a database to a csv on an as/400", None},
 82
 83	// the long tail, all of which is the web's job
 84	{"how do i set up wireguard on debian", None},
 85	{"sqlite vs postgres for a small site", None},
 86	{"how do i make a breakfast burrito", None},
 87	{"what is the status of the artemis program", None},
 88	{"who is the ceo of anthropic", None},
 89}
 90
 91// TestRouterEval needs the model, so it is opt in the same way the live search
 92// tests are. Without it a full `go test ./...` would load the GPU.
 93func TestRouterEval(t *testing.T) {
 94	if os.Getenv("SEARCH_LIVE") == "" {
 95		t.Skip("set SEARCH_LIVE=1 to evaluate routing against the model")
 96	}
 97	m := liveModel(t)
 98	r := Default()
 99
100	only := os.Getenv("SEARCH_EVAL_ONLY")
101	var wrong, ran int
102	for _, c := range routes {
103		if only != "" && !strings.Contains(c.q, only) {
104			continue
105		}
106		ran++
107		ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
108		got := r.Decide(ctx, m, c.q)
109		cancel()
110		if got.Skill != c.want {
111			wrong++
112			t.Errorf("%-45q got %-8s want %-8s (%s)", c.q, got.Skill, c.want, got.Why())
113		}
114	}
115	t.Logf("%d of %d routed correctly", ran-wrong, ran)
116}
117
118// TestOfflineMatch covers the fallback that runs when the model is down. It is
119// allowed to be worse than the model, but it must never claim a skill for a
120// question that is plainly the web's.
121func TestOfflineMatch(t *testing.T) {
122	r := Default()
123	for _, q := range []string{
124		"how do i set up wireguard on debian",
125		"sqlite vs postgres for a small site",
126		"who is the ceo of anthropic",
127		"how do hurricanes form",
128		"write me a python script for restic backups i can run from cron",
129		"build me a dockerfile that runs ollama on docker desktop",
130	} {
131		if got := r.match(q); got != None {
132			t.Errorf("offline matcher claimed %q for %s, should be %s", q, got, None)
133		}
134	}
135	for _, c := range []struct{ q, want string }{
136		{"summary of this https://example.com/post", "page"},
137		{"what is the weather this weekend", "weather"},
138		{"bitcoin price", "markets"},
139		{"what are the odds on the us open", "odds"},
140		{"what time is it in tokyo", "time"},
141	} {
142		if got := r.match(c.q); got != c.want {
143			t.Errorf("offline matcher got %s for %q, want %s", got, c.q, c.want)
144		}
145	}
146}
147
148// TestCardsAreRoutable is the cheap guard on the thing the router depends on:
149// a card with no negative triggers is a card that will over-fire, and a name
150// that is not a bare word cannot be an enum value.
151func TestCardsAreRoutable(t *testing.T) {
152	seen := map[string]bool{}
153	for _, s := range Default().All() {
154		c := s.Card()
155		// A claiming skill never reaches the model, so negative triggers on it
156		// would be prose nothing reads. Its Claims method is the guard the
157		// others use NotFor for.
158		if _, ok := s.(Claimer); ok {
159			if c.Name == "" || c.Does == "" {
160				t.Errorf("%q is an incomplete card", c.Name)
161			}
162			seen[c.Name] = true
163			continue
164		}
165		switch {
166		case c.Name == "" || strings.ContainsAny(c.Name, " \t"):
167			t.Errorf("%q is not usable as an enum value", c.Name)
168		case seen[c.Name]:
169			t.Errorf("two skills answer to %q", c.Name)
170		case c.Does == "":
171			t.Errorf("%s has no description, so the router is guessing", c.Name)
172		case len(c.Fires) < 3:
173			t.Errorf("%s has %d trigger examples, want at least 3", c.Name, len(c.Fires))
174		case len(c.NotFor) < 3:
175			t.Errorf("%s has %d negative triggers, want at least 3", c.Name, len(c.NotFor))
176		}
177		seen[c.Name] = true
178	}
179	if seen[None] {
180		t.Errorf("a skill is named %q, which collides with the no-skill route", None)
181	}
182}