orchard

  1package main
  2
  3import "testing"
  4
  5// The two that Isaac hit, verbatim in shape, plus the answers they must not be
  6// confused with. A detector that fires on a real answer is worse than one that
  7// misses a deferral, since a miss costs a bad turn and a false positive costs
  8// every turn that mentions searching.
  9func TestIsDeferral(t *testing.T) {
 10	defer_ := []string{
 11		"I don't have anything to report on major news over the weekend, but I can search for the latest headlines if you'd like.",
 12		"I do not have access to real time information. Would you like me to look that up?",
 13		"Let me search for that and get back to you.",
 14		"I'll check the current price for you.",
 15		"My training data only goes up to early 2025, so I cannot say what happened last weekend.",
 16		"Shall I look up the schedule?",
 17		"I can look that up for you.",
 18		"Do you want me to fetch the page?",
 19		"I'm going to search for recent coverage of this.",
 20		"I don't have any details about that. Just let me know and I'll dig into it.",
 21	}
 22	for _, s := range defer_ {
 23		if !isDeferral(s) {
 24			t.Errorf("missed a deferral: %q", s)
 25		}
 26	}
 27
 28	answers := []string{
 29		"The S&P closed at 6,412, up 0.4% on the day. https://example.com/quote",
 30		"You can search their site for the part number, it is under Support.",
 31		"Use ripgrep here, since grep -r will walk node_modules and take a minute.",
 32		"gofmt found nothing. The build passes and the three test packages are green.",
 33		"", // nothing at all is not a deferral, it is an empty turn
 34		"Saturday is 61F and dry, Sunday drops to 44F with rain after noon, so pack a shell and a warm layer for the night. https://api.weather.gov/x",
 35		// A long answer that offers more at the end has still answered.
 36		"The backup container was not running cron because the image dropped to the dev user and crond needs root, which is why the nightly restic snapshot stopped on the 14th and nothing alerted. " +
 37			"Switching the container back to root and keeping the restic call itself under the dev user fixes it without giving the backup script more than it needs. " +
 38			"The tag in the snapshot list will still say the old host until the next run, so the gap in the history is real and not a display problem. " +
 39			"You will want to run one by hand to close it. After that the schedule takes over again and there is nothing else to change. " +
 40			"If you want I can check whether the other two containers have the same problem.",
 41	}
 42	for _, s := range answers {
 43		if isDeferral(s) {
 44			t.Errorf("ate a real answer: %q", s)
 45		}
 46	}
 47}
 48
 49func TestResearchNudge(t *testing.T) {
 50	if got := researchNudge(""); got == "" {
 51		t.Fatal("an empty query still has to produce a nudge")
 52	}
 53	got := researchNudge(`major news september 6 2026`)
 54	if want := `"major news september 6 2026"`; !contains(got, want) {
 55		t.Errorf("nudge %q does not carry the query %s", got, want)
 56	}
 57	// A query carrying a quote must not break out of the one around it.
 58	if contains(researchNudge(`say "hi"`), `""`) {
 59		t.Error("inner quotes were not stripped")
 60	}
 61}
 62
 63func contains(s, sub string) bool {
 64	for i := 0; i+len(sub) <= len(s); i++ {
 65		if s[i:i+len(sub)] == sub {
 66			return true
 67		}
 68	}
 69	return false
 70}
 71
 72// The three answers with wrong sums in them on 2026-09-08, and the answers from
 73// the same day that must not be sent back. calc was called zero times that day
 74// with the contract asking for it in as many words, which is why this is a
 75// check rather than a line in the prompt.
 76func TestCountsUpInProseCatchesTheDayItWasWrong(t *testing.T) {
 77	adds := []struct{ name, draft string }{
 78		{"a day total that contradicts its own earlier figure",
 79			"**Total calories for the day so far: 1,190 calories.**\n\nThat's the Bojangles total of 930 " +
 80				"calories plus the Jimmy Dean Sausage, Egg & Cheese Maple Biscuit Roll of 280 calories."},
 81		{"a run of numbers summed wrong",
 82			"Two tablespoons of rice is about 60, two of beans about 80, two of cheese about 40, two of " +
 83				"corn about 40, and two of cooked chicken about 40, with the tortilla 250. That's around " +
 84				"510 for the fillings and 250 for the wrap, so about 760 total."},
 85		{"a list with a stated total",
 86			"- 4-piece Supreme — 500 cal [1]\n- Mashed potatoes — 120 cal [1]\n- Biscuit — 310 cal [1]\n\n" +
 87				"**Bojangles total: 930 calories.**"},
 88	}
 89	for _, c := range adds {
 90		if !countsUpInProse(c.draft) {
 91			t.Errorf("%s went through uncounted", c.name)
 92		}
 93	}
 94
 95	leaves := []struct{ name, draft string }{
 96		{"a comparison, which is not a sum",
 97			"The pinto beans have 7 g protein and the dirty rice has 5 g, so the beans win."},
 98		{"prose with numbers and no total claimed",
 99			"The RTX 5090 MSRP is $1,999 and the card is sitting at $5,799.99 right now, which is 190% above it."},
100		{"a total inside a fence, which is code and not a claim",
101			"Here is the query:\n\n```sql\nSELECT total FROM t WHERE a=1 AND b=2 AND c=3;\n```\n\nRun that."},
102		{"no numbers at all",
103			"That's the total picture, and nothing in it is surprising."},
104	}
105	for _, c := range leaves {
106		if countsUpInProse(c.draft) {
107			t.Errorf("%s was sent back", c.name)
108		}
109	}
110}