orchard

 1package skills
 2
 3import (
 4	"context"
 5	"errors"
 6	"net/http"
 7	"os"
 8	"strings"
 9	"testing"
10	"time"
11)
12
13// TestSkillsRun exercises each skill against its own upstream. None of these
14// touch the search engine, so they cost nothing against its rate limit, but
15// they are still live calls and stay behind the same flag.
16func TestSkillsRun(t *testing.T) {
17	if os.Getenv("SEARCH_LIVE") == "" {
18		t.Skip("set SEARCH_LIVE=1 to run skills against their live sources")
19	}
20	d := Deps{
21		HTTP: &http.Client{Timeout: 30 * time.Second},
22		UA:   "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
23		Now:  time.Now,
24	}
25	cases := []struct {
26		skill Skill
27		q     string
28		// want is a fragment the answer has to carry, so a skill that returns
29		// a well formed answer about the wrong thing still fails.
30		want string
31	}{
32		{Calculator{}, "what is 30 * 27", "810"},
33		{Convert{}, "30 celsius to fahrenheit", "86.0"},
34		{Convert{}, "how many km in 5 miles", "8.05"},
35		{Convert{}, "100 usd in eur", "EUR"},
36		{Time{}, "what time is it in tokyo", "Tokyo"},
37		{Time{}, "how many days until christmas", "Christmas"},
38		{Weather{}, "what is the weather this weekend", "°F"},
39		{Markets{}, "what is the S&P 500 right now", "S&P 500"},
40		{Odds{}, "what are the odds on the us open", "%"},
41		{Sports{}, "did Liverpool win their last match", "Liverpool"},
42	}
43	for _, c := range cases {
44		name := c.skill.Card().Name + "/" + c.q
45		t.Run(name, func(t *testing.T) {
46			ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
47			defer cancel()
48			res, err := c.skill.Run(ctx, c.q, d)
49			if errors.Is(err, errThrottled) {
50				// Not a code failure. Running this suite repeatedly is itself
51				// enough to get rate limited, which is the thing the cache and
52				// the narrowed sweep exist to avoid in normal use.
53				t.Skipf("%s is throttling, try again later", c.skill.Card().Name)
54			}
55			if err != nil {
56				t.Fatalf("error: %v", err)
57			}
58			if res == nil {
59				t.Fatal("declined the question it is meant to answer")
60			}
61			if !strings.Contains(res.Text, c.want) {
62				t.Errorf("answer does not carry %q:\n%s", c.want, res.Text)
63			}
64			t.Logf("%s in %s\n%s", res.Skill, res.Elapsed, res.Text)
65		})
66	}
67}
68
69// TestSkillsDecline is the other half, and the more important one. A skill
70// handed a question it cannot answer has to return nothing rather than answer
71// about something else, because the router will occasionally send it one.
72func TestSkillsDecline(t *testing.T) {
73	d := Deps{HTTP: &http.Client{Timeout: 15 * time.Second}, Now: time.Now}
74	cases := []struct {
75		skill Skill
76		q     string
77	}{
78		{Calculator{}, "how many calories in a banana"},
79		{Convert{}, "what currency does japan use"},
80		{Convert{}, "why is the euro weak"},
81		{Time{}, "when was the declaration of independence signed"},
82		{Weather{}, "what is the weather in tokyo"},
83		{Markets{}, "should i buy a house"},
84		{Odds{}, "the"},
85		{Odds{}, "how is the us open going"},
86		{Odds{}, "what's the score in the us open"},
87	}
88	for _, c := range cases {
89		t.Run(c.skill.Card().Name+"/"+c.q, func(t *testing.T) {
90			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
91			defer cancel()
92			res, _ := c.skill.Run(ctx, c.q, d)
93			if res != nil {
94				t.Errorf("answered a question it should have declined:\n%s", res.Text)
95			}
96		})
97	}
98}