orchard

  1package skills
  2
  3import (
  4	"context"
  5	"errors"
  6	"fmt"
  7	"net/http"
  8	"sync"
  9	"time"
 10)
 11
 12// ESPN answers 202 with an empty body when it is throttling, which is the same
 13// signature DuckDuckGo uses and is easy to read as an empty scoreboard rather
 14// than as a refusal. It showed up here because one sports question fans out to
 15// every league at once, so a single question is seventeen requests of about a
 16// megabyte, and a few questions in a row look like scraping.
 17//
 18// Two things fix it and both live in this file. The scoreboard for a league on
 19// a date does not change fast enough to fetch twice in five minutes, and once
 20// an upstream starts refusing there is no point asking again for a while.
 21var errThrottled = errors.New("upstream is throttling")
 22
 23const (
 24	scoreboardTTL = 5 * time.Minute
 25	breakerFor    = 10 * time.Minute
 26)
 27
 28type cached struct {
 29	body []byte
 30	at   time.Time
 31}
 32
 33type upstream struct {
 34	mu      sync.Mutex
 35	seen    map[string]cached
 36	blocked map[string]time.Time
 37}
 38
 39var shared = &upstream{seen: map[string]cached{}, blocked: map[string]time.Time{}}
 40
 41// blockedUntil reports whether a host is in its cooldown.
 42func (u *upstream) blockedUntil(host string) bool {
 43	u.mu.Lock()
 44	defer u.mu.Unlock()
 45	until, ok := u.blocked[host]
 46	return ok && time.Now().Before(until)
 47}
 48
 49func (u *upstream) block(host string) {
 50	u.mu.Lock()
 51	defer u.mu.Unlock()
 52	u.blocked[host] = time.Now().Add(breakerFor)
 53}
 54
 55func (u *upstream) get(key string) ([]byte, bool) {
 56	u.mu.Lock()
 57	defer u.mu.Unlock()
 58	c, ok := u.seen[key]
 59	if !ok || time.Since(c.at) > scoreboardTTL {
 60		return nil, false
 61	}
 62	return c.body, true
 63}
 64
 65func (u *upstream) put(key string, body []byte) {
 66	u.mu.Lock()
 67	defer u.mu.Unlock()
 68	// Bounded so a long-running process cannot grow this without limit. The
 69	// whole point is a handful of leagues for a few minutes.
 70	if len(u.seen) > 64 {
 71		u.seen = map[string]cached{}
 72	}
 73	u.seen[key] = cached{body: body, at: time.Now()}
 74}
 75
 76// getJSONCached is getJSON with the cache and the breaker in front of it, for
 77// the upstreams that get asked the same thing repeatedly.
 78func getJSONCached(ctx context.Context, d Deps, host, url string, out any) error {
 79	if shared.blockedUntil(host) {
 80		return errThrottled
 81	}
 82	if body, ok := shared.get(url); ok {
 83		return decode(body, out)
 84	}
 85	body, err := fetch(ctx, d, url)
 86	if err != nil {
 87		if errors.Is(err, errThrottled) {
 88			shared.block(host)
 89		}
 90		return err
 91	}
 92	shared.put(url, body)
 93	return decode(body, out)
 94}
 95
 96func fetch(ctx context.Context, d Deps, url string) ([]byte, error) {
 97	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 98	if err != nil {
 99		return nil, err
100	}
101	req.Header.Set("User-Agent", d.UA)
102	req.Header.Set("Accept", "application/json")
103
104	client := d.HTTP
105	if client == nil {
106		client = http.DefaultClient
107	}
108	resp, err := client.Do(req)
109	if err != nil {
110		return nil, err
111	}
112	defer resp.Body.Close()
113
114	// 202 with nothing in it is the throttle, not a slow answer.
115	if resp.StatusCode == http.StatusAccepted {
116		return nil, errThrottled
117	}
118	if resp.StatusCode != http.StatusOK {
119		return nil, fmt.Errorf("%s: %s", url, resp.Status)
120	}
121	return readAll(resp)
122}