orchard

  1package tools
  2
  3import (
  4	"fmt"
  5	"sync"
  6	"time"
  7)
  8
  9// A spend limit on the hosts that ban an address for asking too often.
 10//
 11// The gap between two calls was never the thing that got this address blocked.
 12// Six seconds of spacing was already in place and DuckDuckGo blocked it anyway,
 13// because a gap bounds the rate and nothing bounded the total. A model given
 14// six tool rounds and a nudge to keep going can spend an afternoon's worth of
 15// searches on one question and still honour every gap.
 16//
 17// So there are three ceilings and a call has to clear all of them. When one is
 18// spent, searching stops and says so, which is the honest answer and the one
 19// that lets the limit expire instead of renewing it.
 20//
 21// The numbers come from what the duckduckgo_search library and the people
 22// running into this recommend, held well under their ceiling rather than at it:
 23// that library asks for two seconds between calls and says to wait fifteen
 24// after an error, and the figure repeated for an address is to stay under
 25// thirty requests a minute. Nothing official is published, since scraping the
 26// HTML endpoint is against their terms in the first place, so the right posture
 27// is to be a light user rather than to find the edge.
 28type budget struct {
 29	gap    time.Duration
 30	minute int
 31	hour   int
 32	day    int
 33}
 34
 35var budgets = map[string]budget{
 36	// Six seconds is already stricter than the two that library asks for, and
 37	// it is kept because a person asking one question does not notice it. The
 38	// pools below are the part that was missing.
 39	//
 40	// Lowered from 45 an hour and 300 a day after 2026-09-08, when DuckDuckGo
 41	// blocked this address at roughly 130 requests over five hours. Neither
 42	// ceiling was ever reached, so neither was protecting anything: sustained
 43	// volume is what it objects to, not a burst. Every search this site makes
 44	// now goes through this count, so the ceiling is the whole of what leaves.
 45	SearchHost:                 {gap: 6 * time.Second, minute: 8, hour: 30, day: 200},
 46	"cdn.espn.com":             {gap: 3 * time.Second, minute: 15, hour: 120, day: 900},
 47	"query1.finance.yahoo.com": {gap: 3 * time.Second, minute: 15, hour: 120, day: 900},
 48	"api.coingecko.com":        {gap: 2 * time.Second, minute: 20, hour: 200, day: 1500},
 49	// The widget hosts. A chart the reader flips between four ranges is four
 50	// calls in a few seconds, so the gap here is small and the ceiling is what
 51	// stops a page left open from becoming a poller.
 52	"query2.finance.yahoo.com":       {gap: 3 * time.Second, minute: 15, hour: 120, day: 900},
 53	"api.open-meteo.com":             {gap: 500 * time.Millisecond, minute: 40, hour: 400, day: 3000},
 54	"air-quality-api.open-meteo.com": {gap: 500 * time.Millisecond, minute: 40, hour: 400, day: 3000},
 55	// Unofficial and the only pollen there is, so it is paced well under what
 56	// dash already asks of it rather than at a limit nobody published.
 57	"www.pollen.com": {gap: 2 * time.Second, minute: 10, hour: 60, day: 400},
 58}
 59
 60var defaultBudget = budget{gap: 400 * time.Millisecond, minute: 60, hour: 900, day: 8000}
 61
 62func budgetFor(host string) budget {
 63	if b, ok := budgets[host]; ok {
 64		return b
 65	}
 66	return defaultBudget
 67}
 68
 69// spend is one host's running count, kept as plain timestamps because a few
 70// hundred a day is nothing to hold and an exact window beats a decaying
 71// approximation when the whole point is not to go over.
 72type spend struct {
 73	at []time.Time
 74}
 75
 76// trim drops what has aged out of the longest window, which is what keeps the
 77// slice from growing all day.
 78func (s *spend) trim(now time.Time) {
 79	cut := now.Add(-24 * time.Hour)
 80	i := 0
 81	for i < len(s.at) && s.at[i].Before(cut) {
 82		i++
 83	}
 84	if i > 0 {
 85		s.at = append(s.at[:0], s.at[i:]...)
 86	}
 87}
 88
 89func (s *spend) since(now time.Time, d time.Duration) int {
 90	cut := now.Add(-d)
 91	n := 0
 92	for i := len(s.at) - 1; i >= 0; i-- {
 93		if s.at[i].Before(cut) {
 94			break
 95		}
 96		n++
 97	}
 98	return n
 99}
100
101// left reports how much of each pool remains, and when the tightest spent one
102// frees up. A zero duration means nothing is spent.
103func (s *spend) left(now time.Time, b budget) (minute, hour, day int, free time.Duration) {
104	minute = b.minute - s.since(now, time.Minute)
105	hour = b.hour - s.since(now, time.Hour)
106	day = b.day - s.since(now, 24*time.Hour)
107	if len(s.at) == 0 {
108		return
109	}
110	oldest := func(d time.Duration) time.Duration {
111		cut := now.Add(-d)
112		for _, t := range s.at {
113			if !t.Before(cut) {
114				return time.Until(t.Add(d))
115			}
116		}
117		return 0
118	}
119	switch {
120	case minute <= 0:
121		free = oldest(time.Minute)
122	case hour <= 0:
123		free = oldest(time.Hour)
124	case day <= 0:
125		free = oldest(24 * time.Hour)
126	}
127	return
128}
129
130// Budgets holds every host's spend. It is separate from the penalty box: the
131// box is what a host told us, and this is what we decided to allow ourselves.
132type Budgets struct {
133	mu  sync.Mutex
134	by  map[string]*spend
135	now func() time.Time
136}
137
138func NewBudgets() *Budgets {
139	return &Budgets{by: map[string]*spend{}, now: time.Now}
140}
141
142func (b *Budgets) get(host string) *spend {
143	s, ok := b.by[host]
144	if !ok {
145		s = &spend{}
146		b.by[host] = s
147	}
148	return s
149}
150
151// Take records one call against a host, or refuses when a pool is spent. The
152// refusal names which ceiling and when it frees, because "search is off" with
153// no reason reads like a bug.
154// A nil Budgets allows everything, so a Deps built by hand in a test does not
155// panic in the one place every outbound call goes through.
156func (b *Budgets) Take(host string) error {
157	if b == nil {
158		return nil
159	}
160	b.mu.Lock()
161	defer b.mu.Unlock()
162	now := b.now()
163	bud := budgetFor(host)
164	s := b.get(host)
165	s.trim(now)
166
167	minute, hour, day, free := s.left(now, bud)
168	switch {
169	case day <= 0:
170		return fmt.Errorf("the daily search budget for %s is spent (%d), and it frees up in %s",
171			host, bud.day, round(free))
172	case hour <= 0:
173		return fmt.Errorf("the hourly search budget for %s is spent (%d), and it frees up in %s",
174			host, bud.hour, round(free))
175	case minute <= 0:
176		return fmt.Errorf("this minute's search budget for %s is spent (%d), and it frees up in %s",
177			host, bud.minute, round(free))
178	}
179	s.at = append(s.at, now)
180	return nil
181}
182
183// Left is what the page reports, so a reader can see the pool draining rather
184// than finding out when it is gone.
185func (b *Budgets) Left(host string) (minute, hour, day int, free time.Duration) {
186	if b == nil {
187		return 0, 0, 0, 0
188	}
189	b.mu.Lock()
190	defer b.mu.Unlock()
191	now := b.now()
192	s := b.get(host)
193	s.trim(now)
194	return s.left(now, budgetFor(host))
195}
196
197// Restore replays the timestamps the last process wrote, so a deploy does not
198// hand the model a fresh day's allowance.
199func (b *Budgets) Restore(host string, at []time.Time) {
200	if b == nil {
201		return
202	}
203	b.mu.Lock()
204	defer b.mu.Unlock()
205	s := b.get(host)
206	s.at = append(s.at, at...)
207	s.trim(b.now())
208}
209
210// Spent is what to persist, so the counts survive a restart the way the penalty
211// box does.
212func (b *Budgets) Spent(host string) []time.Time {
213	if b == nil {
214		return nil
215	}
216	b.mu.Lock()
217	defer b.mu.Unlock()
218	s := b.get(host)
219	s.trim(b.now())
220	out := make([]time.Time, len(s.at))
221	copy(out, s.at)
222	return out
223}
224
225func round(d time.Duration) time.Duration {
226	if d > time.Minute {
227		return d.Round(time.Minute)
228	}
229	return d.Round(time.Second)
230}