orchard

 1package main
 2
 3import (
 4	"compress/gzip"
 5	"io"
 6	"net/http"
 7	"net/http/cookiejar"
 8	"strings"
 9	"time"
10)
11
12// Looking like a browser is not about the User-Agent alone. A request carrying
13// one browser header and nothing else is a more obvious scraper than one
14// carrying none, because no real browser has ever sent that combination. So
15// this sends the whole set Chrome sends, in Chrome's order, keeps cookies
16// across requests the way a browser does, and asks for compression it can
17// actually decode.
18//
19// Keep the version current. An outdated Chrome is itself a fingerprint, and
20// this was found advertising Chrome 131 in September 2026, nearly a year stale.
21const (
22	chromeMajor = "152"
23	browserUA   = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " +
24		"Chrome/" + chromeMajor + ".0.0.0 Safari/537.36"
25	acceptHTML = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8"
26	clientHint = `"Chromium";v="` + chromeMajor + `", "Google Chrome";v="` + chromeMajor + `", "Not?A_Brand";v="24"`
27)
28
29// newHTTPClient carries a cookie jar. DuckDuckGo sets preference cookies on a
30// first visit and a client that never returns them looks like a fresh stranger
31// on every query, which is exactly the pattern rate limiting looks for.
32func newHTTPClient() *http.Client {
33	jar, _ := cookiejar.New(nil)
34	return &http.Client{
35		Timeout: 25 * time.Second,
36		Jar:     jar,
37	}
38}
39
40// browserHeaders sets what Chrome sends on a top-level navigation.
41func browserHeaders(req *http.Request, referer string) {
42	h := req.Header
43	h.Set("sec-ch-ua", clientHint)
44	h.Set("sec-ch-ua-mobile", "?0")
45	h.Set("sec-ch-ua-platform", `"Windows"`)
46	h.Set("Upgrade-Insecure-Requests", "1")
47	h.Set("User-Agent", browserUA)
48	h.Set("Accept", acceptHTML)
49	h.Set("Sec-Fetch-Site", "none")
50	h.Set("Sec-Fetch-Mode", "navigate")
51	h.Set("Sec-Fetch-User", "?1")
52	h.Set("Sec-Fetch-Dest", "document")
53	// Only gzip and deflate are advertised because those are what this can
54	// decode. Claiming br and zstd and then failing to read them is worse than
55	// not claiming them.
56	h.Set("Accept-Encoding", "gzip, deflate")
57	h.Set("Accept-Language", "en-US,en;q=0.9")
58	if referer != "" {
59		h.Set("Referer", referer)
60		h.Set("Sec-Fetch-Site", "same-origin")
61	}
62}
63
64// readBody transparently decompresses. Setting Accept-Encoding by hand turns
65// off Go's automatic gzip handling, so this has to do it.
66func readBody(resp *http.Response, limit int64) ([]byte, error) {
67	var r io.Reader = resp.Body
68	if strings.Contains(strings.ToLower(resp.Header.Get("Content-Encoding")), "gzip") {
69		zr, err := gzip.NewReader(resp.Body)
70		if err != nil {
71			return nil, err
72		}
73		defer zr.Close()
74		r = zr
75	}
76	return io.ReadAll(io.LimitReader(r, limit))
77}