orchard

  1package main
  2
  3import (
  4	"bytes"
  5	"context"
  6	"encoding/json"
  7	"fmt"
  8	"io"
  9	"net/http"
 10	"strconv"
 11	"time"
 12)
 13
 14// userAgent has to look like a browser. Yahoo answers Go's default
 15// "Go-http-client/2.0" with a block page rather than JSON, and the other three
 16// upstreams are friendlier about it but no worse for having one.
 17const userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
 18
 19// The RSS feeds want the opposite. An outlet behind AWS WAF bot control
 20// challenges the string above for claiming to be Chrome while carrying none of
 21// the headers Chrome sends, and answers 202 with an empty body and
 22// "x-amzn-waf-action: challenge". Naming the program passes every feed here.
 23const feedAgent = "dash.bythewood.me (+https://dash.bythewood.me)"
 24
 25// One client for every upstream. These are all plain JSON GETs over TLS to
 26// hosts that keep connections alive, so pooling is what makes a 30 second poll
 27// cost a round trip instead of a handshake.
 28var client = &http.Client{
 29	Timeout: 12 * time.Second,
 30	Transport: &http.Transport{
 31		MaxIdleConnsPerHost: 4,
 32		IdleConnTimeout:     90 * time.Second,
 33	},
 34}
 35
 36// maxBody caps what a compromised or confused upstream can make this process
 37// allocate. The largest real response here is Yahoo's twelve symbol spark at
 38// about 30KB.
 39const maxBody = 4 << 20
 40
 41// getJSON runs one guarded request and decodes it into out. A refusal from the
 42// guard comes back as an error without a request going out, which is the point
 43// of it.
 44func getJSON(ctx context.Context, g *Guard, endpoint, url string, out any) error {
 45	return getJSONHeaders(ctx, g, endpoint, url, nil, out)
 46}
 47
 48// parseRetryAfter reads both forms in RFC 9110 section 10.2.3, a delay in
 49// seconds or an HTTP date. An unparseable value is no value.
 50func parseRetryAfter(v string) time.Duration {
 51	if v == "" {
 52		return 0
 53	}
 54	if secs, err := strconv.Atoi(v); err == nil {
 55		if secs < 0 {
 56			return 0
 57		}
 58		return time.Duration(secs) * time.Second
 59	}
 60	if t, err := http.ParseTime(v); err == nil {
 61		if d := time.Until(t); d > 0 {
 62			return d
 63		}
 64	}
 65	return 0
 66}
 67
 68// getJSONHeaders is getJSON with extra request headers. Nasdaq answers an
 69// Accept it does not recognise with an HTML challenge page rather than JSON.
 70func getJSONHeaders(ctx context.Context, g *Guard, endpoint, url string, headers map[string]string, out any) error {
 71	if err := g.Reserve(ctx, endpoint); err != nil {
 72		return err
 73	}
 74
 75	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 76	if err != nil {
 77		return err
 78	}
 79	req.Header.Set("User-Agent", userAgent)
 80	req.Header.Set("Accept", "application/json")
 81	for k, v := range headers {
 82		req.Header.Set(k, v)
 83	}
 84
 85	resp, err := client.Do(req)
 86	if err != nil {
 87		g.Fail(endpoint, 0, 0)
 88		return err
 89	}
 90	defer resp.Body.Close()
 91
 92	if resp.StatusCode != http.StatusOK {
 93		g.Fail(endpoint, resp.StatusCode, parseRetryAfter(resp.Header.Get("Retry-After")))
 94		return fmt.Errorf("%s: http %d", endpoint, resp.StatusCode)
 95	}
 96
 97	if err := json.NewDecoder(io.LimitReader(resp.Body, maxBody)).Decode(out); err != nil {
 98		g.Fail(endpoint, resp.StatusCode, 0)
 99		return fmt.Errorf("%s: %w", endpoint, err)
100	}
101
102	g.Succeed(endpoint)
103	return nil
104}
105
106// postJSON is getJSON for the one upstream that only speaks GraphQL. Guarded
107// identically, since a POST to somebody else's endpoint is no cheaper than a
108// GET.
109func postJSON(ctx context.Context, g *Guard, endpoint, url string, body, out any) error {
110	if err := g.Reserve(ctx, endpoint); err != nil {
111		return err
112	}
113
114	encoded, err := json.Marshal(body)
115	if err != nil {
116		return err
117	}
118
119	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(encoded))
120	if err != nil {
121		return err
122	}
123	req.Header.Set("User-Agent", userAgent)
124	req.Header.Set("Accept", "application/json")
125	req.Header.Set("Content-Type", "application/json")
126
127	resp, err := client.Do(req)
128	if err != nil {
129		g.Fail(endpoint, 0, 0)
130		return err
131	}
132	defer resp.Body.Close()
133
134	if resp.StatusCode != http.StatusOK {
135		g.Fail(endpoint, resp.StatusCode, parseRetryAfter(resp.Header.Get("Retry-After")))
136		return fmt.Errorf("%s: http %d", endpoint, resp.StatusCode)
137	}
138
139	if err := json.NewDecoder(io.LimitReader(resp.Body, maxBody)).Decode(out); err != nil {
140		g.Fail(endpoint, resp.StatusCode, 0)
141		return fmt.Errorf("%s: %w", endpoint, err)
142	}
143
144	g.Succeed(endpoint)
145	return nil
146}