orchard

  1package main
  2
  3import (
  4	"errors"
  5	"fmt"
  6	"net/http"
  7	"net/url"
  8	"strings"
  9	"time"
 10
 11	"golang.org/x/net/html"
 12)
 13
 14// Result is one hit off a search engine, before anything has been fetched.
 15type Result struct {
 16	URL     string
 17	Title   string
 18	Snippet string
 19}
 20
 21// SearchDDG runs one query against DuckDuckGo, retrying a rate limit.
 22//
 23// 202 is how DuckDuckGo says slow down, and it clears on its own in a few
 24// seconds, so backing off beats failing the whole question. It gives up rather
 25// than hammering, and the SERP cache is what keeps this rare in normal use.
 26func SearchDDG(client *http.Client, query string, limit int) ([]Result, error) {
 27	var err error
 28	for attempt, wait := range []time.Duration{0, 3 * time.Second, 8 * time.Second} {
 29		if wait > 0 {
 30			time.Sleep(wait)
 31		}
 32		var out []Result
 33		out, err = searchOnce(client, query, limit)
 34		if err == nil {
 35			return out, nil
 36		}
 37		if !errors.Is(err, errRateLimited) {
 38			return nil, err
 39		}
 40		_ = attempt
 41	}
 42	return nil, err
 43}
 44
 45var errRateLimited = errors.New("the search source is asking us to slow down, try again in a minute")
 46
 47func searchOnce(client *http.Client, query string, limit int) ([]Result, error) {
 48	// GET rather than POST. A person reaching this page arrives by navigating
 49	// to it, and POST is what a scraper does.
 50	endpoint := "https://html.duckduckgo.com/html/?" + url.Values{
 51		"q": {query}, "kl": {"us-en"},
 52	}.Encode()
 53	req, err := http.NewRequest("GET", endpoint, nil)
 54	if err != nil {
 55		return nil, err
 56	}
 57	browserHeaders(req, "https://duckduckgo.com/")
 58
 59	resp, err := client.Do(req)
 60	if err != nil {
 61		return nil, err
 62	}
 63	defer resp.Body.Close()
 64
 65	if resp.StatusCode == http.StatusAccepted {
 66		return nil, errRateLimited
 67	}
 68	if resp.StatusCode != http.StatusOK {
 69		return nil, fmt.Errorf("search source: %s", resp.Status)
 70	}
 71
 72	body, err := readBody(resp, 4<<20)
 73	if err != nil {
 74		return nil, err
 75	}
 76	doc, err := html.Parse(strings.NewReader(string(body)))
 77	if err != nil {
 78		return nil, err
 79	}
 80	return parseDDG(doc, limit), nil
 81}
 82
 83func parseDDG(doc *html.Node, limit int) []Result {
 84	var out []Result
 85	seen := map[string]bool{}
 86
 87	var walk func(*html.Node)
 88	walk = func(n *html.Node) {
 89		if len(out) >= limit {
 90			return
 91		}
 92		if n.Type == html.ElementNode && n.Data == "div" && hasClass(n, "result") && !hasClass(n, "result--ad") {
 93			r := Result{}
 94			var scan func(*html.Node)
 95			scan = func(m *html.Node) {
 96				if m.Type == html.ElementNode && m.Data == "a" {
 97					switch {
 98					case hasClass(m, "result__a"):
 99						r.Title = textOf(m)
100						r.URL = cleanDDGHref(attr(m, "href"))
101					case hasClass(m, "result__snippet") && r.Snippet == "":
102						r.Snippet = textOf(m)
103					}
104				}
105				if m.Type == html.ElementNode && m.Data == "div" && hasClass(m, "result__snippet") && r.Snippet == "" {
106					r.Snippet = textOf(m)
107				}
108				for c := m.FirstChild; c != nil; c = c.NextSibling {
109					scan(c)
110				}
111			}
112			scan(n)
113			if r.URL != "" && !seen[r.URL] && strings.HasPrefix(r.URL, "http") {
114				seen[r.URL] = true
115				out = append(out, r)
116			}
117			return
118		}
119		for c := n.FirstChild; c != nil; c = c.NextSibling {
120			walk(c)
121		}
122	}
123	walk(doc)
124	return out
125}
126
127// cleanDDGHref unwraps DDG's /l/?uddg= redirector, which is what the HTML
128// endpoint returns rather than the destination.
129func cleanDDGHref(href string) string {
130	if href == "" {
131		return ""
132	}
133	if strings.HasPrefix(href, "//") {
134		href = "https:" + href
135	}
136	u, err := url.Parse(href)
137	if err != nil {
138		return ""
139	}
140	if q := u.Query().Get("uddg"); q != "" {
141		return q
142	}
143	return u.String()
144}
145
146func hasClass(n *html.Node, class string) bool {
147	for _, f := range strings.Fields(attr(n, "class")) {
148		if f == class {
149			return true
150		}
151	}
152	return false
153}
154
155func attr(n *html.Node, key string) string {
156	for _, a := range n.Attr {
157		if a.Key == key {
158			return a.Val
159		}
160	}
161	return ""
162}
163
164func textOf(n *html.Node) string {
165	var b strings.Builder
166	var walk func(*html.Node)
167	walk = func(m *html.Node) {
168		if m.Type == html.TextNode {
169			b.WriteString(m.Data)
170		}
171		for c := m.FirstChild; c != nil; c = c.NextSibling {
172			walk(c)
173		}
174	}
175	walk(n)
176	return strings.Join(strings.Fields(b.String()), " ")
177}