orchard

  1package tools
  2
  3import (
  4	"context"
  5	"fmt"
  6	"html"
  7	"net/url"
  8	"regexp"
  9	"strings"
 10)
 11
 12var (
 13	ddgResult = regexp.MustCompile(`(?is)<a rel="nofollow" class="result__a" href="(.*?)".*?>(.*?)</a>.*?class="result__snippet".*?>(.*?)</a>`)
 14	tagStrip  = regexp.MustCompile(`(?is)<[^>]+>`)
 15	// RE2 has no backreferences, so a paired tag needs its own expression
 16	// rather than one alternation closing on \1.
 17	blockTags = func() []*regexp.Regexp {
 18		var out []*regexp.Regexp
 19		for _, t := range []string{"script", "style", "nav", "header", "footer", "aside", "form", "svg", "noscript", "template"} {
 20			out = append(out, regexp.MustCompile(`(?is)<`+t+`[^>]*>.*?</`+t+`\s*>`))
 21		}
 22		return out
 23	}()
 24	articleRe = []*regexp.Regexp{
 25		regexp.MustCompile(`(?is)<article[^>]*>(.*?)</article\s*>`),
 26		regexp.MustCompile(`(?is)<main[^>]*>(.*?)</main\s*>`),
 27	}
 28	spaces   = regexp.MustCompile(`[ \t]+`)
 29	blankRun = regexp.MustCompile(`\n{3,}`)
 30)
 31
 32// Text pulls readable prose out of an HTML page. Forty lines of standard
 33// library beat a readability port on real article pages when this was measured
 34// for search, and it keeps the byline and the date that readability throws away.
 35func Text(h string) string {
 36	for _, re := range blockTags {
 37		h = re.ReplaceAllString(h, " ")
 38	}
 39	// The body of an <article> or <main> is the page, and everything around
 40	// it is furniture. Falling through with the whole document is fine when
 41	// neither is present.
 42	for _, re := range articleRe {
 43		if m := re.FindStringSubmatch(h); m != nil && len(m[1]) > 200 {
 44			h = m[1]
 45			break
 46		}
 47	}
 48	h = regexp.MustCompile(`(?i)</(p|div|li|h[1-6]|tr|section)>`).ReplaceAllString(h, "\n")
 49	h = regexp.MustCompile(`(?i)<br\s*/?>`).ReplaceAllString(h, "\n")
 50	h = tagStrip.ReplaceAllString(h, " ")
 51	h = html.UnescapeString(h)
 52	h = spaces.ReplaceAllString(h, " ")
 53	var keep []string
 54	for _, line := range strings.Split(h, "\n") {
 55		if l := strings.TrimSpace(line); l != "" {
 56			keep = append(keep, l)
 57		}
 58	}
 59	return blankRun.ReplaceAllString(strings.Join(keep, "\n"), "\n\n")
 60}
 61
 62type SearchHit struct {
 63	Title   string `json:"title"`
 64	URL     string `json:"url"`
 65	Snippet string `json:"snippet"`
 66}
 67
 68var WebSearch = Tool{
 69	Name: "web_search",
 70	Description: "Search the web and get back titles, urls and snippets. Use it for anything current, " +
 71		"local, priced or contested. Snippets are short, so follow up with web_fetch when you need what a page actually says.",
 72	Schema: obj(map[string]any{
 73		"query": str("what to search for, as a person would type it"),
 74		"n":     integer("how many results to return, default 6"),
 75	}, "query"),
 76	Run: func(ctx context.Context, d *Deps, a map[string]any) (any, error) {
 77		q := argStr(a, "query")
 78		if q == "" {
 79			return nil, fmt.Errorf("query is required")
 80		}
 81		n := int(argNum(a, "n", 6))
 82		if n < 1 || n > 12 {
 83			n = 6
 84		}
 85		body, err := get(ctx, d, "https://html.duckduckgo.com/html/?q="+url.QueryEscape(q), "text/html")
 86		if err != nil {
 87			return nil, err
 88		}
 89		hits := parseDDG(string(body), n)
 90		if len(hits) == 0 {
 91			return nil, fmt.Errorf("the search engine returned nothing, which usually means it is refusing us")
 92		}
 93		return map[string]any{"query": q, "results": hits}, nil
 94	},
 95}
 96
 97// parseDDG pulls the results out of a DuckDuckGo HTML page. A limit of zero
 98// means every one, which is what a caller filtering them itself needs.
 99func parseDDG(body string, limit int) []SearchHit {
100	var hits []SearchHit
101	for _, m := range ddgResult.FindAllStringSubmatch(body, -1) {
102		href := m[1]
103		// DuckDuckGo wraps results in its own redirector, and the real address
104		// is the uddg parameter inside it.
105		if strings.HasPrefix(href, "//duckduckgo.com/l/") {
106			if u, e := url.Parse("https:" + href); e == nil {
107				if real := u.Query().Get("uddg"); real != "" {
108					href = real
109				}
110			}
111		}
112		hits = append(hits, SearchHit{
113			Title:   strings.TrimSpace(Text(m[2])),
114			URL:     href,
115			Snippet: strings.TrimSpace(Text(m[3])),
116		})
117		if limit > 0 && len(hits) >= limit {
118			break
119		}
120	}
121	return hits
122}
123
124var WebFetch = Tool{
125	Name:        "web_fetch",
126	Description: "Fetch one url and return its readable text. Use it after web_search when a snippet is not enough.",
127	Schema: obj(map[string]any{
128		"url":       str("the full address, including https://"),
129		"max_chars": integer("how much text to return, default 6000"),
130	}, "url"),
131	Run: func(ctx context.Context, d *Deps, a map[string]any) (any, error) {
132		u, err := publicURL(argStr(a, "url"))
133		if err != nil {
134			return nil, err
135		}
136		max := int(argNum(a, "max_chars", 12000))
137		if max < 500 || max > 24000 {
138			max = 12000
139		}
140		body, err := get(ctx, d, u, "text/html")
141		if err != nil {
142			return nil, err
143		}
144		txt := Text(string(body))
145		out := map[string]any{"url": u, "text": txt, "chars": len(txt)}
146		if len(txt) > max {
147			out["text"] = txt[:max]
148			out["truncated"] = true
149			out["note"] = "This is the start of a long page and it is usually enough to answer from. " +
150				"Fetching the same url again returns the same text, so do not repeat this call."
151		}
152		if len(strings.TrimSpace(txt)) < 400 {
153			out["note"] = "This page returned very little readable text, which usually means it needs " +
154				"JavaScript or is behind a wall. Fetching it again will not help. Say so, or try another source."
155		}
156		return out, nil
157	},
158}