orchard

  1package main
  2
  3import (
  4	"context"
  5	"net/http"
  6	"sort"
  7	"strings"
  8	"sync"
  9
 10	"golang.org/x/net/html"
 11	"golang.org/x/net/html/atom"
 12)
 13
 14// The sources answer where the research came from, which is not the same as
 15// where the thing itself lives. Asked for the most popular Go project and told
 16// "Ollama", a reader wants a link to Ollama, not only to the listicle that said
 17// so.
 18//
 19// Candidates come from the links already harvested off the fetched pages, so
 20// finding one costs no extra search: a page that names a project nearly always
 21// links to it. Each candidate is then fetched and checked to actually be the
 22// thing before it is shown, because a wrong link is worse than none.
 23
 24// EntityLink is a verified link to something an answer named.
 25type EntityLink struct {
 26	Name  string
 27	URL   string
 28	Title string
 29	Host  string
 30}
 31
 32const (
 33	maxEntities = 4
 34	// Each check is one HTTP request, so the fan-out is bounded.
 35	maxCandidatesPerEntity = 3
 36)
 37
 38// linkEntities finds what the answer named and resolves each to a checked URL.
 39func (e *Engine) linkEntities(ctx context.Context, question, answer string, links []Link) []EntityLink {
 40	if len(links) == 0 {
 41		return nil
 42	}
 43	names := e.namedEntities(ctx, question, answer)
 44	if len(names) == 0 {
 45		return nil
 46	}
 47
 48	var (
 49		mu  sync.Mutex
 50		out []EntityLink
 51		wg  sync.WaitGroup
 52	)
 53	for _, name := range names {
 54		cands := rankCandidates(name, links)
 55		if len(cands) == 0 {
 56			continue
 57		}
 58		wg.Add(1)
 59		go func(name string, cands []Link) {
 60			defer wg.Done()
 61			for _, c := range cands {
 62				if title, ok := verifyLink(ctx, e.client, c.URL, name); ok {
 63					mu.Lock()
 64					out = append(out, EntityLink{
 65						Name: name, URL: c.URL, Title: title, Host: hostname(c.URL),
 66					})
 67					mu.Unlock()
 68					return
 69				}
 70			}
 71		}(name, cands)
 72	}
 73	wg.Wait()
 74
 75	sort.Slice(out, func(a, b int) bool { return out[a].Name < out[b].Name })
 76	return out
 77}
 78
 79// namedEntities asks which specific things the answer named. Schema
 80// constrained, and told to return nothing when the answer names nothing
 81// linkable, which is the common case for a question like the deepest river.
 82func (e *Engine) namedEntities(ctx context.Context, question, answer string) []string {
 83	schema := map[string]any{
 84		"type": "object",
 85		"properties": map[string]any{
 86			"entities": map[string]any{
 87				"type":     "array",
 88				"items":    map[string]any{"type": "string"},
 89				"maxItems": maxEntities,
 90			},
 91		},
 92		"required":             []string{"entities"},
 93		"additionalProperties": false,
 94	}
 95	system := strings.Join([]string{
 96		"You list the specific named things in an answer that a reader would want a link to.",
 97		"Include software projects, products, companies, tools, books, films and organisations, by their exact name.",
 98		"Exclude generic nouns, common concepts, units, places, and anything that is not a specific named thing someone could visit a page for.",
 99		"Return an empty list when the answer names nothing like that. An empty list is a normal and common answer.",
100	}, " ")
101	var out struct {
102		Entities []string `json:"entities"`
103	}
104	if err := e.llm.Structured(ctx, system,
105		"Question: "+question+"\n\nAnswer:\n"+truncate(plainText(answer), 1600),
106		200, schema, &out); err != nil {
107		return nil
108	}
109
110	var names []string
111	seen := map[string]bool{}
112	for _, n := range out.Entities {
113		n = strings.TrimSpace(n)
114		k := strings.ToLower(n)
115		if len(n) < 2 || len(n) > 60 || seen[k] {
116			continue
117		}
118		seen[k] = true
119		names = append(names, n)
120	}
121	return names
122}
123
124// rankCandidates orders the harvested links by how well each looks like the
125// home of the named thing. Anchor text matching first, then the host and path.
126func rankCandidates(name string, links []Link) []Link {
127	type scored struct {
128		link  Link
129		score int
130	}
131	slug := slugify(name)
132	if slug == "" {
133		return nil
134	}
135	lower := strings.ToLower(name)
136
137	var all []scored
138	for _, l := range links {
139		s := 0
140		text := strings.ToLower(l.Text)
141		host := strings.ToLower(hostname(l.URL))
142		path := strings.ToLower(l.URL)
143
144		switch {
145		case text == lower:
146			s += 6
147		case strings.Contains(text, lower):
148			s += 3
149		}
150		// The strongest signal there is: the thing's name is the domain.
151		if strings.HasPrefix(registrable(host), slug+".") || registrable(host) == slug+".com" {
152			s += 6
153		} else if strings.Contains(strings.ReplaceAll(host, "-", ""), slug) {
154			s += 3
155		}
156		if strings.Contains(strings.ReplaceAll(path, "-", ""), slug) {
157			s++
158		}
159		// A repository or a docs page is usually the canonical home for the
160		// kind of thing that gets named in these answers.
161		if strings.Contains(host, "github.com") && strings.Contains(path, slug) {
162			s += 3
163		}
164		if s > 0 {
165			all = append(all, scored{l, s})
166		}
167	}
168	sort.SliceStable(all, func(a, b int) bool { return all[a].score > all[b].score })
169
170	var out []Link
171	for i, c := range all {
172		if i >= maxCandidatesPerEntity || c.score < 3 {
173			break
174		}
175		out = append(out, c.link)
176	}
177	return out
178}
179
180func slugify(s string) string {
181	return strings.Map(func(r rune) rune {
182		switch {
183		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
184			return r
185		case r >= 'A' && r <= 'Z':
186			return r + 32
187		}
188		return -1
189	}, s)
190}
191
192// verifyLink fetches a candidate and confirms the page is about the thing. This
193// is the difference between a link and a guess: a 404, a parked domain, or a
194// page about something else all fail here rather than being handed over.
195func verifyLink(ctx context.Context, client *http.Client, target, name string) (string, bool) {
196	req, err := http.NewRequestWithContext(ctx, "GET", target, nil)
197	if err != nil {
198		return "", false
199	}
200	browserHeaders(req, "")
201	resp, err := client.Do(req)
202	if err != nil {
203		return "", false
204	}
205	defer resp.Body.Close()
206	if resp.StatusCode != http.StatusOK {
207		return "", false
208	}
209	if ct := resp.Header.Get("Content-Type"); ct != "" && !strings.Contains(ct, "html") {
210		return "", false
211	}
212	body, err := readBody(resp, 512<<10)
213	if err != nil {
214		return "", false
215	}
216	doc, err := html.Parse(strings.NewReader(string(body)))
217	if err != nil {
218		return "", false
219	}
220
221	title := metaTitle(doc)
222	slug := slugify(name)
223	// The name has to appear in the title, the description, or the first
224	// heading. A page that never says what it is called is not evidence.
225	haystack := slugify(title) + " " + slugify(metaContent(doc, "og:description")) +
226		" " + slugify(metaContent(doc, "description")) + " " + slugify(firstHeading(doc))
227	if !strings.Contains(strings.ReplaceAll(haystack, " ", ""), slug) {
228		return "", false
229	}
230	return title, true
231}
232
233func firstHeading(root *html.Node) string {
234	var out string
235	var walk func(*html.Node)
236	walk = func(n *html.Node) {
237		if out != "" {
238			return
239		}
240		if n.Type == html.ElementNode && (n.DataAtom == atom.H1 || n.DataAtom == atom.H2) {
241			out = strings.Join(strings.Fields(textContent(n)), " ")
242			return
243		}
244		for c := n.FirstChild; c != nil; c = c.NextSibling {
245			walk(c)
246		}
247	}
248	walk(root)
249	return out
250}