orchard

  1package main
  2
  3import (
  4	"bytes"
  5	"encoding/json"
  6	"fmt"
  7	"net/http"
  8	neturl "net/url"
  9	"strings"
 10	"time"
 11
 12	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
 13	"golang.org/x/net/html"
 14	"golang.org/x/net/html/atom"
 15)
 16
 17// Page is a fetched and cleaned document, ready to chunk into passages.
 18type Page struct {
 19	URL       string
 20	Title     string
 21	Site      string
 22	Published string // ISO date, empty when the page carries none
 23	Markdown  string
 24	Links     []Link
 25}
 26
 27// Link is an outbound link found in a page's article body, with the words that
 28// were linked.
 29//
 30// These are harvested rather than searched for. A page that answers "the most
 31// popular Go project" almost always links to the project it names, so the
 32// canonical URL is already in hand and costs no extra search.
 33type Link struct {
 34	URL  string
 35	Text string
 36}
 37
 38// chrome are the elements that never carry article text. Dropping them and then
 39// taking the semantic content element beats a full-page conversion by enough to
 40// matter, and it measures level with a readability port on real article pages.
 41var chrome = map[atom.Atom]bool{
 42	atom.Script: true, atom.Style: true, atom.Nav: true, atom.Header: true,
 43	atom.Footer: true, atom.Aside: true, atom.Form: true, atom.Iframe: true,
 44	atom.Noscript: true, atom.Svg: true, atom.Button: true, atom.Select: true,
 45	atom.Textarea: true, atom.Dialog: true, atom.Template: true,
 46}
 47
 48// Fetch pulls a URL and reduces it to markdown plus whatever metadata the page
 49// was willing to state about itself.
 50func Fetch(client *http.Client, target string) (*Page, error) {
 51	req, err := http.NewRequest("GET", target, nil)
 52	if err != nil {
 53		return nil, err
 54	}
 55	browserHeaders(req, "")
 56
 57	resp, err := client.Do(req)
 58	if err != nil {
 59		return nil, err
 60	}
 61	defer resp.Body.Close()
 62	if resp.StatusCode != http.StatusOK {
 63		return nil, fmt.Errorf("%s: %s", target, resp.Status)
 64	}
 65	if ct := resp.Header.Get("Content-Type"); ct != "" && !strings.Contains(ct, "html") {
 66		return nil, fmt.Errorf("%s: not html (%s)", target, ct)
 67	}
 68
 69	body, err := readBody(resp, 8<<20)
 70	if err != nil {
 71		return nil, err
 72	}
 73	doc, err := html.Parse(bytes.NewReader(body))
 74	if err != nil {
 75		return nil, err
 76	}
 77
 78	p := &Page{URL: target}
 79	p.Title = metaTitle(doc)
 80	p.Site = metaContent(doc, "og:site_name")
 81	if t, ok := pubDate(doc); ok {
 82		p.Published = t.Format("2006-01-02")
 83	}
 84
 85	// Before stripAndPick, which removes script tags in place and takes the
 86	// structured data with them.
 87	recipe, hasRecipe := recipeFromJSONLD(doc)
 88
 89	content := stripAndPick(doc)
 90	p.Links = harvestLinks(content, target)
 91	md, err := htmltomarkdown.ConvertString(renderNode(content))
 92	if err != nil {
 93		return nil, err
 94	}
 95	p.Markdown = tidyMarkdown(md)
 96	if len(strings.Fields(p.Markdown)) < 40 {
 97		return nil, fmt.Errorf("%s: too little text", target)
 98	}
 99	// A recipe page publishes its ingredients with quantities as data and then
100	// describes them without quantities in the prose, so the structured copy
101	// goes first where the passage ranking will find it.
102	if hasRecipe {
103		p.Markdown = recipe.Markdown() + "\n\n" + p.Markdown
104	}
105	return p, nil
106}
107
108// stripAndPick removes page chrome, then returns the semantic content element
109// if there is one. Without a readability port this is what separates an article
110// from the navigation around it, and on real article pages it measures level
111// with one.
112func stripAndPick(root *html.Node) *html.Node {
113	var prune func(*html.Node)
114	prune = func(n *html.Node) {
115		var next *html.Node
116		for c := n.FirstChild; c != nil; c = next {
117			next = c.NextSibling
118			if c.Type == html.ElementNode && chrome[c.DataAtom] {
119				n.RemoveChild(c)
120				continue
121			}
122			if c.Type == html.CommentNode {
123				n.RemoveChild(c)
124				continue
125			}
126			prune(c)
127		}
128	}
129	prune(root)
130
131	var best *html.Node
132	bestLen := 0
133	var walk func(*html.Node)
134	walk = func(n *html.Node) {
135		if n.Type == html.ElementNode && (n.DataAtom == atom.Article || n.DataAtom == atom.Main) {
136			if l := textLen(n); l > bestLen {
137				best, bestLen = n, l
138			}
139		}
140		for c := n.FirstChild; c != nil; c = c.NextSibling {
141			walk(c)
142		}
143	}
144	walk(root)
145	if best != nil && bestLen > 400 {
146		return best
147	}
148	return root
149}
150
151// harvestLinks pulls the outbound links out of an article body. Same-host links
152// are dropped because they are navigation, and so is anything whose anchor text
153// is too short or too long to name a thing.
154func harvestLinks(root *html.Node, pageURL string) []Link {
155	base, err := neturl.Parse(pageURL)
156	if err != nil {
157		return nil
158	}
159	seen := map[string]bool{}
160	var out []Link
161
162	var walk func(*html.Node)
163	walk = func(n *html.Node) {
164		if n.Type == html.ElementNode && n.DataAtom == atom.A {
165			href := ""
166			for _, a := range n.Attr {
167				if a.Key == "href" {
168					href = a.Val
169				}
170			}
171			if u, err := neturl.Parse(href); err == nil && href != "" {
172				abs := base.ResolveReference(u)
173				text := strings.Join(strings.Fields(textContent(n)), " ")
174				if keepLink(abs, base, text) && !seen[abs.String()] {
175					seen[abs.String()] = true
176					out = append(out, Link{URL: abs.String(), Text: text})
177				}
178			}
179		}
180		for c := n.FirstChild; c != nil; c = c.NextSibling {
181			walk(c)
182		}
183	}
184	walk(root)
185	return out
186}
187
188func keepLink(u, base *neturl.URL, text string) bool {
189	if u.Scheme != "http" && u.Scheme != "https" {
190		return false
191	}
192	if u.Host == "" || sameSite(u.Host, base.Host) {
193		return false
194	}
195	if len(text) < 2 || len(text) > 80 {
196		return false
197	}
198	// Sharing widgets and the sites every article links to regardless of what
199	// it is about.
200	for _, junk := range []string{
201		"facebook.com", "twitter.com", "x.com", "instagram.com", "pinterest.com",
202		"linkedin.com", "reddit.com/submit", "t.me/share", "whatsapp.com",
203		"doubleclick", "googletagmanager", "amazon-adsystem",
204	} {
205		if strings.Contains(u.Host+u.Path, junk) {
206			return false
207		}
208	}
209	return true
210}
211
212func sameSite(a, b string) bool {
213	return registrable(a) == registrable(b)
214}
215
216// registrable is a rough eTLD+1. It only has to tell one publisher from
217// another, so the two label rule is enough and a public suffix list would be a
218// dependency for nothing.
219func registrable(host string) string {
220	host = strings.TrimPrefix(strings.ToLower(host), "www.")
221	parts := strings.Split(host, ".")
222	if len(parts) <= 2 {
223		return host
224	}
225	return strings.Join(parts[len(parts)-2:], ".")
226}
227
228func textContent(n *html.Node) string {
229	if n.Type == html.TextNode {
230		return n.Data
231	}
232	var b strings.Builder
233	for c := n.FirstChild; c != nil; c = c.NextSibling {
234		b.WriteString(textContent(c))
235	}
236	return b.String()
237}
238
239func textLen(n *html.Node) int {
240	if n.Type == html.TextNode {
241		return len(strings.TrimSpace(n.Data))
242	}
243	total := 0
244	for c := n.FirstChild; c != nil; c = c.NextSibling {
245		total += textLen(c)
246	}
247	return total
248}
249
250func renderNode(n *html.Node) string {
251	var b bytes.Buffer
252	html.Render(&b, n)
253	return b.String()
254}
255
256// tidyMarkdown drops the wreckage a conversion leaves behind: image lines,
257// tracking parameters, and the runs of blank lines that come from stripped
258// elements.
259func tidyMarkdown(md string) string {
260	lines := strings.Split(md, "\n")
261	out := make([]string, 0, len(lines))
262	blanks := 0
263	for _, l := range lines {
264		t := strings.TrimSpace(l)
265		if t == "" {
266			blanks++
267			if blanks > 1 {
268				continue
269			}
270			out = append(out, "")
271			continue
272		}
273		blanks = 0
274		// A line that is nothing but images carries no text for the model.
275		if strings.HasPrefix(t, "![") && !strings.Contains(t, "](#") && len(stripImages(t)) < 3 {
276			continue
277		}
278		out = append(out, stripTracking(l))
279	}
280	return strings.TrimSpace(strings.Join(out, "\n"))
281}
282
283func stripImages(s string) string {
284	for {
285		i := strings.Index(s, "![")
286		if i < 0 {
287			return strings.TrimSpace(s)
288		}
289		j := strings.Index(s[i:], ")")
290		if j < 0 {
291			return strings.TrimSpace(s[:i])
292		}
293		s = s[:i] + s[i+j+1:]
294	}
295}
296
297var trackingParams = []string{"utm_source", "utm_campaign", "utm_medium", "utm_content", "utm_term"}
298
299func stripTracking(s string) string {
300	for _, p := range trackingParams {
301		for {
302			i := strings.Index(s, "?"+p+"=")
303			if i < 0 {
304				i = strings.Index(s, "&"+p+"=")
305			}
306			if i < 0 {
307				break
308			}
309			end := i + 1
310			for end < len(s) && s[end] != '&' && s[end] != ')' && s[end] != ' ' && s[end] != '"' {
311				end++
312			}
313			s = s[:i] + s[end:]
314		}
315	}
316	return s
317}
318
319// ---- metadata ----
320
321func metaTitle(root *html.Node) string {
322	if v := metaContent(root, "og:title"); v != "" {
323		return v
324	}
325	var out string
326	var walk func(*html.Node)
327	walk = func(n *html.Node) {
328		if out != "" {
329			return
330		}
331		if n.Type == html.ElementNode && n.DataAtom == atom.Title && n.FirstChild != nil {
332			out = strings.Join(strings.Fields(n.FirstChild.Data), " ")
333			return
334		}
335		for c := n.FirstChild; c != nil; c = c.NextSibling {
336			walk(c)
337		}
338	}
339	walk(root)
340	return out
341}
342
343func metaContent(root *html.Node, key string) string {
344	var out string
345	var walk func(*html.Node)
346	walk = func(n *html.Node) {
347		if out != "" {
348			return
349		}
350		if n.Type == html.ElementNode && n.DataAtom == atom.Meta {
351			var k, v string
352			for _, a := range n.Attr {
353				switch a.Key {
354				case "property", "name", "itemprop":
355					k = strings.ToLower(a.Val)
356				case "content":
357					v = a.Val
358				}
359			}
360			if k == strings.ToLower(key) && v != "" {
361				out = v
362				return
363			}
364		}
365		for c := n.FirstChild; c != nil; c = c.NextSibling {
366			walk(c)
367		}
368	}
369	walk(root)
370	return out
371}
372
373// pubDate reads a publication date out of structured metadata. It finds one on
374// roughly half of real pages; the rest have it only as prose in the byline,
375// which is what the model's extraction pass is asked to pick up.
376func pubDate(root *html.Node) (time.Time, bool) {
377	if t, ok := fromJSONLD(root); ok {
378		return t, true
379	}
380	metas := map[string]string{}
381	var times []string
382	var walk func(*html.Node)
383	walk = func(n *html.Node) {
384		if n.Type == html.ElementNode {
385			switch n.DataAtom {
386			case atom.Meta:
387				var k, v string
388				for _, a := range n.Attr {
389					switch a.Key {
390					case "property", "name", "itemprop":
391						k = strings.ToLower(a.Val)
392					case "content":
393						v = a.Val
394					}
395				}
396				if k != "" && v != "" {
397					metas[k] = v
398				}
399			case atom.Time:
400				for _, a := range n.Attr {
401					if a.Key == "datetime" {
402						times = append(times, a.Val)
403					}
404				}
405				if n.FirstChild != nil && n.FirstChild.Type == html.TextNode {
406					times = append(times, n.FirstChild.Data)
407				}
408			}
409		}
410		for c := n.FirstChild; c != nil; c = c.NextSibling {
411			walk(c)
412		}
413	}
414	walk(root)
415
416	for _, k := range []string{
417		"article:published_time", "og:article:published_time", "datepublished",
418		"publishdate", "date", "dc.date", "dc.date.issued", "sailthru.date",
419		"parsely-pub-date", "article:modified_time",
420	} {
421		if t, ok := parseDate(metas[k]); ok {
422			return t, true
423		}
424	}
425	for _, v := range times {
426		if t, ok := parseDate(v); ok {
427			return t, true
428		}
429	}
430	return time.Time{}, false
431}
432
433func fromJSONLD(root *html.Node) (time.Time, bool) {
434	var out time.Time
435	found := false
436	var walk func(*html.Node)
437	walk = func(n *html.Node) {
438		if found {
439			return
440		}
441		if n.Type == html.ElementNode && n.DataAtom == atom.Script {
442			for _, a := range n.Attr {
443				if a.Key == "type" && strings.Contains(a.Val, "ld+json") && n.FirstChild != nil {
444					var v any
445					if json.Unmarshal([]byte(n.FirstChild.Data), &v) == nil {
446						if t, ok := digDate(v); ok {
447							out, found = t, true
448							return
449						}
450					}
451				}
452			}
453		}
454		for c := n.FirstChild; c != nil; c = c.NextSibling {
455			walk(c)
456		}
457	}
458	walk(root)
459	return out, found
460}
461
462func digDate(v any) (time.Time, bool) {
463	switch x := v.(type) {
464	case map[string]any:
465		for _, k := range []string{"datePublished", "dateCreated", "uploadDate"} {
466			if s, ok := x[k].(string); ok {
467				if t, ok := parseDate(s); ok {
468					return t, true
469				}
470			}
471		}
472		for _, sub := range x {
473			if t, ok := digDate(sub); ok {
474				return t, true
475			}
476		}
477	case []any:
478		for _, sub := range x {
479			if t, ok := digDate(sub); ok {
480				return t, true
481			}
482		}
483	}
484	return time.Time{}, false
485}
486
487func parseDate(s string) (time.Time, bool) {
488	s = strings.TrimSpace(s)
489	if s == "" {
490		return time.Time{}, false
491	}
492	for _, f := range []string{
493		time.RFC3339, "2006-01-02T15:04:05Z0700", "2006-01-02T15:04:05",
494		"2006-01-02 15:04:05", "2006-01-02", "01/02/2006",
495		"January 2, 2006", "2 January 2006", "Jan 2, 2006", "2 Jan 2006",
496		"02 January 2006", time.RFC1123, time.RFC1123Z,
497	} {
498		if t, err := time.Parse(f, s); err == nil && t.Year() > 1990 && t.Year() < 2100 {
499			return t, true
500		}
501	}
502	return time.Time{}, false
503}
504
505func abs(v float64) float64 {
506	if v < 0 {
507		return -v
508	}
509	return v
510}