orchard

  1package main
  2
  3import (
  4	"bytes"
  5	"crypto/sha256"
  6	"encoding/hex"
  7	"encoding/json"
  8	"net/url"
  9	"strings"
 10
 11	"golang.org/x/net/html"
 12	"golang.org/x/net/html/atom"
 13)
 14
 15// ParsedHTML is everything the checks need from one page. The JSON tags match
 16// stored data read back from older crawls, so renaming one is a migration.
 17type ParsedHTML struct {
 18	Title       string              `json:"title"`
 19	Description string              `json:"description"`
 20	Canonical   string              `json:"canonical"`
 21	RobotsMeta  string              `json:"robots_meta"`
 22	Viewport    string              `json:"viewport"`
 23	Lang        string              `json:"lang"`
 24	OG          OpenGraph           `json:"og"`
 25	Twitter     TwitterCard         `json:"twitter"`
 26	Headings    map[string][]string `json:"headings"`
 27	Links       []Link              `json:"links"`
 28	Images      []Image             `json:"images"`
 29	Resources   []string            `json:"resources"`
 30	JSONLD      []json.RawMessage   `json:"json_ld"`
 31	JSONLDBad   int                 `json:"json_ld_bad"`
 32	Favicon     string              `json:"favicon"`
 33	Forms       []Form              `json:"forms"`
 34	WordCount   int                 `json:"word_count"`
 35	TextHash    string              `json:"text_hash"`
 36}
 37
 38type OpenGraph struct {
 39	Title       string `json:"title"`
 40	Description string `json:"description"`
 41	Image       string `json:"image"`
 42	URL         string `json:"url"`
 43}
 44
 45type TwitterCard struct {
 46	Card        string `json:"card"`
 47	Title       string `json:"title"`
 48	Description string `json:"description"`
 49}
 50
 51type Link struct {
 52	URL  string   `json:"url"`
 53	Text string   `json:"text"`
 54	Rel  []string `json:"rel"`
 55}
 56
 57// Image keeps a missing alt attribute distinct from an empty one, since
 58// `alt=""` is the right markup for a decorative image.
 59type Image struct {
 60	Src string  `json:"src"`
 61	Alt *string `json:"alt"`
 62}
 63
 64type Form struct {
 65	Action    string      `json:"action"`
 66	Inputs    []FormInput `json:"inputs"`
 67	LabelFors []string    `json:"label_fors"`
 68}
 69
 70type FormInput struct {
 71	Type      string  `json:"type"`
 72	Name      *string `json:"name"`
 73	ID        *string `json:"id"`
 74	AriaLabel *string `json:"aria_label"`
 75}
 76
 77func attr(n *html.Node, name string) (string, bool) {
 78	for _, a := range n.Attr {
 79		if a.Key == name {
 80			return a.Val, true
 81		}
 82	}
 83	return "", false
 84}
 85
 86func attrOr(n *html.Node, name, fallback string) string {
 87	if v, ok := attr(n, name); ok {
 88		return v
 89	}
 90	return fallback
 91}
 92
 93func attrPtr(n *html.Node, name string) *string {
 94	if v, ok := attr(n, name); ok {
 95		return &v
 96	}
 97	return nil
 98}
 99
100// collapse squeezes any run of whitespace down to one space, which is what
101// every comparison in the checks assumes it has been given.
102func collapse(s string) string {
103	return strings.Join(strings.Fields(s), " ")
104}
105
106func textOf(n *html.Node) string {
107	var b strings.Builder
108	var walk func(*html.Node)
109	walk = func(node *html.Node) {
110		if node.Type == html.ElementNode {
111			switch node.DataAtom {
112			case atom.Script, atom.Style, atom.Noscript:
113				return
114			}
115		}
116		if node.Type == html.TextNode {
117			b.WriteString(node.Data)
118			// A separator, because "<b>foo</b><b>bar</b>" is two words on the
119			// page and would otherwise hash and count as one.
120			b.WriteByte(' ')
121		}
122		for c := node.FirstChild; c != nil; c = c.NextSibling {
123			walk(c)
124		}
125	}
126	walk(n)
127	return collapse(b.String())
128}
129
130// rawTextOf concatenates a node's direct text children without textOf's skip
131// list, for <script type="application/ld+json"> whose contents are data.
132func rawTextOf(n *html.Node) string {
133	var b strings.Builder
134	for c := n.FirstChild; c != nil; c = c.NextSibling {
135		if c.Type == html.TextNode {
136			b.WriteString(c.Data)
137		}
138	}
139	return b.String()
140}
141
142func resolve(base *url.URL, ref string) string {
143	ref = strings.TrimSpace(ref)
144	if ref == "" {
145		return ""
146	}
147	u, err := base.Parse(ref)
148	if err != nil {
149		return ref
150	}
151	return u.String()
152}
153
154func parseHTML(body []byte, pageURL string) (*ParsedHTML, error) {
155	doc, err := html.Parse(bytes.NewReader(body))
156	if err != nil {
157		return nil, err
158	}
159	base, err := url.Parse(pageURL)
160	if err != nil {
161		return nil, err
162	}
163
164	p := &ParsedHTML{
165		Headings:  map[string][]string{},
166		Links:     []Link{},
167		Images:    []Image{},
168		Resources: []string{},
169		JSONLD:    []json.RawMessage{},
170		Forms:     []Form{},
171	}
172	for _, level := range []string{"h1", "h2", "h3", "h4", "h5", "h6"} {
173		p.Headings[level] = []string{}
174	}
175
176	var walk func(*html.Node)
177	walk = func(n *html.Node) {
178		if n.Type != html.ElementNode {
179			for c := n.FirstChild; c != nil; c = c.NextSibling {
180				walk(c)
181			}
182			return
183		}
184
185		switch n.DataAtom {
186		case atom.Html:
187			p.Lang = strings.TrimSpace(attrOr(n, "lang", ""))
188
189		case atom.Title:
190			if p.Title == "" {
191				p.Title = textOf(n)
192			}
193
194		case atom.Meta:
195			content := strings.TrimSpace(attrOr(n, "content", ""))
196			// Both spellings, because og: uses property= and twitter: uses
197			// name=, and plenty of real pages have those backwards.
198			switch strings.ToLower(attrOr(n, "name", "")) {
199			case "description":
200				setIfEmpty(&p.Description, content)
201			case "robots":
202				setIfEmpty(&p.RobotsMeta, content)
203			case "viewport":
204				setIfEmpty(&p.Viewport, content)
205			case "twitter:card":
206				setIfEmpty(&p.Twitter.Card, content)
207			case "twitter:title":
208				setIfEmpty(&p.Twitter.Title, content)
209			case "twitter:description":
210				setIfEmpty(&p.Twitter.Description, content)
211			}
212			switch strings.ToLower(attrOr(n, "property", "")) {
213			case "og:title":
214				setIfEmpty(&p.OG.Title, content)
215			case "og:description":
216				setIfEmpty(&p.OG.Description, content)
217			case "og:image":
218				setIfEmpty(&p.OG.Image, content)
219			case "og:url":
220				setIfEmpty(&p.OG.URL, content)
221			}
222
223		case atom.Link:
224			href := attrOr(n, "href", "")
225			rels := strings.Fields(strings.ToLower(attrOr(n, "rel", "")))
226			for _, rel := range rels {
227				if rel == "canonical" && p.Canonical == "" {
228					p.Canonical = resolve(base, href)
229				}
230				// "icon", "shortcut icon" and "apple-touch-icon" all contain
231				// "icon", so an apple-touch-icon alone is not flagged.
232				if strings.Contains(rel, "icon") && p.Favicon == "" && href != "" {
233					p.Favicon = resolve(base, href)
234				}
235			}
236			if href != "" {
237				p.Resources = append(p.Resources, resolve(base, href))
238			}
239
240		case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6:
241			key := n.Data
242			p.Headings[key] = append(p.Headings[key], textOf(n))
243
244		case atom.A:
245			if href, ok := attr(n, "href"); ok {
246				href = strings.TrimSpace(href)
247				if href != "" &&
248					!strings.HasPrefix(href, "#") &&
249					!strings.HasPrefix(strings.ToLower(href), "javascript:") &&
250					!strings.HasPrefix(strings.ToLower(href), "mailto:") &&
251					!strings.HasPrefix(strings.ToLower(href), "tel:") {
252					p.Links = append(p.Links, Link{
253						URL:  resolve(base, href),
254						Text: textOf(n),
255						Rel:  strings.Fields(strings.ToLower(attrOr(n, "rel", ""))),
256					})
257				}
258			}
259
260		case atom.Img:
261			src := strings.TrimSpace(attrOr(n, "src", ""))
262			img := Image{Alt: attrPtr(n, "alt")}
263			if src != "" {
264				img.Src = resolve(base, src)
265				p.Resources = append(p.Resources, img.Src)
266			}
267			p.Images = append(p.Images, img)
268
269		case atom.Script:
270			if strings.EqualFold(attrOr(n, "type", ""), "application/ld+json") {
271				// rawTextOf and not textOf, since textOf skips script elements
272				// entirely, which is right for the word count and wrong here.
273				raw := strings.TrimSpace(rawTextOf(n))
274				if raw != "" {
275					if json.Valid([]byte(raw)) {
276						p.JSONLD = append(p.JSONLD, json.RawMessage(raw))
277					} else {
278						// Counted, not appended as a null, so a page whose
279						// valid JSON-LD is `null` is not a parse failure.
280						p.JSONLDBad++
281					}
282				}
283			}
284			if src := attrOr(n, "src", ""); src != "" {
285				p.Resources = append(p.Resources, resolve(base, src))
286			}
287
288		case atom.Iframe, atom.Source:
289			if src := attrOr(n, "src", ""); src != "" {
290				p.Resources = append(p.Resources, resolve(base, src))
291			}
292
293		case atom.Form:
294			p.Forms = append(p.Forms, parseForm(n, base, pageURL))
295			// Descend anyway, since a form can contain links, images and
296			// headings the other checks want.
297		}
298
299		for c := n.FirstChild; c != nil; c = c.NextSibling {
300			walk(c)
301		}
302	}
303	walk(doc)
304
305	// Parsing has decoded HTML entities by here, so `&nbsp;` does not count as
306	// a word.
307	text := textOf(doc)
308	p.WordCount = len(strings.Fields(text))
309	sum := sha256.Sum256([]byte(text))
310	p.TextHash = hex.EncodeToString(sum[:])
311
312	return p, nil
313}
314
315func setIfEmpty(dst *string, v string) {
316	if *dst == "" {
317		*dst = v
318	}
319}
320
321func parseForm(form *html.Node, base *url.URL, pageURL string) Form {
322	f := Form{Inputs: []FormInput{}, LabelFors: []string{}}
323
324	action, ok := attr(form, "action")
325	if !ok || strings.TrimSpace(action) == "" {
326		// An empty or absent action submits to the current page.
327		f.Action = pageURL
328	} else {
329		f.Action = resolve(base, action)
330	}
331
332	seenLabels := map[string]bool{}
333	var walk func(*html.Node)
334	walk = func(n *html.Node) {
335		if n.Type == html.ElementNode {
336			switch n.DataAtom {
337			case atom.Input, atom.Textarea, atom.Select:
338				f.Inputs = append(f.Inputs, FormInput{
339					// Per the HTML spec an input with no type attribute is a
340					// text input, which is also the one that most needs a label.
341					Type:      strings.ToLower(attrOr(n, "type", "text")),
342					Name:      attrPtr(n, "name"),
343					ID:        attrPtr(n, "id"),
344					AriaLabel: attrPtr(n, "aria-label"),
345				})
346			case atom.Label:
347				if v, ok := attr(n, "for"); ok && !seenLabels[v] {
348					seenLabels[v] = true
349					f.LabelFors = append(f.LabelFors, v)
350				}
351			}
352		}
353		for c := n.FirstChild; c != nil; c = c.NextSibling {
354			walk(c)
355		}
356	}
357	walk(form)
358
359	return f
360}