orchard

  1package main
  2
  3import (
  4	"context"
  5	"fmt"
  6	"io"
  7	"log/slog"
  8	"net/http"
  9	"net/url"
 10	"regexp"
 11	"strings"
 12	"time"
 13
 14	"github.com/temoto/robotstxt"
 15)
 16
 17const (
 18	// PageCap is the fence against a calendar or a faceted search that
 19	// generates URLs forever.
 20	PageCap = 500
 21	// Concurrency stays low because this is pointed at somebody else's site,
 22	// sometimes a small one on shared hosting.
 23	Concurrency = 4
 24	// ExternalLinkCap bounds the outbound HEAD probes, since a large site can
 25	// surface thousands of distinct external links.
 26	ExternalLinkCap = 500
 27	// CrawlDeadline sits under the scheduler's fifteen-minute watchdog, so an
 28	// overrunning crawl ends itself and records what it found instead of being
 29	// killed.
 30	CrawlDeadline = 540 * time.Second
 31
 32	// MaxBodyBytes caps one HTML body, because buffering a mislabeled download
 33	// four times over could exhaust the container.
 34	MaxBodyBytes = 5 * 1024 * 1024
 35
 36	crawlerRequestTimeout = 15 * time.Second
 37	externalLinkTimeout   = 8 * time.Second
 38
 39	// A real user agent with a contact URL, unlike the checker's Chrome
 40	// impersonation, so an operator reading their logs knows who to reach.
 41	crawlerUserAgent = "status (+" + baseURL + ")"
 42)
 43
 44type FetchResult struct {
 45	URL          string
 46	RequestedURL string
 47	Status       int
 48	Headers      map[string]string
 49	Body         []byte
 50	ContentType  string
 51	ElapsedMS    int64
 52	RedirectHops int
 53	Err          string
 54}
 55
 56func newCrawlClient() *http.Client {
 57	return &http.Client{
 58		Timeout: crawlerRequestTimeout,
 59		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 60			if len(via) >= 10 {
 61				return http.ErrUseLastResponse
 62			}
 63			return nil
 64		},
 65	}
 66}
 67
 68// fetchPage retrieves one URL and never fails. A failed fetch comes back as
 69// status 0 with the error text, which reads as an unreachable link.
 70func fetchPage(ctx context.Context, client *http.Client, rawURL string) FetchResult {
 71	started := time.Now()
 72	result := FetchResult{URL: rawURL, RequestedURL: rawURL, Headers: map[string]string{}}
 73
 74	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
 75	if err != nil {
 76		result.Err = err.Error()
 77		result.ElapsedMS = time.Since(started).Milliseconds()
 78		return result
 79	}
 80	req.Header.Set("User-Agent", crawlerUserAgent)
 81
 82	resp, err := client.Do(req)
 83	if err != nil {
 84		result.Err = err.Error()
 85		result.ElapsedMS = time.Since(started).Milliseconds()
 86		return result
 87	}
 88	defer resp.Body.Close()
 89
 90	result.URL = canonicalURL(resp.Request.URL)
 91	result.Status = resp.StatusCode
 92	// Go follows redirects inside Do and hands back only the final response,
 93	// with the chain reachable through resp.Request.
 94	for r := resp.Request; r != nil; r = r.Response.Request {
 95		if r.Response == nil {
 96			break
 97		}
 98		result.RedirectHops++
 99	}
100
101	for k, v := range resp.Header {
102		if len(v) > 0 {
103			result.Headers[strings.ToLower(k)] = v[0]
104		}
105	}
106	result.ContentType = strings.ToLower(result.Headers["content-type"])
107
108	// Only HTML bodies are read, so a linked 200MB video costs a round trip
109	// instead of 200MB of memory.
110	if resp.StatusCode == http.StatusOK && strings.Contains(result.ContentType, "text/html") {
111		body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes))
112		if err != nil && len(body) == 0 {
113			result.Err = err.Error()
114		}
115		if len(body) == MaxBodyBytes {
116			slog.Info(fmt.Sprintf("body cap %d hit for %s", MaxBodyBytes, rawURL), slog.String("component", "crawler"))
117		}
118		result.Body = body
119	} else {
120		// Drained so the connection goes back to the pool instead of being
121		// dropped and re-handshaked.
122		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024))
123	}
124
125	result.ElapsedMS = time.Since(started).Milliseconds()
126	return result
127}
128
129// canonicalURL renders a fetched URL with an explicit "/" path, since
130// url.String() keeps the empty path it was given and nothing else spells it
131// that way.
132func canonicalURL(u *url.URL) string {
133	if u.Path == "" {
134		clone := *u
135		clone.Path = "/"
136		return clone.String()
137	}
138	return u.String()
139}
140
141// headStatus probes an external link. A 403, 405 or 501 usually means the
142// server refuses HEAD rather than that the link is broken, so those retry.
143func headStatus(ctx context.Context, client *http.Client, rawURL string) int {
144	ctx, cancel := context.WithTimeout(ctx, externalLinkTimeout)
145	defer cancel()
146
147	try := func(method string) int {
148		req, err := http.NewRequestWithContext(ctx, method, rawURL, nil)
149		if err != nil {
150			return 0
151		}
152		req.Header.Set("User-Agent", crawlerUserAgent)
153		resp, err := client.Do(req)
154		if err != nil {
155			return 0
156		}
157		defer resp.Body.Close()
158		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024))
159		return resp.StatusCode
160	}
161
162	status := try(http.MethodHead)
163	switch status {
164	case http.StatusForbidden, http.StatusMethodNotAllowed, http.StatusNotImplemented:
165		return try(http.MethodGet)
166	}
167	return status
168}
169
170// probeCompression reports the server's Content-Encoding for a URL, or "" when
171// the response came back uncompressed. Accept-Encoding is set by hand because
172// Go's transport otherwise decodes for you and strips the header off.
173func probeCompression(ctx context.Context, client *http.Client, rawURL string) string {
174	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
175	if err != nil {
176		return ""
177	}
178	req.Header.Set("User-Agent", crawlerUserAgent)
179	req.Header.Set("Accept-Encoding", "gzip, br, zstd, deflate")
180
181	resp, err := client.Do(req)
182	if err != nil {
183		return ""
184	}
185	defer resp.Body.Close()
186	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024))
187
188	enc := strings.ToLower(strings.TrimSpace(resp.Header.Get("Content-Encoding")))
189	if enc == "" || enc == "identity" {
190		return ""
191	}
192	return enc
193}
194
195// Robots wraps a parsed robots.txt. A missing or unparseable file allows
196// everything, so a typo cannot produce an empty audit that reads as healthy.
197type Robots struct {
198	group *robotstxt.Group
199}
200
201func (r *Robots) Allowed(rawURL string) bool {
202	if r == nil || r.group == nil {
203		return true
204	}
205	u, err := url.Parse(rawURL)
206	if err != nil {
207		return true
208	}
209	path := u.EscapedPath()
210	if path == "" {
211		path = "/"
212	}
213	if u.RawQuery != "" {
214		path += "?" + u.RawQuery
215	}
216	return r.group.Test(path)
217}
218
219type RobotsCtx struct {
220	URL               string
221	Exists            bool
222	Raw               string
223	ReferencesSitemap bool
224}
225
226func loadRobots(ctx context.Context, client *http.Client, origin string) (*Robots, RobotsCtx) {
227	robotsURL := strings.TrimSuffix(origin, "/") + "/robots.txt"
228	out := RobotsCtx{URL: robotsURL}
229
230	result := fetchPageAllowingText(ctx, client, robotsURL)
231	if result.Status != http.StatusOK || len(result.Body) == 0 {
232		return &Robots{}, out
233	}
234
235	out.Exists = true
236	out.Raw = string(result.Body)
237	for _, line := range strings.Split(out.Raw, "\n") {
238		if strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "sitemap:") {
239			out.ReferencesSitemap = true
240			break
241		}
242	}
243
244	data, err := robotstxt.FromBytes(result.Body)
245	if err != nil {
246		slog.Info(fmt.Sprintf("robots.txt at %s did not parse, allowing everything: %v", robotsURL, err), slog.String("component", "crawler"))
247		return &Robots{}, out
248	}
249	return &Robots{group: data.FindGroup("*")}, out
250}
251
252// fetchPageAllowingText is fetchPage for robots.txt and sitemap.xml, whose
253// bodies fetchPage would throw away as non-HTML.
254func fetchPageAllowingText(ctx context.Context, client *http.Client, rawURL string) FetchResult {
255	started := time.Now()
256	result := FetchResult{URL: rawURL, RequestedURL: rawURL, Headers: map[string]string{}}
257
258	req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
259	if err != nil {
260		result.Err = err.Error()
261		return result
262	}
263	req.Header.Set("User-Agent", crawlerUserAgent)
264
265	resp, err := client.Do(req)
266	if err != nil {
267		result.Err = err.Error()
268		result.ElapsedMS = time.Since(started).Milliseconds()
269		return result
270	}
271	defer resp.Body.Close()
272
273	result.URL = canonicalURL(resp.Request.URL)
274	result.Status = resp.StatusCode
275	result.Body, _ = io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes))
276	result.ElapsedMS = time.Since(started).Milliseconds()
277	return result
278}
279
280var locPattern = regexp.MustCompile(`(?is)<loc>\s*([^<]+?)\s*</loc>`)
281
282// loadSitemap collects the page URLs a site advertises. Sitemaps nest, so it
283// follows index documents too, capped at twenty against a self-referring one.
284func loadSitemap(ctx context.Context, client *http.Client, origin, robotsText string) []string {
285	var queue []string
286	for _, line := range strings.Split(robotsText, "\n") {
287		trimmed := strings.TrimSpace(line)
288		if len(trimmed) < 8 || !strings.EqualFold(trimmed[:8], "sitemap:") {
289			continue
290		}
291		// Sliced off the original and not the lowercased copy, since a URL path
292		// is case sensitive.
293		if target := strings.TrimSpace(trimmed[8:]); target != "" {
294			queue = append(queue, target)
295		}
296	}
297	if len(queue) == 0 {
298		queue = append(queue, strings.TrimSuffix(origin, "/")+"/sitemap.xml")
299	}
300
301	seen := map[string]bool{}
302	var urls []string
303
304	for len(queue) > 0 && len(seen) < 20 {
305		current := queue[0]
306		queue = queue[1:]
307		if seen[current] {
308			continue
309		}
310		seen[current] = true
311
312		result := fetchPageAllowingText(ctx, client, current)
313		if result.Status != http.StatusOK {
314			continue
315		}
316		for _, m := range locPattern.FindAllStringSubmatch(string(result.Body), -1) {
317			loc := strings.TrimSpace(m[1])
318			if loc == "" {
319				continue
320			}
321			lower := strings.ToLower(loc)
322			if strings.HasSuffix(lower, ".xml") || strings.Contains(lower, "sitemap") {
323				queue = append(queue, loc)
324			} else {
325				urls = append(urls, loc)
326			}
327		}
328	}
329	return urls
330}
331
332// sameSite counts the www variant as the same site in both directions, or
333// every link between a site's apex and its www host reports as external.
334func sameSite(rawURL, host string) bool {
335	u, err := url.Parse(rawURL)
336	if err != nil {
337		return false
338	}
339	target := strings.ToLower(u.Hostname())
340	if target == "" {
341		return false
342	}
343	host = strings.ToLower(host)
344	return target == host ||
345		target == "www."+host ||
346		host == "www."+target
347}
348
349// crawlerHostileHosts answer 403 or 404 to anything that is not a browser, so
350// the HEAD probe skips them. Keep the list short, it suppresses real findings.
351var crawlerHostileHosts = map[string]bool{
352	"linkedin.com":     true,
353	"www.linkedin.com": true,
354}
355
356func isCrawlerHostile(rawURL string) bool {
357	u, err := url.Parse(rawURL)
358	if err != nil {
359		return false
360	}
361	host := strings.ToLower(u.Hostname())
362	return crawlerHostileHosts[host] || strings.HasSuffix(host, ".linkedin.com")
363}