orchard

  1package main
  2
  3import (
  4	"context"
  5	"fmt"
  6	"log/slog"
  7	"net/http"
  8	"net/url"
  9	"sort"
 10	"strings"
 11	"sync"
 12	"time"
 13)
 14
 15// Page is one crawled URL plus, when it was HTML, everything parsed out of it.
 16type Page struct {
 17	URL          string
 18	RequestedURL string
 19	Status       int
 20	ContentType  string
 21	ElapsedMS    int64
 22	Bytes        int
 23	Headers      map[string]string
 24	RedirectHops int
 25	Err          string
 26	IsHTML       bool
 27	HTML         *ParsedHTML
 28}
 29
 30// CrawlResult is everything one crawl learned, before any of it is judged.
 31type CrawlResult struct {
 32	StartURL           string
 33	Host               string
 34	Pages              []*Page
 35	ExternalLinkStatus map[string]int
 36	SitemapURLs        []string
 37	Robots             RobotsCtx
 38	// Compression is the start URL's Content-Encoding, or "" if it answered
 39	// uncompressed.
 40	Compression string
 41}
 42
 43// normalizeURL is the key pages are deduplicated by. Dropping the trailing
 44// slash can merge two paths a server really does serve differently, but
 45// keeping it crawls most sites twice.
 46func normalizeURL(raw string) string {
 47	u, err := url.Parse(raw)
 48	if err != nil {
 49		return raw
 50	}
 51	u.Fragment = ""
 52	s := u.String()
 53	if trimmed := strings.TrimSuffix(s, "/"); trimmed != "" {
 54		return trimmed
 55	}
 56	return s
 57}
 58
 59// RunSEOSpider crawls a site and returns the insights. progress, if non-nil,
 60// is called with the running page count after each batch.
 61func RunSEOSpider(ctx context.Context, startURL string, progress func(pages int)) ([]Insight, error) {
 62	started := time.Now()
 63	slog.Info(fmt.Sprintf("starting %s", startURL), slog.String("component", "crawler"))
 64
 65	result, err := crawl(ctx, startURL, progress)
 66	if err != nil {
 67		return nil, err
 68	}
 69	insights := runChecks(result)
 70
 71	slog.Info(fmt.Sprintf("done %s: %d pages, %d insights, %.1fs",
 72		startURL, len(result.Pages), len(insights), time.Since(started).Seconds()), slog.String("component", "crawler"))
 73	return insights, nil
 74}
 75
 76func crawl(ctx context.Context, startURL string, progress func(pages int)) (*CrawlResult, error) {
 77	start, err := parseHTTPURL(startURL)
 78	if err != nil {
 79		return nil, err
 80	}
 81	host := start.Hostname()
 82	origin := start.Scheme + "://" + start.Host
 83
 84	ctx, cancel := context.WithTimeout(ctx, CrawlDeadline)
 85	defer cancel()
 86
 87	client := newCrawlClient()
 88
 89	compression := probeCompression(ctx, client, startURL)
 90	robots, robotsCtx := loadRobots(ctx, client, origin)
 91	sitemapURLs := loadSitemap(ctx, client, origin, robotsCtx.Raw)
 92
 93	var (
 94		seen    = map[string]bool{}
 95		fetched = map[string]bool{}
 96		queue   []string
 97		pages   []*Page
 98	)
 99
100	enqueue := func(raw string) {
101		key := normalizeURL(raw)
102		if seen[key] {
103			return
104		}
105		seen[key] = true
106		queue = append(queue, raw)
107	}
108
109	enqueue(startURL)
110	// The sitemap is a second entry point and not just a checklist, since it
111	// surfaces pages nothing links to.
112	for i, u := range sitemapURLs {
113		if i >= PageCap {
114			break
115		}
116		if sameSite(u, host) {
117			enqueue(u)
118		}
119	}
120
121	hitDeadline := false
122	for len(queue) > 0 && len(pages) < PageCap {
123		if ctx.Err() != nil {
124			hitDeadline = true
125			break
126		}
127
128		var batch []string
129		for len(queue) > 0 && len(batch) < Concurrency && len(pages)+len(batch) < PageCap {
130			next := queue[0]
131			queue = queue[1:]
132			if !robots.Allowed(next) {
133				continue
134			}
135			batch = append(batch, next)
136		}
137		if len(batch) == 0 {
138			break
139		}
140
141		results := make([]FetchResult, len(batch))
142		var wg sync.WaitGroup
143		for i, u := range batch {
144			wg.Add(1)
145			go func(i int, u string) {
146				defer wg.Done()
147				results[i] = fetchPage(ctx, client, u)
148			}(i, u)
149		}
150		wg.Wait()
151
152		for _, r := range results {
153			// A redirect can land two queued URLs on the same page, so dedupe
154			// on the final URL, which the queue cannot know yet.
155			finalKey := normalizeURL(r.URL)
156			if fetched[finalKey] {
157				seen[finalKey] = true
158				continue
159			}
160			fetched[finalKey] = true
161			seen[finalKey] = true
162
163			page := &Page{
164				URL:          r.URL,
165				RequestedURL: r.RequestedURL,
166				Status:       r.Status,
167				ContentType:  r.ContentType,
168				ElapsedMS:    r.ElapsedMS,
169				Bytes:        len(r.Body),
170				Headers:      r.Headers,
171				RedirectHops: r.RedirectHops,
172				Err:          r.Err,
173				IsHTML:       r.Status == 200 && strings.Contains(r.ContentType, "text/html"),
174			}
175
176			if page.IsHTML {
177				parsed, err := parseHTML(r.Body, r.URL)
178				if err != nil {
179					slog.Error(fmt.Sprintf("parse failed for %s: %v", r.URL, err), slog.String("component", "crawler"))
180					page.IsHTML = false
181				} else {
182					page.HTML = parsed
183					for _, link := range parsed.Links {
184						if sameSite(link.URL, host) {
185							enqueue(link.URL)
186						}
187					}
188				}
189			}
190
191			pages = append(pages, page)
192		}
193		if progress != nil {
194			progress(len(pages))
195		}
196	}
197
198	if hitDeadline || ctx.Err() != nil {
199		slog.Info(fmt.Sprintf("hit deadline for %s after %d pages", startURL, len(pages)), slog.String("component", "crawler"))
200	}
201
202	return &CrawlResult{
203		StartURL:           startURL,
204		Host:               host,
205		Pages:              pages,
206		ExternalLinkStatus: probeExternalLinks(ctx, client, pages, host, startURL),
207		SitemapURLs:        sitemapURLs,
208		Robots:             robotsCtx,
209		Compression:        compression,
210	}, nil
211}
212
213// probeExternalLinks HEADs every distinct off-site link. Sorting before the
214// cap keeps the probed subset the same from one run to the next.
215func probeExternalLinks(ctx context.Context, client *http.Client, pages []*Page, host, startURL string) map[string]int {
216	external := map[string]bool{}
217	for _, p := range pages {
218		if !p.IsHTML || p.HTML == nil {
219			continue
220		}
221		for _, link := range p.HTML.Links {
222			if sameSite(link.URL, host) || isCrawlerHostile(link.URL) {
223				continue
224			}
225			external[link.URL] = true
226		}
227	}
228
229	status := map[string]int{}
230	if len(external) == 0 || ctx.Err() != nil {
231		return status
232	}
233
234	urls := make([]string, 0, len(external))
235	for u := range external {
236		urls = append(urls, u)
237	}
238	sort.Strings(urls)
239	if len(urls) > ExternalLinkCap {
240		slog.Info(fmt.Sprintf("capping external link probes at %d of %d for %s",
241			ExternalLinkCap, len(urls), startURL), slog.String("component", "crawler"))
242		urls = urls[:ExternalLinkCap]
243	}
244
245	var mu sync.Mutex
246	for i := 0; i < len(urls); i += Concurrency {
247		// Without this a slow tail of probes runs past the crawl budget and the
248		// watchdog records a finished crawl as an interruption.
249		if ctx.Err() != nil {
250			slog.Info(fmt.Sprintf("hit deadline during external link probes for %s", startURL), slog.String("component", "crawler"))
251			break
252		}
253
254		end := min(i+Concurrency, len(urls))
255		var wg sync.WaitGroup
256		for _, u := range urls[i:end] {
257			wg.Add(1)
258			go func(u string) {
259				defer wg.Done()
260				code := headStatus(ctx, client, u)
261				mu.Lock()
262				status[u] = code
263				mu.Unlock()
264			}(u)
265		}
266		wg.Wait()
267	}
268	return status
269}