orchard

   1package main
   2
   3import (
   4	"bytes"
   5	"context"
   6	"database/sql"
   7	"encoding/json"
   8	"io/fs"
   9	"net/http"
  10	"net/http/httptest"
  11	"os"
  12	"path/filepath"
  13	"strings"
  14	"testing"
  15	"time"
  16
  17	"github.com/google/uuid"
  18	"status.bythewood.me/web"
  19)
  20
  21// fullPageData populates every field, so a render hits every branch that reads
  22// a value and not only the nil paths.
  23func fullPageData(t *testing.T) PageData {
  24	t.Helper()
  25
  26	pct := 99.4
  27	avg := int64(87)
  28	dur := int64(1234)
  29	pages := int64(42)
  30	now := time.Now()
  31	score := 0.42
  32
  33	return PageData{
  34		Title:         "Example",
  35		Description:   "A description.",
  36		Path:          "/",
  37		Canonical:     baseURL + "/",
  38		Staging:       true,
  39		Authenticated: true,
  40		Year:          now.Year(),
  41		BaseURL:       baseURL,
  42		SourceURL:     sourceURL,
  43		SiteName:      siteName,
  44		AuthorName:    authorName,
  45		Script:        "/static/base.js",
  46		Styles:        []string{"/static/base.css"},
  47		PageScript:    "/static/pages.js",
  48		PageStyles:    []string{"/static/pages.css"},
  49
  50		Next:  "/properties",
  51		Error: "Invalid password.",
  52
  53		TotalChecks:     130042,
  54		TotalProperties: 3,
  55		FirstCheckAt:    "Jun 5, 2022",
  56		Query:           "example",
  57		GeneratedAt:     "2026-08-26 21:00 UTC",
  58
  59		Property: &PropertyView{
  60			ID:               uuid.New().String(),
  61			URL:              "https://example.com",
  62			Name:             "example.com",
  63			IsPublic:         true,
  64			IsProtected:      false,
  65			CurrentStatus:    200,
  66			AvgResponseTime:  142,
  67			RecentUptimePct:  &pct,
  68			RecentTickStream: []string{"up", "up", "down", "up"},
  69			TotalChecks:      1440,
  70
  71			CrawlState:          "idle",
  72			LastCrawlSuccessAt:  &now,
  73			LastCrawlDurationMS: &dur,
  74			LastCrawlPagesCount: &pages,
  75			NextRunAtCrawler:    &now,
  76
  77			LighthouseState: "idle",
  78			LighthouseScores: &Scores{
  79				Performance: 98, Accessibility: 100, BestPractices: 96, SEO: 91,
  80			},
  81			LighthouseDetails: &Details{
  82				Metrics: []Metric{{
  83					ID: "largest-contentful-paint", Acronym: "LCP",
  84					Title: "Largest Contentful Paint", DisplayValue: "1.2 s",
  85					Score: &score, Weight: 25,
  86				}},
  87				Opportunities: []Opportunity{{
  88					ID: "unused-css-rules", Title: "Reduce unused CSS",
  89					DisplayValue: "Potential savings of 12 KiB", SavingsMS: 1350,
  90				}},
  91			},
  92			LastLighthouseSuccessAt:  &now,
  93			LastLighthouseDurationMS: &dur,
  94			NextLighthouseRunAt:      &now,
  95			AvgLighthouseScore:       &avg,
  96
  97			AlertState: "up",
  98			CreatedAt:  now,
  99			UpdatedAt:  now,
 100
 101			IsHTTPS:                   true,
 102			HasMIMEType:               true,
 103			HasContentSniffProtection: true,
 104			HasClickjackProtection:    false,
 105			HidesServerVersion:        true,
 106			HasHSTS:                   true,
 107			HasHSTSPreload:            false,
 108			HasSecurityIssue:          true,
 109
 110			CrawlerInsights: []Insight{
 111				{URL: "https://example.com/a", Issue: "Page has no title", Type: typeSEO, Severity: sevError},
 112			},
 113		},
 114
 115		InsightGroups: []InsightGroup{{
 116			Type: typeSEO,
 117			Items: []Insight{
 118				{URL: "https://example.com/a", Issue: "Page has no title", Type: typeSEO, Severity: sevError},
 119				{URL: "https://example.com/b", Issue: "Thin content (12 words)", Type: typeSEO, Severity: sevWarn, Item: "x"},
 120			},
 121		}},
 122		ResponseTimes: []ResponseTimePoint{
 123			{Label: now.Format(time.RFC3339), Total: 142, DNS: &avg, TCP: &avg, TLS: &avg, TTFB: &avg},
 124			{Label: now.Format(time.RFC3339), Total: 138},
 125		},
 126		StatusCodes:  []LabelCount{{Label: 200, Count: 1400}, {Label: 526, Count: 40}},
 127		UptimeSlices: []LabelPercent{{Label: "Uptime", Count: 97.2}, {Label: "Downtime", Count: 2.8}},
 128	}
 129}
 130
 131// emptyPageData is the nil-everywhere state a template is most likely to break
 132// on.
 133func emptyPageData() PageData {
 134	return PageData{
 135		Title:    "Example",
 136		SiteName: siteName,
 137		BaseURL:  baseURL,
 138		Year:     2026,
 139		Property: &PropertyView{
 140			ID:            uuid.New().String(),
 141			URL:           "https://example.com",
 142			Name:          "example.com",
 143			CurrentStatus: 200,
 144			CrawlState:    "idle",
 145		},
 146	}
 147}
 148
 149func TestTemplatesExecute(t *testing.T) {
 150	templates, err := fs.Sub(templateFS, "templates")
 151	if err != nil {
 152		t.Fatal(err)
 153	}
 154
 155	pages := []string{
 156		"home.html",
 157		"properties.html", "property.html", "notfound.html",
 158	}
 159
 160	renderer, err := web.NewRenderer(templates, templateFuncs,
 161		[]string{"base.html", "partials.html"}, pages)
 162	if err != nil {
 163		t.Fatalf("parsing templates: %v", err)
 164	}
 165
 166	full := fullPageData(t)
 167	full.Properties = []*PropertyView{full.Property}
 168
 169	empty := emptyPageData()
 170	empty.Properties = []*PropertyView{empty.Property}
 171
 172	for _, page := range pages {
 173		for name, data := range map[string]PageData{"full": full, "empty": empty} {
 174			t.Run(page+"/"+name, func(t *testing.T) {
 175				// Render through a recorder, since Renderer.Render panics on a
 176				// template error and that panic is the failure being tested for.
 177				rec := httptest.NewRecorder()
 178				defer func() {
 179					if r := recover(); r != nil {
 180						t.Fatalf("rendering %s: %v", page, r)
 181					}
 182				}()
 183				renderer.Render(rec, http.StatusOK, page, data)
 184
 185				if rec.Code != http.StatusOK {
 186					t.Fatalf("rendering %s: status %d", page, rec.Code)
 187				}
 188				if rec.Body.Len() == 0 {
 189					t.Fatalf("rendering %s produced no output", page)
 190				}
 191			})
 192		}
 193	}
 194}
 195
 196// Covers the {{else}} arm of the range, which neither fixture above reaches.
 197func TestPropertiesListWithNoProperties(t *testing.T) {
 198	templates, _ := fs.Sub(templateFS, "templates")
 199	renderer, err := web.NewRenderer(templates, templateFuncs,
 200		[]string{"base.html", "partials.html"}, []string{"properties.html"})
 201	if err != nil {
 202		t.Fatal(err)
 203	}
 204
 205	rec := httptest.NewRecorder()
 206	renderer.Render(rec, http.StatusOK, "properties.html", PageData{
 207		Title: "Properties", SiteName: siteName, Year: 2026,
 208	})
 209	if !strings.Contains(rec.Body.String(), "no properties yet") {
 210		t.Error("empty properties list did not render its empty state")
 211	}
 212}
 213
 214func TestReportTemplatesExecute(t *testing.T) {
 215	full := fullPageData(t)
 216	empty := emptyPageData()
 217	empty.GeneratedAt = "2026-08-26 21:00 UTC"
 218	empty.BaseURL = baseURL
 219
 220	for _, name := range []string{"report.typ", "report.md"} {
 221		for label, data := range map[string]PageData{"full": full, "empty": empty} {
 222			t.Run(name+"/"+label, func(t *testing.T) {
 223				var buf bytes.Buffer
 224				if err := reportTemplates.ExecuteTemplate(&buf, name, data); err != nil {
 225					t.Fatalf("executing %s: %v", name, err)
 226				}
 227				if buf.Len() == 0 {
 228					t.Fatalf("%s produced no output", name)
 229				}
 230			})
 231		}
 232	}
 233}
 234
 235// "//" starts a Typst line comment and a URL is full of them, so an unescaped
 236// property URL swallows the rest of its line in the PDF.
 237func TestReportEscapesTypstComment(t *testing.T) {
 238	data := fullPageData(t)
 239	data.Property.URL = "https://example.com/a//b"
 240
 241	var buf bytes.Buffer
 242	if err := reportTemplates.ExecuteTemplate(&buf, "report.typ", data); err != nil {
 243		t.Fatal(err)
 244	}
 245	if strings.Contains(buf.String(), "https://example.com") {
 246		t.Error("property URL reached the Typst source with its slashes unescaped")
 247	}
 248	if !strings.Contains(buf.String(), `https:\/\/example.com`) {
 249		t.Error("property URL was not escaped the way typstMD escapes it")
 250	}
 251}
 252
 253// TestSchemaAcceptsLegacyDatabase writes through every column of an inherited
 254// database, so schema drift fails here and not on deploy day.
 255func TestSchemaAcceptsLegacyDatabase(t *testing.T) {
 256	path := filepath.Join(t.TempDir(), "db.sqlite3")
 257
 258	legacySchema, err := os.ReadFile("testdata/legacy_schema.sql")
 259	if err != nil {
 260		t.Fatal(err)
 261	}
 262
 263	seed, err := sql.Open("sqlite", path)
 264	if err != nil {
 265		t.Fatal(err)
 266	}
 267	if _, err := seed.Exec(string(legacySchema)); err != nil {
 268		t.Fatalf("applying the legacy schema: %v", err)
 269	}
 270	// An older migration tool's bookkeeping table, which this app ignores and
 271	// must not trip over. Dropping it for real would break a rollback.
 272	if _, err := seed.Exec(`CREATE TABLE _sqlx_migrations (version BIGINT PRIMARY KEY);
 273		INSERT INTO _sqlx_migrations VALUES (1), (2);`); err != nil {
 274		t.Fatal(err)
 275	}
 276
 277	id := uuid.New()
 278	now := time.Now().UnixMilli()
 279	if _, err := seed.Exec(
 280		`INSERT INTO properties (id, url, is_public, is_protected, alert_state, created_at, updated_at)
 281		 VALUES (?, 'https://example.com', 1, 0, 'up', ?, ?)`, id[:], now, now); err != nil {
 282		t.Fatal(err)
 283	}
 284	if _, err := seed.Exec(
 285		`INSERT INTO checks (property_id, status_code, response_ms, headers, created_at)
 286		 VALUES (?, 200, 142, '{}', ?)`, id[:], now); err != nil {
 287		t.Fatal(err)
 288	}
 289	seed.Close()
 290
 291	db, err := openDB(path)
 292	if err != nil {
 293		t.Fatalf("opening a legacy database: %v", err)
 294	}
 295	defer db.Close()
 296
 297	ctx := context.Background()
 298
 299	p, err := getProperty(ctx, db, id)
 300	if err != nil {
 301		t.Fatalf("reading the existing property: %v", err)
 302	}
 303	if p == nil {
 304		t.Fatal("the existing property was not found; the cutover would lose it")
 305	}
 306	if p.URL != "https://example.com" || p.Name() != "example.com" {
 307		t.Errorf("property read back wrong: %+v", p)
 308	}
 309
 310	dns, tcp, tls, ttfb := int64(3), int64(11), int64(29), int64(64)
 311	if _, err := db.ExecContext(ctx,
 312		`INSERT INTO checks (property_id, status_code, response_ms, headers,
 313		   dns_ms, tcp_ms, tls_ms, ttfb_ms, created_at)
 314		 VALUES (?, 200, 107, '{}', ?, ?, ?, ?, ?)`,
 315		id[:], dns, tcp, tls, ttfb, now); err != nil {
 316		t.Fatalf("writing a check with phase timings: %v", err)
 317	}
 318
 319	checks, err := recentChecks(ctx, db, id, 10)
 320	if err != nil {
 321		t.Fatal(err)
 322	}
 323	if len(checks) != 2 {
 324		t.Fatalf("expected 2 checks, got %d", len(checks))
 325	}
 326	// The older row must read back with nil phases and not zeros, since the
 327	// chart draws a gap for nil and a floor for 0.
 328	var legacy *Check
 329	for i := range checks {
 330		if checks[i].ResponseMS == 142 {
 331			legacy = &checks[i]
 332		}
 333	}
 334	if legacy == nil {
 335		t.Fatal("the pre-migration check row did not read back")
 336	}
 337	if legacy.DNSMS != nil || legacy.TCPMS != nil || legacy.TLSMS != nil || legacy.TTFBMS != nil {
 338		t.Error("a pre-migration check row came back with non-nil phase timings")
 339	}
 340}
 341
 342// TestSchemaIsIdempotent covers the ordinary restart, where openDB has to be a
 343// no-op against a database it already created.
 344func TestSchemaIsIdempotent(t *testing.T) {
 345	path := filepath.Join(t.TempDir(), "db.sqlite3")
 346	for i := range 3 {
 347		db, err := openDB(path)
 348		if err != nil {
 349			t.Fatalf("open %d: %v", i, err)
 350		}
 351		db.Close()
 352	}
 353}
 354
 355func TestSecurityPosture(t *testing.T) {
 356	headers := func(m map[string]string) []Check {
 357		encoded, _ := json.Marshal(m)
 358		return []Check{{StatusCode: 200, Headers: string(encoded)}}
 359	}
 360
 361	t.Run("a fully hardened response passes everything", func(t *testing.T) {
 362		v := &PropertyView{URL: "https://example.com", CurrentStatus: 200}
 363		v.applySecurityPosture(headers(map[string]string{
 364			"content-type":              "text/html; charset=utf-8",
 365			"x-content-type-options":    "nosniff",
 366			"x-frame-options":           "SAMEORIGIN",
 367			"strict-transport-security": "max-age=31536000; includeSubDomains; preload",
 368		}))
 369		if v.HasSecurityIssue {
 370			t.Errorf("a hardened response was reported as having an issue: %+v", v)
 371		}
 372	})
 373
 374	t.Run("an hsts max-age under a year does not count", func(t *testing.T) {
 375		v := &PropertyView{URL: "https://example.com"}
 376		v.applySecurityPosture(headers(map[string]string{
 377			"strict-transport-security": "max-age=86400",
 378		}))
 379		if v.HasHSTS {
 380			t.Error("a one-day max-age was accepted as HSTS")
 381		}
 382	})
 383
 384	t.Run("an absurd max-age does not overflow into a failure", func(t *testing.T) {
 385		v := &PropertyView{URL: "https://example.com"}
 386		v.applySecurityPosture(headers(map[string]string{
 387			"strict-transport-security": "max-age=99999999999999999999999",
 388		}))
 389		// It does not parse as an int64, so it reports absent rather than wrapping
 390		// negative. A panic here is the failure.
 391		_ = v.HasHSTS
 392	})
 393
 394	t.Run("any server header spelling defeats the version check", func(t *testing.T) {
 395		for _, header := range []string{"server", "x-server", "powered-by", "x-powered-by"} {
 396			v := &PropertyView{URL: "https://example.com"}
 397			v.applySecurityPosture(headers(map[string]string{header: "nginx/1.2.3"}))
 398			if v.HidesServerVersion {
 399				t.Errorf("%s leaked the server version but the check passed", header)
 400			}
 401		}
 402	})
 403
 404	t.Run("a property with no checks yet is not reported as insecure HTTPS", func(t *testing.T) {
 405		v := &PropertyView{URL: "https://example.com"}
 406		v.applySecurityPosture(nil)
 407		if !v.IsHTTPS {
 408			t.Error("an https:// URL with no checks was reported as not HTTPS")
 409		}
 410	})
 411
 412	t.Run("526 is what marks a certificate invalid", func(t *testing.T) {
 413		v := &PropertyView{URL: "https://example.com", CurrentStatus: 526}
 414		v.applySecurityPosture(nil)
 415		if !v.InvalidCert {
 416			t.Error("a 526 did not mark the certificate invalid")
 417		}
 418	})
 419}
 420
 421// TestAlertStateMachine pins the asymmetry in the alerting, two strikes to go
 422// down and one success to come back.
 423func TestAlertStateMachine(t *testing.T) {
 424	db, id := freshDB(t)
 425	ctx := context.Background()
 426	notifier := &Notifier{client: http.DefaultClient, base: "http://127.0.0.1:1", topic: "test"}
 427
 428	p, err := getProperty(ctx, db, id)
 429	if err != nil {
 430		t.Fatal(err)
 431	}
 432
 433	state := func() string {
 434		var s string
 435		if err := db.QueryRow("SELECT alert_state FROM properties WHERE id = ?", id[:]).Scan(&s); err != nil {
 436			t.Fatal(err)
 437		}
 438		return s
 439	}
 440	record := func(code int64) {
 441		if _, err := db.Exec(
 442			`INSERT INTO checks (property_id, status_code, response_ms, headers, created_at)
 443			 VALUES (?, ?, 100, '{}', ?)`, id[:], code, nowMS()); err != nil {
 444			t.Fatal(err)
 445		}
 446		if err := advanceAlertState(ctx, db, notifier, p, code); err != nil {
 447			t.Fatal(err)
 448		}
 449	}
 450
 451	if state() != "up" {
 452		t.Fatalf("a new property should start up, got %q", state())
 453	}
 454
 455	record(503)
 456	if state() != "up" {
 457		t.Error("one failure took the property down; it should need two")
 458	}
 459
 460	record(503)
 461	if state() != "down" {
 462		t.Error("two consecutive failures did not take the property down")
 463	}
 464
 465	record(200)
 466	if state() != "up" {
 467		t.Error("a single success did not bring the property back up")
 468	}
 469}
 470
 471// TestAlertStateIgnoresNonConsecutiveFailures is the flapping case, where
 472// fail, recover, fail must not fire an outage.
 473func TestAlertStateIgnoresNonConsecutiveFailures(t *testing.T) {
 474	db, id := freshDB(t)
 475	ctx := context.Background()
 476	notifier := &Notifier{client: http.DefaultClient, base: "http://127.0.0.1:1", topic: "test"}
 477	p, _ := getProperty(ctx, db, id)
 478
 479	// created_at is explicit and increasing, since the state machine orders by
 480	// it and same-millisecond rows would order arbitrarily.
 481	base := nowMS()
 482	for i, code := range []int64{503, 200, 503} {
 483		if _, err := db.Exec(
 484			`INSERT INTO checks (property_id, status_code, response_ms, headers, created_at)
 485			 VALUES (?, ?, 100, '{}', ?)`, id[:], code, base+int64(i)); err != nil {
 486			t.Fatal(err)
 487		}
 488		if err := advanceAlertState(ctx, db, notifier, p, code); err != nil {
 489			t.Fatal(err)
 490		}
 491	}
 492
 493	var s string
 494	if err := db.QueryRow("SELECT alert_state FROM properties WHERE id = ?", id[:]).Scan(&s); err != nil {
 495		t.Fatal(err)
 496	}
 497	if s != "up" {
 498		t.Errorf("a flapping site was marked down; state machine says %q", s)
 499	}
 500}
 501
 502func freshDB(t *testing.T) (*sql.DB, uuid.UUID) {
 503	t.Helper()
 504	db, err := openDB(filepath.Join(t.TempDir(), "db.sqlite3"))
 505	if err != nil {
 506		t.Fatal(err)
 507	}
 508	t.Cleanup(func() { db.Close() })
 509
 510	id, err := createProperty(context.Background(), db, "https://example.com")
 511	if err != nil {
 512		t.Fatal(err)
 513	}
 514	return db, id
 515}
 516
 517func TestRenderAlert(t *testing.T) {
 518	ctx := AlertContext{
 519		ID: "abc", Name: "example.com", URL: "https://example.com",
 520		CurrentStatus: 503, AvgResponseMS: 184,
 521	}
 522
 523	down, ok := renderAlert("down", ctx)
 524	if !ok {
 525		t.Fatal("the down alert did not render")
 526	}
 527	if !strings.Contains(down.Title, "example.com") || !strings.Contains(down.Message, "503") {
 528		t.Errorf("the down alert lost its detail: %+v", down)
 529	}
 530	if down.Click != baseURL+"/abc" {
 531		t.Errorf("the down alert links to %q, which is not an absolute dashboard URL", down.Click)
 532	}
 533	// Read on a phone, where an outage that looks like a recovery gets missed.
 534	if down.Priority == "default" {
 535		t.Error("the outage alert has the same priority as a recovery")
 536	}
 537
 538	if _, ok := renderAlert("sideways", ctx); ok {
 539		t.Error("an unknown alert kind rendered instead of being refused")
 540	}
 541}
 542
 543func page(url string, html *ParsedHTML) *Page {
 544	return &Page{URL: url, Status: 200, IsHTML: true, HTML: html, ContentType: "text/html"}
 545}
 546
 547func parsed(t *testing.T, body, url string) *ParsedHTML {
 548	t.Helper()
 549	p, err := parseHTML([]byte(body), url)
 550	if err != nil {
 551		t.Fatal(err)
 552	}
 553	return p
 554}
 555
 556func TestParseHTML(t *testing.T) {
 557	p := parsed(t, `<!doctype html><html lang="en"><head>
 558		<title>  A   title  </title>
 559		<meta name="description" content="A description.">
 560		<meta property="og:title" content="OG">
 561		<meta name="twitter:card" content="summary">
 562		<link rel="canonical" href="/canonical">
 563		<link rel="shortcut icon" href="/favicon.ico">
 564		<script type="application/ld+json">{"@type":"Thing"}</script>
 565		<script type="application/ld+json">{not json}</script>
 566	</head><body>
 567		<h1>Heading one</h1>
 568		<h3>Skipped a level</h3>
 569		<a href="/a">Link A</a>
 570		<a href="#frag">Fragment</a>
 571		<a href="mailto:[email protected]">Mail</a>
 572		<a href="https://other.example/b" rel="nofollow noopener">Off site</a>
 573		<img src="/img.png" alt="described">
 574		<img src="/bare.png">
 575		<img src="/decorative.png" alt="">
 576		<form action="/submit">
 577			<label for="name">Name</label>
 578			<input type="text" id="name">
 579			<input type="text" id="unlabeled">
 580			<input type="hidden" name="csrf">
 581		</form>
 582		<script>var ignored = "script text";</script>
 583		<style>.ignored { color: red }</style>
 584	</body></html>`, "https://example.com/page")
 585
 586	if p.Title != "A title" {
 587		t.Errorf("title whitespace was not collapsed: %q", p.Title)
 588	}
 589	if p.Lang != "en" {
 590		t.Errorf("lang = %q", p.Lang)
 591	}
 592	if p.Canonical != "https://example.com/canonical" {
 593		t.Errorf("canonical was not resolved against the page URL: %q", p.Canonical)
 594	}
 595	if p.Favicon != "https://example.com/favicon.ico" {
 596		t.Errorf("favicon = %q", p.Favicon)
 597	}
 598	if len(p.JSONLD) != 1 || p.JSONLDBad != 1 {
 599		t.Errorf("JSON-LD: %d valid, %d bad; want 1 and 1", len(p.JSONLD), p.JSONLDBad)
 600	}
 601
 602	// Fragments, mailto and tel are not pages.
 603	if len(p.Links) != 2 {
 604		t.Errorf("expected 2 crawlable links, got %d: %+v", len(p.Links), p.Links)
 605	}
 606
 607	var missing, decorative int
 608	for _, img := range p.Images {
 609		switch {
 610		case img.Alt == nil:
 611			missing++
 612		case *img.Alt == "":
 613			decorative++
 614		}
 615	}
 616	if missing != 1 || decorative != 1 {
 617		t.Errorf("alt handling: %d missing, %d decorative; want 1 and 1", missing, decorative)
 618	}
 619
 620	if len(p.Forms) != 1 {
 621		t.Fatalf("expected one form, got %d", len(p.Forms))
 622	}
 623	if len(p.Forms[0].Inputs) != 3 || len(p.Forms[0].LabelFors) != 1 {
 624		t.Errorf("form parsed as %+v", p.Forms[0])
 625	}
 626
 627	// Script text must not reach the word count, or a page with a big inline
 628	// bundle passes the thin-content check on its JavaScript.
 629	scripty := parsed(t,
 630		`<html><body><p>one two</p><script>var lorem = "ipsum dolor sit amet";</script></body></html>`,
 631		"https://example.com/")
 632	if scripty.WordCount != 2 {
 633		t.Errorf("word count = %d, want 2; script contents reached the visible text", scripty.WordCount)
 634	}
 635}
 636
 637// Entities must be decoded, or "&nbsp;" counts as a word.
 638func TestParseHTMLDecodesEntities(t *testing.T) {
 639	p := parsed(t, `<html><body><p>one&nbsp;two&amp;three</p></body></html>`, "https://example.com/")
 640	if p.WordCount != 2 {
 641		t.Errorf("word count = %d, want 2; entities were not decoded", p.WordCount)
 642	}
 643}
 644
 645func TestSameSite(t *testing.T) {
 646	cases := []struct {
 647		url, host string
 648		want      bool
 649	}{
 650		{"https://example.com/a", "example.com", true},
 651		{"https://www.example.com/a", "example.com", true},
 652		{"https://example.com/a", "www.example.com", true},
 653		{"https://other.example/a", "example.com", false},
 654		// A subdomain is a different site with its own robots.txt and sitemap.
 655		{"https://blog.example.com/a", "example.com", false},
 656		{"not a url", "example.com", false},
 657	}
 658	for _, c := range cases {
 659		if got := sameSite(c.url, c.host); got != c.want {
 660			t.Errorf("sameSite(%q, %q) = %v, want %v", c.url, c.host, got, c.want)
 661		}
 662	}
 663}
 664
 665func TestChecksFindTheObviousFailures(t *testing.T) {
 666	good := parsed(t, `<html lang="en"><head>
 667		<title>A title long enough to sit inside the recommended thirty to sixty</title>
 668		<meta name="description" content="A description written to be comfortably inside the recommended seventy to one hundred and sixty characters.">
 669		<meta name="viewport" content="width=device-width">
 670		<link rel="canonical" href="https://example.com/good">
 671		<link rel="icon" href="/f.ico">
 672		<meta property="og:title" content="t"><meta property="og:description" content="d">
 673		<meta property="og:image" content="i"><meta property="og:url" content="u">
 674		<meta name="twitter:card" content="summary">
 675	</head><body><h1>A heading of a perfectly reasonable length</h1></body></html>`,
 676		"https://example.com/good")
 677
 678	bad := parsed(t, `<html><head><title>Short</title></head><body></body></html>`,
 679		"https://example.com/bad")
 680
 681	result := &CrawlResult{
 682		StartURL: "https://example.com/good",
 683		Host:     "example.com",
 684		Pages: []*Page{
 685			page("https://example.com/good", good),
 686			page("https://example.com/bad", bad),
 687		},
 688		Robots:      RobotsCtx{Exists: true, ReferencesSitemap: true},
 689		SitemapURLs: []string{"https://example.com/good", "https://example.com/bad"},
 690		Compression: "gzip",
 691	}
 692
 693	byURL := map[string][]string{}
 694	for _, i := range runChecks(result) {
 695		byURL[i.URL] = append(byURL[i.URL], i.Issue)
 696	}
 697
 698	for _, want := range []string{
 699		"Page has no meta description",
 700		"Page has no h1",
 701		"Page has no canonical URL",
 702		"HTML lang attribute missing",
 703		"Viewport meta tag missing (mobile)",
 704		"Favicon link missing",
 705		"Twitter card meta tag missing",
 706	} {
 707		if !containsIssue(byURL["https://example.com/bad"], want) {
 708			t.Errorf("the deficient page was not flagged for %q", want)
 709		}
 710	}
 711
 712	// And the good page must not be caught for any of them, since a check that
 713	// fires on everything gets the audit ignored.
 714	for _, unwanted := range []string{
 715		"Page has no title",
 716		"Page has no meta description",
 717		"Page has no h1",
 718		"Page has no canonical URL",
 719		"HTML lang attribute missing",
 720		"Viewport meta tag missing (mobile)",
 721		"Favicon link missing",
 722		"Twitter card meta tag missing",
 723		"Open Graph tags missing",
 724	} {
 725		if containsIssue(byURL["https://example.com/good"], unwanted) {
 726			t.Errorf("a well-formed page was flagged for %q", unwanted)
 727		}
 728	}
 729}
 730
 731func containsIssue(issues []string, want string) bool {
 732	for _, i := range issues {
 733		if strings.HasPrefix(i, want) {
 734			return true
 735		}
 736	}
 737	return false
 738}
 739
 740// TestRedirectChainCheckFires guards a condition that is easy to make
 741// unreachable by recording one hop and testing for more than two.
 742func TestRedirectChainCheckFires(t *testing.T) {
 743	result := &CrawlResult{
 744		StartURL: "https://example.com/",
 745		Host:     "example.com",
 746		Pages: []*Page{
 747			{URL: "https://example.com/final", RequestedURL: "http://example.com", Status: 200, RedirectHops: 2},
 748			{URL: "https://example.com/one-hop", RequestedURL: "http://example.com/one-hop", Status: 200, RedirectHops: 1},
 749			{URL: "https://example.com/direct", RequestedURL: "https://example.com/direct", Status: 200},
 750		},
 751	}
 752
 753	var flagged []string
 754	for _, i := range runChecks(result) {
 755		if strings.HasPrefix(i.Issue, "Redirect chain") {
 756			flagged = append(flagged, i.URL)
 757		}
 758	}
 759
 760	if len(flagged) != 1 || flagged[0] != "https://example.com/final" {
 761		t.Errorf("redirect chain check flagged %v; want only the two-hop URL", flagged)
 762	}
 763}
 764
 765// The same crawl must produce the same findings in the same order every time,
 766// or a weekly report diffs as changed when nothing has.
 767func TestDuplicateChecksAreOrdered(t *testing.T) {
 768	build := func() []*Page {
 769		var pages []*Page
 770		for _, slug := range []string{"a", "b", "c", "d", "e", "f"} {
 771			url := "https://example.com/" + slug
 772			pages = append(pages, page(url, parsed(t,
 773				`<html><head><title>Same title everywhere</title></head><body></body></html>`, url)))
 774		}
 775		return pages
 776	}
 777
 778	var first []string
 779	for run := range 8 {
 780		result := &CrawlResult{StartURL: "https://example.com/", Host: "example.com", Pages: build()}
 781		var order []string
 782		for _, i := range runChecks(result) {
 783			if i.Issue == "Duplicate title" {
 784				order = append(order, i.URL)
 785			}
 786		}
 787		if len(order) != 6 {
 788			t.Fatalf("run %d: expected 6 duplicate-title findings, got %d", run, len(order))
 789		}
 790		if first == nil {
 791			first = order
 792			continue
 793		}
 794		for i := range order {
 795			if order[i] != first[i] {
 796				t.Fatalf("run %d produced a different ordering than run 0:\n %v\n %v", run, first, order)
 797			}
 798		}
 799	}
 800}
 801
 802func TestLighthouseScorePairsAreOrdered(t *testing.T) {
 803	s := &Scores{Performance: 1, Accessibility: 2, BestPractices: 3, SEO: 4}
 804	want := []string{"Performance", "Accessibility", "Best practices", "SEO"}
 805	for run := range 8 {
 806		for i, pair := range s.Pairs() {
 807			if pair.Label != want[i] {
 808				t.Fatalf("run %d: score %d is %q, want %q", run, i, pair.Label, want[i])
 809			}
 810		}
 811	}
 812}
 813
 814func TestParseScoresRejectsNulls(t *testing.T) {
 815	report := map[string]any{"categories": map[string]any{
 816		"performance":    map[string]any{"score": 0.98},
 817		"accessibility":  map[string]any{"score": nil},
 818		"best-practices": map[string]any{"score": 0.96},
 819		"seo":            map[string]any{"score": 0.91},
 820	}}
 821
 822	// A null score means the category could not be evaluated, so storing zero
 823	// would claim the site scored nothing.
 824	if _, err := parseScores(report); err == nil {
 825		t.Fatal("a null score was accepted instead of failing the audit")
 826	}
 827}
 828
 829func TestParseScoresRounds(t *testing.T) {
 830	report := map[string]any{"categories": map[string]any{
 831		"performance":    map[string]any{"score": 0.985},
 832		"accessibility":  map[string]any{"score": 1.0},
 833		"best-practices": map[string]any{"score": 0.0},
 834		"seo":            map[string]any{"score": 0.914},
 835	}}
 836	s, err := parseScores(report)
 837	if err != nil {
 838		t.Fatal(err)
 839	}
 840	if s.Performance != 99 || s.Accessibility != 100 || s.BestPractices != 0 || s.SEO != 91 {
 841		t.Errorf("scores rounded to %+v", s)
 842	}
 843}
 844
 845func TestParseDetailsFiltersNonActionableAudits(t *testing.T) {
 846	report := map[string]any{
 847		"categories": map[string]any{"performance": map[string]any{"auditRefs": []any{
 848			map[string]any{"id": "lcp", "group": "metrics", "weight": 25.0, "acronym": "LCP"},
 849			map[string]any{"id": "passing", "weight": 0.0},
 850			map[string]any{"id": "hidden-audit", "group": "hidden", "weight": 0.0},
 851			map[string]any{"id": "diagnostic", "weight": 0.0},
 852			map[string]any{"id": "real-opportunity", "weight": 0.0},
 853		}}},
 854		"audits": map[string]any{
 855			"lcp":     map[string]any{"score": 0.4, "title": "Largest Contentful Paint", "displayValue": "3.1 s"},
 856			"passing": map[string]any{"score": 0.95, "title": "Already fine"},
 857			// Kept by Lighthouse but no longer scored, so it must not show up
 858			// as a win the site never earned.
 859			"hidden-audit": map[string]any{"score": 0.1, "title": "Time to Interactive"},
 860			// Scores badly but carries no actionable saving.
 861			"diagnostic": map[string]any{"score": 0.1, "title": "Avoid forced reflow"},
 862			"real-opportunity": map[string]any{
 863				"score": 0.2, "title": "Reduce unused CSS",
 864				"details": map[string]any{"overallSavingsMs": 1350.0},
 865			},
 866		},
 867	}
 868
 869	d := parseDetails(report)
 870	if d == nil {
 871		t.Fatal("details did not parse")
 872	}
 873	if len(d.Metrics) != 1 || d.Metrics[0].Acronym != "LCP" {
 874		t.Errorf("metrics = %+v", d.Metrics)
 875	}
 876	if len(d.Opportunities) != 1 || d.Opportunities[0].ID != "real-opportunity" {
 877		t.Errorf("opportunities = %+v; only the one with a real saving should survive", d.Opportunities)
 878	}
 879}
 880
 881func TestAsciiFilename(t *testing.T) {
 882	cases := map[string]string{
 883		"example.com": "example.com",
 884		// A non-ASCII byte cannot go into a header value unencoded.
 885		"Café":     "Caf_",
 886		"...":      "report",
 887		"  spaced": "spaced",
 888		"":         "report",
 889	}
 890	for in, want := range cases {
 891		if got := asciiFilename(in); got != want {
 892			t.Errorf("asciiFilename(%q) = %q, want %q", in, got, want)
 893		}
 894	}
 895}
 896
 897func TestPropertyName(t *testing.T) {
 898	cases := map[string]string{
 899		"https://www.example.com/path": "example.com",
 900		"https://example.com":          "example.com",
 901		"https://example.com:8443/":    "example.com",
 902		// Not a URL at all, so the whole string, rather than a blank row.
 903		"nonsense": "nonsense",
 904	}
 905	for in, want := range cases {
 906		p := &Property{URL: in}
 907		if got := p.Name(); got != want {
 908			t.Errorf("Name() for %q = %q, want %q", in, got, want)
 909		}
 910	}
 911}
 912
 913func TestNext3MinBoundaryIsAligned(t *testing.T) {
 914	next := time.UnixMilli(next3MinBoundary()).UTC()
 915	if next.Second() != 0 || next.Nanosecond() != 0 || next.Minute()%3 != 0 {
 916		t.Errorf("next run at %s is not on a three-minute boundary", next)
 917	}
 918	if !next.After(time.Now().UTC()) {
 919		t.Errorf("next run at %s is not in the future", next)
 920	}
 921}
 922
 923func TestNaturalTime(t *testing.T) {
 924	now := time.Now()
 925	cases := []struct {
 926		in   *time.Time
 927		want string
 928	}{
 929		{nil, "never"},
 930		{ptrTime(now.Add(-30 * time.Second)), "just now"},
 931		{ptrTime(now.Add(-2 * time.Minute)), "2 minutes ago"},
 932		{ptrTime(now.Add(-1 * time.Hour)), "1 hour ago"},
 933		{ptrTime(now.Add(48 * time.Hour)), "2 days from now"},
 934	}
 935	for _, c := range cases {
 936		if got := naturalTime(c.in); got != c.want {
 937			t.Errorf("naturalTime = %q, want %q", got, c.want)
 938		}
 939	}
 940}
 941
 942func ptrTime(t time.Time) *time.Time { return &t }
 943
 944func TestIntcomma(t *testing.T) {
 945	cases := map[int64]string{
 946		0: "0", 999: "999", 1000: "1,000", 130042: "130,042",
 947		1234567: "1,234,567", -4321: "-4,321",
 948	}
 949	for in, want := range cases {
 950		if got := formatNum(in); got != want {
 951			t.Errorf("formatNum(%d) = %q, want %q", in, got, want)
 952		}
 953	}
 954}
 955
 956func TestJSONBlockCannotCloseTheScriptElement(t *testing.T) {
 957	// The values here are what a crawled site can put in an error string, and
 958	// they land in a <script type="application/json"> block. html/template does
 959	// not escape inside that script type and template.JS turns its escaping off
 960	// regardless, so encoding/json is the only thing holding the element shut.
 961	in := map[string]string{"issue": `a <b> & "c" </script><img src=x onerror=alert(1)>`}
 962	out, err := jsonBlock(in)
 963	if err != nil {
 964		t.Fatal(err)
 965	}
 966
 967	for _, bad := range []string{"<", ">", "&"} {
 968		if strings.Contains(string(out), bad) {
 969			t.Errorf("%q reached the block unescaped, which can close it: %s", bad, out)
 970		}
 971	}
 972
 973	// Escaping is transparent to the consumer: JSON.parse gives the browser the
 974	// original string back, so nothing downstream has to know this happened.
 975	var round map[string]string
 976	if err := json.Unmarshal([]byte(out), &round); err != nil {
 977		t.Fatalf("output is not valid JSON: %v", err)
 978	}
 979	if round["issue"] != in["issue"] {
 980		t.Errorf("value did not round-trip: got %q, want %q", round["issue"], in["issue"])
 981	}
 982}
 983
 984// TestStatusPayloadShape pins the field names the dashboard JavaScript reads,
 985// where a rename breaks the live panel with no server error.
 986func TestStatusPayloadShape(t *testing.T) {
 987	pages := int64(12)
 988	p := &Property{
 989		CrawlState: "running", LighthouseState: "idle",
 990		LastCrawlPagesCount: &pages,
 991	}
 992	encoded, err := json.Marshal(buildStatusPayload(p))
 993	if err != nil {
 994		t.Fatal(err)
 995	}
 996
 997	var payload map[string]any
 998	if err := json.Unmarshal(encoded, &payload); err != nil {
 999		t.Fatal(err)
1000	}
1001	for _, key := range []string{"crawler", "lighthouse", "server_time"} {
1002		if _, ok := payload[key]; !ok {
1003			t.Errorf("status payload is missing %q", key)
1004		}
1005	}
1006	crawler := payload["crawler"].(map[string]any)
1007	for _, key := range []string{
1008		"state", "started_at", "last_attempt_at", "last_success_at", "last_error",
1009		"last_duration_ms", "pages_count", "next_run_at", "is_overdue",
1010		"insights_total", "insights_by_severity", "progress",
1011	} {
1012		if _, ok := crawler[key]; !ok {
1013			t.Errorf("crawler status is missing %q", key)
1014		}
1015	}
1016	// A running crawl must report progress, or the bar never appears.
1017	if crawler["progress"] == nil {
1018		t.Error("a running crawl reported no progress")
1019	}
1020}
1021
1022func TestCrawlProgressStaysBelowComplete(t *testing.T) {
1023	huge := int64(PageCap * 10)
1024	p := &Property{CrawlState: "running", LastCrawlPagesCount: &huge}
1025	progress := crawlProgress(p)
1026	if progress == nil || *progress > 0.9 {
1027		t.Errorf("progress = %v; a running crawl must never claim to be finished", progress)
1028	}
1029
1030	idle := &Property{CrawlState: "idle"}
1031	if crawlProgress(idle) != nil {
1032		t.Error("an idle crawl reported progress")
1033	}
1034}
1035
1036// reportTemplates is a package level Must(), so a broken report template takes
1037// the whole binary down at init rather than one route.
1038func TestReportsParse(t *testing.T) {
1039	var names []string
1040	for _, tmpl := range reportTemplates.Templates() {
1041		names = append(names, tmpl.Name())
1042	}
1043	for _, want := range []string{"report.typ", "report.md"} {
1044		found := false
1045		for _, name := range names {
1046			if name == want {
1047				found = true
1048			}
1049		}
1050		if !found {
1051			t.Errorf("%s was not parsed; got %v", want, names)
1052		}
1053	}
1054}
1055
1056// classifyCache answers one question, whether the edge served this itself. It
1057// used to infer origin health from Age, which called a healthy origin dead six
1058// times in an afternoon because Cloudflare's Edge TTL is set by a Cache Rule the
1059// origin never sees.
1060func TestClassifyCache(t *testing.T) {
1061	const cc = "public, max-age=300, stale-while-revalidate=86400, stale-if-error=604800"
1062
1063	cases := []struct {
1064		name    string
1065		headers map[string]string
1066		cached  bool
1067		age     int64
1068	}{
1069		{
1070			name:    "hit is the edge answering by itself",
1071			headers: map[string]string{"cf-cache-status": "HIT", "age": "419", "cache-control": cc},
1072			cached:  true, age: 419,
1073		},
1074		{
1075			name:    "served stale while revalidating is still the edge answering",
1076			headers: map[string]string{"cf-cache-status": "UPDATING", "age": "464", "cache-control": cc},
1077			cached:  true, age: 464,
1078		},
1079		{
1080			// The case that used to fire. A Cache Rule holding a copy for longer
1081			// than the origin asked for is not an outage.
1082			name:    "very old copy is still only a cache hit",
1083			headers: map[string]string{"cf-cache-status": "UPDATING", "age": "61854", "cache-control": cc},
1084			cached:  true, age: 61854,
1085		},
1086		{
1087			name:    "dynamic means the origin was reached",
1088			headers: map[string]string{"cf-cache-status": "DYNAMIC", "age": "99999", "cache-control": cc},
1089			cached:  false, age: 99999,
1090		},
1091		{
1092			name:    "miss means the origin was reached",
1093			headers: map[string]string{"cf-cache-status": "MISS", "cache-control": cc},
1094			cached:  false, age: -1,
1095		},
1096		{
1097			name:    "no cache headers at all",
1098			headers: map[string]string{},
1099			cached:  false, age: -1,
1100		},
1101	}
1102
1103	for _, tc := range cases {
1104		t.Run(tc.name, func(t *testing.T) {
1105			_, age, cached := classifyCache(tc.headers)
1106			if cached != tc.cached {
1107				t.Fatalf("cached = %v, want %v", cached, tc.cached)
1108			}
1109			if got := derefAge(age); got != tc.age {
1110				t.Fatalf("age = %d, want %d", got, tc.age)
1111			}
1112		})
1113	}
1114}
1115
1116// The conclusion has to reach the database and not just the caller, since
1117// advanceAlertState re-reads status_code out of checks and would otherwise see
1118// the edge's 200 in the row that was just written.
1119func TestOriginStaleIsPersistedNotJustReturned(t *testing.T) {
1120	const cc = "public, max-age=300, stale-while-revalidate=86400, stale-if-error=604800"
1121
1122	_, _, cached := classifyCache(map[string]string{
1123		"cf-cache-status": "UPDATING",
1124		"age":             "9000",
1125		"cache-control":   cc,
1126	})
1127	if !cached {
1128		t.Fatal("precondition: this response should read as served from cache")
1129	}
1130
1131	// runCheck writes `effective`, so it has to write something the alert
1132	// machine treats as not-up. The number itself is arbitrary.
1133	if statusOriginStale == 200 {
1134		t.Fatal("statusOriginStale must not be 200, or every stale check reads as up")
1135	}
1136}
1137
1138// Every template the server asks for has to exist. NewRenderer resolves the
1139// list at boot rather than at build, so a page left listed after its file was
1140// deleted compiles, ships, and then crash-loops the container on startup, which
1141// is how it was found on repos.
1142func TestEveryListedTemplateParses(t *testing.T) {
1143	templates, err := fs.Sub(templateFS, "templates")
1144	if err != nil {
1145		t.Fatal(err)
1146	}
1147	if _, err := web.NewRenderer(templates, templateFuncs, layoutTemplates, pageTemplates); err != nil {
1148		t.Fatalf("the template set does not parse: %v", err)
1149	}
1150}