orchard

  1package main
  2
  3import (
  4	"context"
  5	"encoding/json"
  6	"fmt"
  7	"log/slog"
  8	"math"
  9	"net/http"
 10	"regexp"
 11	"sort"
 12	"strconv"
 13	"strings"
 14	"time"
 15
 16	"github.com/google/uuid"
 17)
 18
 19// PropertyView is one property, with everything the templates need already
 20// computed.
 21type PropertyView struct {
 22	ID          string
 23	URL         string
 24	Name        string
 25	IsPublic    bool
 26	IsProtected bool
 27
 28	CurrentStatus    int64
 29	AvgResponseTime  int64
 30	RecentUptimePct  *float64
 31	RecentTickStream []string
 32	TotalChecks      int64
 33
 34	CrawlState          string
 35	CrawlerInsights     []Insight
 36	LastCrawlSuccessAt  *time.Time
 37	LastCrawlError      string
 38	LastCrawlDurationMS *int64
 39	LastCrawlPagesCount *int64
 40	NextRunAtCrawler    *time.Time
 41	CrawlStartedAt      *time.Time
 42
 43	LighthouseState          string
 44	LighthouseScores         *Scores
 45	LighthouseDetails        *Details
 46	LastLighthouseSuccessAt  *time.Time
 47	LastLighthouseError      string
 48	LastLighthouseDurationMS *int64
 49	NextLighthouseRunAt      *time.Time
 50	LighthouseStartedAt      *time.Time
 51	AvgLighthouseScore       *int64
 52
 53	AlertState string
 54	CreatedAt  time.Time
 55	UpdatedAt  time.Time
 56
 57	// Security posture, read off the most recent response's headers.
 58	IsHTTPS                   bool
 59	InvalidCert               bool
 60	HasMIMEType               bool
 61	HasContentSniffProtection bool
 62	HasClickjackProtection    bool
 63	HidesServerVersion        bool
 64	HasHSTS                   bool
 65	HasHSTSPreload            bool
 66	HasSecurityIssue          bool
 67}
 68
 69type InsightGroup struct {
 70	Type  string
 71	Items []Insight
 72}
 73
 74// ResponseTimePoint is one bar of the phase-breakdown chart. The four phases
 75// are pointers because rows written before those columns existed have none.
 76type ResponseTimePoint struct {
 77	Label string `json:"label"`
 78	Total int64  `json:"total"`
 79	DNS   *int64 `json:"dns"`
 80	TCP   *int64 `json:"tcp"`
 81	TLS   *int64 `json:"tls"`
 82	TTFB  *int64 `json:"ttfb"`
 83}
 84
 85type LabelCount struct {
 86	Label int64 `json:"label"`
 87	Count int64 `json:"count"`
 88}
 89
 90type LabelPercent struct {
 91	Label string  `json:"label"`
 92	Count float64 `json:"count"`
 93}
 94
 95func msToTime(ms *int64) *time.Time {
 96	if ms == nil {
 97		return nil
 98	}
 99	t := time.UnixMilli(*ms).UTC()
100	return &t
101}
102
103func deref(s *string) string {
104	if s == nil {
105		return ""
106	}
107	return *s
108}
109
110// buildPropertyView computes everything the dashboard shows for one property,
111// over one 100-check window so every figure on it covers the same span.
112func (s *site) buildPropertyView(ctx context.Context, p *Property) (*PropertyView, error) {
113	recent, err := recentChecks(ctx, s.db, p.ID, 100)
114	if err != nil {
115		return nil, err
116	}
117	total, err := countChecks(ctx, s.db, p.ID)
118	if err != nil {
119		return nil, err
120	}
121
122	v := &PropertyView{
123		ID:          p.ID.String(),
124		URL:         p.URL,
125		Name:        p.Name(),
126		IsPublic:    p.IsPublic,
127		IsProtected: p.IsProtected,
128		TotalChecks: total,
129
130		CrawlState:          p.CrawlState,
131		LastCrawlSuccessAt:  msToTime(p.LastCrawlSuccessAt),
132		LastCrawlError:      deref(p.LastCrawlError),
133		LastCrawlDurationMS: p.LastCrawlDurationMS,
134		LastCrawlPagesCount: p.LastCrawlPagesCount,
135		NextRunAtCrawler:    msToTime(p.NextRunAtCrawler),
136		CrawlStartedAt:      msToTime(p.CrawlStartedAt),
137
138		LighthouseState:          p.LighthouseState,
139		LastLighthouseSuccessAt:  msToTime(p.LastLighthouseSuccessAt),
140		LastLighthouseError:      deref(p.LastLighthouseError),
141		LastLighthouseDurationMS: p.LastLighthouseDurationMS,
142		NextLighthouseRunAt:      msToTime(p.NextLighthouseRunAt),
143		LighthouseStartedAt:      msToTime(p.LighthouseStartedAt),
144
145		AlertState: p.AlertState,
146		CreatedAt:  time.UnixMilli(p.CreatedAt).UTC(),
147		UpdatedAt:  time.UnixMilli(p.UpdatedAt).UTC(),
148	}
149
150	// A property with no checks yet reads as 200 rather than 0, so a freshly
151	// added site is not a red zero until its first probe.
152	v.CurrentStatus = 200
153	if len(recent) > 0 {
154		v.CurrentStatus = recent[0].StatusCode
155	}
156
157	// The rolling average covers 31 checks and not 100, so it describes the
158	// last hour and a half.
159	if window := min(len(recent), 31); window > 0 {
160		var sum int64
161		for _, c := range recent[:window] {
162			sum += c.ResponseMS
163		}
164		v.AvgResponseTime = sum / int64(window)
165	}
166
167	if len(recent) > 0 {
168		var up int
169		for _, c := range recent {
170			if c.StatusCode == 200 {
171				up++
172			}
173		}
174		pct := math.Round(float64(up)/float64(len(recent))*1000) / 10
175		v.RecentUptimePct = &pct
176	}
177
178	// Oldest first, so the bar reads left to right like the charts above it.
179	ticks := make([]string, 0, len(recent))
180	for i := len(recent) - 1; i >= 0; i-- {
181		if recent[i].StatusCode == 200 {
182			ticks = append(ticks, "up")
183		} else {
184			ticks = append(ticks, "down")
185		}
186	}
187	if len(ticks) > 30 {
188		ticks = ticks[len(ticks)-30:]
189	}
190	v.RecentTickStream = ticks
191
192	v.applySecurityPosture(recent)
193	v.applyStoredJSON(p)
194
195	return v, nil
196}
197
198var hstsMaxAge = regexp.MustCompile(`max-age=(\d+)`)
199
200// applySecurityPosture reads the latest response's headers rather than probing
201// again, so it costs nothing and can be up to three minutes stale.
202func (v *PropertyView) applySecurityPosture(recent []Check) {
203	headers := map[string]string{}
204	if len(recent) > 0 {
205		var raw map[string]string
206		if err := json.Unmarshal([]byte(recent[0].Headers), &raw); err == nil {
207			for k, val := range raw {
208				headers[strings.ToLower(k)] = strings.ToLower(val)
209			}
210		}
211	}
212
213	v.IsHTTPS = strings.HasPrefix(v.URL, "https://")
214	v.InvalidCert = v.CurrentStatus == 526
215
216	_, v.HasMIMEType = headers["content-type"]
217	v.HasContentSniffProtection = headers["x-content-type-options"] == "nosniff"
218
219	switch headers["x-frame-options"] {
220	case "deny", "sameorigin", "allow-from":
221		v.HasClickjackProtection = true
222	}
223
224	// Four spellings, because all four turn up in the wild and a server
225	// announcing its exact version is naming which CVEs to try.
226	v.HidesServerVersion = true
227	for _, h := range []string{"server", "x-server", "powered-by", "x-powered-by"} {
228		if _, ok := headers[h]; ok {
229			v.HidesServerVersion = false
230			break
231		}
232	}
233
234	hsts := headers["strict-transport-security"]
235	if m := hstsMaxAge.FindStringSubmatch(hsts); m != nil {
236		// A year is what the browser preload lists require, and anything
237		// shorter reports as absent. ParseInt because a twenty digit max-age
238		// is legal text that would overflow a hand-rolled accumulator.
239		if seconds, err := strconv.ParseInt(m[1], 10, 64); err == nil {
240			v.HasHSTS = seconds >= 31_536_000
241		}
242	}
243	v.HasHSTSPreload = strings.Contains(hsts, "preload")
244
245	v.HasSecurityIssue = !v.IsHTTPS ||
246		!v.HasMIMEType ||
247		!v.HasContentSniffProtection ||
248		!v.HasClickjackProtection ||
249		!v.HidesServerVersion ||
250		!v.HasHSTS ||
251		!v.HasHSTSPreload
252}
253
254// applyStoredJSON decodes the three JSON columns, each degrading to empty
255// rather than failing the request, so a malformed result costs one panel.
256func (v *PropertyView) applyStoredJSON(p *Property) {
257	if p.CrawlerInsights != nil {
258		if err := json.Unmarshal([]byte(*p.CrawlerInsights), &v.CrawlerInsights); err != nil {
259			slog.Info(fmt.Sprintf("property %s: crawler insights did not decode: %v", v.ID, err))
260			v.CrawlerInsights = nil
261		}
262	}
263
264	if p.LighthouseScores != nil {
265		var scores Scores
266		if err := json.Unmarshal([]byte(*p.LighthouseScores), &scores); err == nil {
267			v.LighthouseScores = &scores
268			avg := int64(math.Round(float64(
269				scores.Performance+scores.Accessibility+scores.BestPractices+scores.SEO) / 4))
270			v.AvgLighthouseScore = &avg
271		} else {
272			slog.Info(fmt.Sprintf("property %s: lighthouse scores did not decode: %v", v.ID, err))
273		}
274	}
275
276	if p.LighthouseDetails != nil && *p.LighthouseDetails != "null" {
277		var details Details
278		if err := json.Unmarshal([]byte(*p.LighthouseDetails), &details); err == nil {
279			v.LighthouseDetails = &details
280		}
281	}
282}
283
284// groupInsights buckets findings by type and orders each bucket errors first.
285// Types are sorted and the inner sort is stable, so the order holds.
286func groupInsights(insights []Insight) []InsightGroup {
287	buckets := map[string][]Insight{}
288	for _, i := range insights {
289		kind := i.Type
290		if kind == "" {
291			kind = "other"
292		}
293		buckets[kind] = append(buckets[kind], i)
294	}
295
296	kinds := make([]string, 0, len(buckets))
297	for k := range buckets {
298		kinds = append(kinds, k)
299	}
300	sort.Strings(kinds)
301
302	rank := func(sev string) int {
303		switch sev {
304		case sevError:
305			return 0
306		case sevWarn:
307			return 1
308		}
309		return 2
310	}
311
312	groups := make([]InsightGroup, 0, len(kinds))
313	for _, k := range kinds {
314		items := buckets[k]
315		sort.SliceStable(items, func(a, b int) bool {
316			return rank(items[a].Severity) < rank(items[b].Severity)
317		})
318		groups = append(groups, InsightGroup{Type: k, Items: items})
319	}
320	return groups
321}
322
323func (s *site) dashboard(w http.ResponseWriter, r *http.Request, id uuid.UUID) {
324	p, err := getProperty(r.Context(), s.db, id)
325	if err != nil {
326		slog.Info(fmt.Sprintf("dashboard %s: %v", id, err))
327		http.Error(w, "internal server error", http.StatusInternalServerError)
328		return
329	}
330	if p == nil {
331		s.notFound(w, r)
332		return
333	}
334
335	authed := s.auth.Authenticated(r)
336	if !p.IsPublic && !authed {
337		// A redirect to login rather than a 404, so a bookmark lands somewhere
338		// useful. It leaks that the id exists, which is fine for a v4 UUID.
339		http.Redirect(w, r, "/login", http.StatusSeeOther)
340		return
341	}
342
343	view, err := s.buildPropertyView(r.Context(), p)
344	if err != nil {
345		slog.Info(fmt.Sprintf("dashboard %s: %v", id, err))
346		http.Error(w, "internal server error", http.StatusInternalServerError)
347		return
348	}
349
350	data := s.page(r, view.Name, "Status for "+view.Name)
351	data.Property = view
352	data.InsightGroups = groupInsights(view.CrawlerInsights)
353
354	// Report export is operator-only even for a public property, since the PDF
355	// path spawns a Typst compile for anyone who finds the URL.
356	if format := r.URL.Query().Get("report"); format != "" {
357		if !authed {
358			http.Redirect(w, r, "/"+view.ID, http.StatusSeeOther)
359			return
360		}
361		if format != "md" {
362			format = "pdf"
363		}
364		data.GeneratedAt = time.Now().Format("2006-01-02 15:04 MST")
365		s.renderReport(w, r, format, view.Name, data)
366		return
367	}
368
369	recent, err := recentChecks(r.Context(), s.db, id, 31)
370	if err != nil {
371		slog.Info(fmt.Sprintf("dashboard %s charts: %v", id, err))
372	}
373	// Oldest first, so the time axis reads left to right.
374	for i := len(recent) - 1; i >= 0; i-- {
375		c := recent[i]
376		data.ResponseTimes = append(data.ResponseTimes, ResponseTimePoint{
377			Label: time.UnixMilli(c.CreatedAt).UTC().Format(time.RFC3339),
378			Total: c.ResponseMS,
379			DNS:   c.DNSMS,
380			TCP:   c.TCPMS,
381			TLS:   c.TLSMS,
382			TTFB:  c.TTFBMS,
383		})
384	}
385
386	codes, err := countStatusCodes(r.Context(), s.db, id)
387	if err != nil {
388		slog.Info(fmt.Sprintf("dashboard %s status codes: %v", id, err))
389	}
390	for _, c := range codes {
391		data.StatusCodes = append(data.StatusCodes, LabelCount{Label: c.Code, Count: c.Count})
392	}
393
394	up, down, err := countUptime(r.Context(), s.db, id)
395	if err != nil {
396		slog.Info(fmt.Sprintf("dashboard %s uptime: %v", id, err))
397	}
398	pct := func(n int64) float64 {
399		if total := up + down; total > 0 {
400			return math.Round(float64(n)/float64(total)*10000) / 100
401		}
402		return 0
403	}
404	data.UptimeSlices = []LabelPercent{
405		{Label: "Uptime", Count: pct(up)},
406		{Label: "Downtime", Count: pct(down)},
407	}
408
409	data.PageScript = s.propsScript
410	data.PageStyles = s.propsStyles
411	s.renderer.Render(w, http.StatusOK, "property.html", data)
412}
413
414// statusPayload is what the dashboard's polling JavaScript reads. Its shape is
415// fixed by frontend/static_src/properties/scripts/property_crawl_status.js.
416type statusPayload struct {
417	Crawler    crawlerStatus    `json:"crawler"`
418	Lighthouse lighthouseStatus `json:"lighthouse"`
419	ServerTime string           `json:"server_time"`
420	OK         *bool            `json:"ok,omitempty"`
421	Reason     string           `json:"reason,omitempty"`
422}
423
424type crawlerStatus struct {
425	State              string         `json:"state"`
426	StartedAt          *string        `json:"started_at"`
427	LastAttemptAt      *string        `json:"last_attempt_at"`
428	LastSuccessAt      *string        `json:"last_success_at"`
429	LastError          *string        `json:"last_error"`
430	LastDurationMS     *int64         `json:"last_duration_ms"`
431	PagesCount         *int64         `json:"pages_count"`
432	NextRunAt          *string        `json:"next_run_at"`
433	IsOverdue          bool           `json:"is_overdue"`
434	InsightsTotal      int            `json:"insights_total"`
435	InsightsBySeverity map[string]int `json:"insights_by_severity"`
436	Progress           *float64       `json:"progress"`
437}
438
439type lighthouseStatus struct {
440	State          string  `json:"state"`
441	StartedAt      *string `json:"started_at"`
442	LastAttemptAt  *string `json:"last_attempt_at"`
443	LastSuccessAt  *string `json:"last_success_at"`
444	LastError      *string `json:"last_error"`
445	LastDurationMS *int64  `json:"last_duration_ms"`
446	NextRunAt      *string `json:"next_run_at"`
447	IsOverdue      bool    `json:"is_overdue"`
448	Scores         *Scores `json:"scores"`
449}
450
451func isoOrNil(ms *int64) *string {
452	if ms == nil {
453		return nil
454	}
455	s := time.UnixMilli(*ms).UTC().Format(time.RFC3339)
456	return &s
457}
458
459func nilIfEmpty(s *string) *string {
460	if s == nil || *s == "" {
461		return nil
462	}
463	return s
464}
465
466// crawlProgress estimates a running crawl against PageCap, capped at 0.9 so
467// the bar never claims to be finished while it is still working.
468func crawlProgress(p *Property) *float64 {
469	if p.CrawlState != "running" {
470		return nil
471	}
472	pages := int64(0)
473	if p.LastCrawlPagesCount != nil {
474		pages = *p.LastCrawlPagesCount
475	}
476	progress := 0.05
477	if pages > 0 {
478		progress = math.Min(float64(pages)/float64(PageCap), 0.9)
479	}
480	return &progress
481}
482
483func buildStatusPayload(p *Property) statusPayload {
484	now := nowMS()
485
486	severity := map[string]int{sevError: 0, sevWarn: 0, sevInfo: 0}
487	insightsTotal := 0
488	if p.CrawlerInsights != nil {
489		var insights []Insight
490		if err := json.Unmarshal([]byte(*p.CrawlerInsights), &insights); err == nil {
491			insightsTotal = len(insights)
492			for _, i := range insights {
493				sev := i.Severity
494				if _, known := severity[sev]; !known {
495					sev = sevInfo
496				}
497				severity[sev]++
498			}
499		}
500	}
501
502	overdue := func(next *int64) bool { return next != nil && *next <= now }
503
504	payload := statusPayload{
505		Crawler: crawlerStatus{
506			State:              p.CrawlState,
507			StartedAt:          isoOrNil(p.CrawlStartedAt),
508			LastAttemptAt:      isoOrNil(p.LastRunAtCrawler),
509			LastSuccessAt:      isoOrNil(p.LastCrawlSuccessAt),
510			LastError:          nilIfEmpty(p.LastCrawlError),
511			LastDurationMS:     p.LastCrawlDurationMS,
512			PagesCount:         p.LastCrawlPagesCount,
513			NextRunAt:          isoOrNil(p.NextRunAtCrawler),
514			IsOverdue:          overdue(p.NextRunAtCrawler),
515			InsightsTotal:      insightsTotal,
516			InsightsBySeverity: severity,
517			Progress:           crawlProgress(p),
518		},
519		Lighthouse: lighthouseStatus{
520			State:          p.LighthouseState,
521			StartedAt:      isoOrNil(p.LighthouseStartedAt),
522			LastAttemptAt:  isoOrNil(p.LastLighthouseRunAt),
523			LastSuccessAt:  isoOrNil(p.LastLighthouseSuccessAt),
524			LastError:      nilIfEmpty(p.LastLighthouseError),
525			LastDurationMS: p.LastLighthouseDurationMS,
526			NextRunAt:      isoOrNil(p.NextLighthouseRunAt),
527			IsOverdue:      overdue(p.NextLighthouseRunAt),
528		},
529		ServerTime: time.Now().UTC().Format(time.RFC3339),
530	}
531
532	if p.LighthouseScores != nil {
533		var scores Scores
534		if err := json.Unmarshal([]byte(*p.LighthouseScores), &scores); err == nil {
535			payload.Lighthouse.Scores = &scores
536		}
537	}
538
539	return payload
540}
541
542// writeJSON is the single place a JSON response is written, so the header and
543// the encoder settings cannot drift between endpoints.
544func writeJSON(w http.ResponseWriter, status int, payload any) {
545	w.Header().Set("Content-Type", "application/json; charset=utf-8")
546	w.WriteHeader(status)
547	enc := json.NewEncoder(w)
548	// Go escapes <, > and & by default, which corrupts a response fetch()
549	// parses, and these error strings come from crawled sites.
550	enc.SetEscapeHTML(false)
551	if err := enc.Encode(payload); err != nil {
552		slog.Info(fmt.Sprintf("write json: %v", err))
553	}
554}
555
556func (s *site) propertyStatus(w http.ResponseWriter, r *http.Request) {
557	p, ok := s.lookupForJSON(w, r)
558	if !ok {
559		return
560	}
561	if !p.IsPublic && !s.auth.Authenticated(r) {
562		writeJSON(w, http.StatusForbidden, map[string]any{"error": "forbidden"})
563		return
564	}
565	writeJSON(w, http.StatusOK, buildStatusPayload(p))
566}
567
568// lookupForJSON resolves the {id} path value, answering in JSON on failure.
569func (s *site) lookupForJSON(w http.ResponseWriter, r *http.Request) (*Property, bool) {
570	id, err := uuid.Parse(r.PathValue("id"))
571	if err != nil {
572		writeJSON(w, http.StatusNotFound, map[string]any{"error": "not_found"})
573		return nil, false
574	}
575	p, err := getProperty(r.Context(), s.db, id)
576	if err != nil {
577		slog.Info(fmt.Sprintf("lookup %s: %v", id, err))
578		writeJSON(w, http.StatusInternalServerError, map[string]any{"error": "server_error"})
579		return nil, false
580	}
581	if p == nil {
582		writeJSON(w, http.StatusNotFound, map[string]any{"error": "not_found"})
583		return nil, false
584	}
585	return p, true
586}
587
588// requeue is the shared body of the two "run it now" buttons. It runs nothing
589// itself, it moves the due time into the past for the scheduler to pick up.
590func (s *site) requeue(w http.ResponseWriter, r *http.Request, kind string) {
591	p, ok := s.lookupForJSON(w, r)
592	if !ok {
593		return
594	}
595
596	var state, dueColumn, errColumn string
597	switch kind {
598	case "crawl":
599		state, dueColumn, errColumn = p.CrawlState, "next_run_at_crawler", "last_crawl_error"
600	default:
601		state, dueColumn, errColumn = p.LighthouseState, "next_lighthouse_run_at", "last_lighthouse_error"
602	}
603
604	if state == "queued" || state == "running" {
605		payload := buildStatusPayload(p)
606		no := false
607		payload.OK = &no
608		payload.Reason = "already_running"
609		writeJSON(w, http.StatusConflict, payload)
610		return
611	}
612
613	now := nowMS()
614	if _, err := s.db.ExecContext(r.Context(),
615		"UPDATE properties SET "+dueColumn+" = ?, "+errColumn+" = NULL, updated_at = ? WHERE id = ?",
616		now, now, p.ID[:]); err != nil {
617		slog.Info(fmt.Sprintf("requeue %s for %s: %v", kind, p.URL, err))
618		writeJSON(w, http.StatusInternalServerError, map[string]any{"ok": false, "error": "server_error"})
619		return
620	}
621
622	updated, err := getProperty(r.Context(), s.db, p.ID)
623	if err != nil || updated == nil {
624		updated = p
625	}
626	payload := buildStatusPayload(updated)
627	yes := true
628	payload.OK = &yes
629	writeJSON(w, http.StatusOK, payload)
630}
631
632func (s *site) propertyRecrawl(w http.ResponseWriter, r *http.Request) {
633	s.requeue(w, r, "crawl")
634}
635
636func (s *site) propertyRerunLighthouse(w http.ResponseWriter, r *http.Request) {
637	s.requeue(w, r, "lighthouse")
638}