orchard

  1package main
  2
  3import (
  4	"context"
  5	"errors"
  6	"fmt"
  7	"net"
  8	"net/http"
  9	"strings"
 10	"sync"
 11	"time"
 12)
 13
 14// The health strip. This is a public page, so a row carries a name, an up or
 15// down, a latency and a count of errors, and never a path, a message, a status
 16// code or an address. What logging hands over is already limited to that, and
 17// the limit lives at both ends so neither has to remember for the other.
 18//
 19// This is not a second status.bythewood.me. There are no phase timings, no
 20// history and no alerting here, because status already does all of that
 21// properly and a strip of dots is what a dashboard wants.
 22
 23// probeEvery is the strip's cadence. Slower than the markets panel because a
 24// site that has been up for eight hours is not news, and every probe is against
 25// Isaac's own machine rather than someone else's endpoint.
 26const probeEvery = 60 * time.Second
 27
 28// aggregateURL is a container name on the orchard-edge bridge, never the public
 29// hostname. Caddy refuses this path from outside.
 30const aggregateURL = "http://orchard-logging:8000/aggregate"
 31
 32// Monitored is one row. Source is the label it ships its logs under and, for a
 33// site, the first hostname label, so it is also half the container name.
 34type Monitored struct {
 35	Label  string
 36	Source string
 37
 38	// Host is the public hostname, and is empty for something that has none.
 39	// Without one there is no fallback probe, so it reads unknown rather than
 40	// down wherever the bridge is not reachable.
 41	Host string
 42
 43	// Bridge overrides the usual <container>:8000/healthz.
 44	Bridge string
 45
 46	// AnyStatus means any HTTP response at all proves the process is serving.
 47	// Caddy has no health endpoint and answers its catch-all with a 404, which
 48	// is still Caddy answering.
 49	AnyStatus bool
 50}
 51
 52var monitored = []Monitored{
 53	{Label: "Portfolio", Source: "isaacbythewood", Host: "isaacbythewood.com"},
 54	{Label: "Blog", Source: "blog", Host: "blog.bythewood.me"},
 55	{Label: "Analytics", Source: "analytics", Host: "analytics.bythewood.me"},
 56	{Label: "Auth", Source: "auth", Host: "auth.bythewood.me"},
 57	{Label: "Status", Source: "status", Host: "status.bythewood.me"},
 58	{Label: "Logging", Source: "logging", Host: "logging.bythewood.me"},
 59	{Label: "Repos", Source: "repos", Host: "repos.bythewood.me"},
 60	{Label: "Search", Source: "search", Host: "search.bythewood.me"},
 61	{Label: "Chat", Source: "chat", Host: "chat.bythewood.me"},
 62	{Label: "LLM", Source: "llm", Host: "llm.bythewood.me"},
 63	{Label: "Dash", Source: "dash", Host: "dash.bythewood.me"},
 64
 65	// The edge itself, and the only part of it on this strip: Caddy is the one
 66	// that ships its access log, so it is the one with an error count to show
 67	// beside the others. cloudflared and ntfy log to stdout and cannot write to
 68	// a socket, so neither has anything here to be counted.
 69	{Label: "Caddy", Source: "caddy", Bridge: "http://orchard-caddy:80/", AnyStatus: true},
 70}
 71
 72type SystemRow struct {
 73	Label     string `json:"label"`
 74	Host      string `json:"host"`
 75	URL       string `json:"url"`
 76	State     string `json:"state"`
 77	Errors    int64  `json:"errors"`
 78	KnowError bool   `json:"know_error"`
 79
 80	// How long this site takes to answer at the slow end of a day, read off
 81	// its own access log rather than measured here. The probe runs over the
 82	// Docker bridge, so timing it reported the bridge: every site came back
 83	// between one and two milliseconds whatever it was actually doing.
 84	Response string `json:"response"`
 85
 86	// Traffic over the same window as Errors, plus where it sits against this
 87	// site's own preceding week. Busy is relative here or it is meaningless:
 88	// a day that would be dead for a news site is a good day for a personal
 89	// blog, so the comparison is always against the site itself.
 90	Requests int64  `json:"requests"`
 91	KnowTraf bool   `json:"know_traf"`
 92	Level    int    `json:"level"`
 93	Trend    string `json:"trend"`
 94}
 95
 96type Systems struct {
 97	Rows     []SystemRow `json:"rows"`
 98	Up       int         `json:"up"`
 99	Total    int         `json:"total"`
100	Errors   int64       `json:"errors"`
101	Requests int64       `json:"requests"`
102	Window   int         `json:"window"`
103	Checked  string      `json:"checked"`
104}
105
106// probeClient is separate from the shared JSON client because these are health
107// probes and a pooled connection would report a warm socket rather than a
108// reachable site. Nothing here is trying to be a real uptime measurement, that
109// is what status is for, but a probe that reuses a socket to a container that
110// died two seconds ago is wrong rather than approximate.
111var probeClient = &http.Client{
112	Timeout: 5 * time.Second,
113	Transport: &http.Transport{
114		DisableKeepAlives: true,
115	},
116	// A health check that follows a redirect is checking somewhere else.
117	CheckRedirect: func(*http.Request, []*http.Request) error {
118		return http.ErrUseLastResponse
119	},
120}
121
122// probeResult separates the three things a probe attempt can mean. Collapsing
123// them was a real bug: the goroutines start at once, the guard paced all but one of
124// them, and a paced probe read as an unreachable site, so the strip reported
125// every other site down while every one of them was serving.
126type probeResult int
127
128const (
129	probeAnswered probeResult = iota
130	// The name does not resolve, so this is the wrong route rather than a
131	// failure and the caller should try the next one.
132	probeNoRoute
133	// The guard said no, so nothing went out and nothing was learned.
134	probeSkipped
135)
136
137// probe answers whether one site is serving, and how fast.
138//
139// The container on the bridge is asked first and the public hostname is only a
140// fallback, because Cloudflare will happily serve a cached 200 for /healthz
141// long after the origin behind it has stopped answering. Two of these sites
142// return CF-Cache-Status: HIT for that path right now, so a public probe is not
143// evidence the site is up. When the fallback is what answered, the row says
144// cached rather than up and means it.
145func probe(ctx context.Context, g *Guard, m Monitored) SystemRow {
146	row := SystemRow{Label: m.Label, Host: m.Host, State: "unknown"}
147	if m.Host != "" {
148		row.URL = visitURL(m.Host)
149	}
150
151	bridge := m.Bridge
152	switch {
153	case bridge != "":
154	case m.Source == selfSource:
155		// Asking the bridge for our own name works in the container and not in
156		// a dev run, and loopback is the same answer in both.
157		bridge = "http://127.0.0.1:8000/healthz"
158	default:
159		bridge = "http://orchard-" + m.Source + ":8000/healthz"
160	}
161
162	attempts := []struct {
163		url    string
164		public bool
165	}{{bridge, false}}
166	if m.Host != "" {
167		attempts = append(attempts, struct {
168			url    string
169			public bool
170		}{"https://" + m.Host + "/healthz", true})
171	}
172
173	for _, attempt := range attempts {
174		state, result := hit(ctx, g, attempt.url, attempt.public, m.AnyStatus)
175		switch result {
176		case probeAnswered:
177			row.State = state
178			return row
179		case probeSkipped:
180			// Nothing was measured, so the row stays unknown rather than
181			// claiming a site is down on the strength of a paced request.
182			return row
183		}
184	}
185
186	return row
187}
188
189// hit runs one request. public marks a request that went over the internet, so
190// an edge cache hit can be reported as what it is.
191func hit(ctx context.Context, g *Guard, url string, public, anyStatus bool) (state string, result probeResult) {
192	if err := g.Allow("uptime"); err != nil {
193		return "", probeSkipped
194	}
195
196	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
197	if err != nil {
198		return "", probeSkipped
199	}
200	req.Header.Set("User-Agent", "dash.bythewood.me health strip")
201	req.Header.Set("Cache-Control", "no-cache")
202
203	resp, err := probeClient.Do(req)
204	if err != nil {
205		var dns *net.DNSError
206		if errors.As(err, &dns) {
207			return "", probeNoRoute
208		}
209		g.Fail("uptime", 0, 0)
210		return "down", probeAnswered
211	}
212	defer resp.Body.Close()
213
214	g.Succeed("uptime")
215
216	state = "down"
217	if anyStatus || resp.StatusCode == http.StatusOK {
218		state = "up"
219		if public && cacheHit(resp.Header.Get("CF-Cache-Status")) {
220			state = "cached"
221		}
222	}
223	return state, probeAnswered
224}
225
226// humanMS keeps a response time to six characters. These sites answer an
227// embedded page in a fraction of a millisecond and repos answers a git pack in a
228// fifth of a second, so the column has to carry four orders of magnitude without
229// going ragged. Anything under a millisecond is still printed rather than
230// rounded to "<1ms", or half the strip reads as the same number when the sites
231// are a factor of ten apart.
232func humanMS(ms float64) string {
233	switch {
234	case ms <= 0:
235		return ""
236	case ms >= 1000:
237		return fmt.Sprintf("%.1fs", ms/1000)
238	case ms >= 10:
239		return fmt.Sprintf("%.0fms", ms)
240	case ms >= 1:
241		return fmt.Sprintf("%.1fms", ms)
242	default:
243		return fmt.Sprintf("%.2fms", ms)
244	}
245}
246
247func cacheHit(status string) bool {
248	switch strings.ToUpper(status) {
249	case "HIT", "STALE", "UPDATING", "REVALIDATED":
250		return true
251	}
252	return false
253}
254
255type aggregatePayload struct {
256	WindowHours int `json:"window_hours"`
257	Sources     []struct {
258		Source        string  `json:"source"`
259		Errors        int64   `json:"errors"`
260		Requests      int64   `json:"requests"`
261		BaselineDaily float64 `json:"baseline_daily"`
262		P95MS         float64 `json:"p95_ms"`
263	} `json:"sources"`
264}
265
266// buildSystems probes every site at once and folds in the error counts logging
267// keeps. A logging site that is down costs the strip its error column and
268// nothing else.
269func buildSystems(ctx context.Context, g *Guard, now time.Time) Systems {
270	rows := make([]SystemRow, len(monitored))
271
272	var wg sync.WaitGroup
273	for i, m := range monitored {
274		wg.Add(1)
275		go func() {
276			defer wg.Done()
277			rows[i] = probe(ctx, g, m)
278		}()
279	}
280	wg.Wait()
281
282	s := Systems{Rows: rows, Total: len(rows), Checked: now.UTC().Format("15:04:05")}
283
284	var payload aggregatePayload
285	if err := getJSON(ctx, g, "logging", aggregateURL, &payload); err == nil {
286		type counts struct {
287			errors, requests int64
288			baseline, p95    float64
289		}
290		by := map[string]counts{}
291		var busiest int64
292		for _, src := range payload.Sources {
293			by[src.Source] = counts{src.Errors, src.Requests, src.BaselineDaily, src.P95MS}
294			busiest = max(busiest, src.Requests)
295		}
296
297		s.Window = payload.WindowHours
298		for i, m := range monitored {
299			c, ok := by[m.Source]
300			if !ok {
301				continue
302			}
303			rows[i].Errors, rows[i].KnowError = c.errors, true
304			rows[i].Requests, rows[i].KnowTraf = c.requests, true
305			rows[i].Level = trafficLevel(c.requests, busiest)
306			rows[i].Trend = trafficTrend(c.requests, c.baseline)
307			rows[i].Response = humanMS(c.p95)
308			s.Errors += c.errors
309			s.Requests += c.requests
310		}
311	}
312
313	for _, r := range rows {
314		if r.State == "up" || r.State == "cached" {
315			s.Up++
316		}
317	}
318	return s
319}
320
321// visitURL is what the strip links to. The campaign tag is so the sites can see
322// in their own analytics that a visit came from here, which is the only way to
323// tell dash traffic apart from anything else arriving at a bare hostname.
324func visitURL(host string) string {
325	return "https://" + host + "/?utm_source=dash.bythewood.me&utm_medium=referral&utm_campaign=systems"
326}
327
328// trafficLevel puts a site's day on a four step scale against the busiest site
329// here, so the bars compare like with like on one machine rather than against
330// some idea of what a busy site is.
331func trafficLevel(requests, busiest int64) int {
332	if requests <= 0 || busiest <= 0 {
333		return 0
334	}
335	switch share := float64(requests) / float64(busiest); {
336	case share >= 0.6:
337		return 4
338	case share >= 0.3:
339		return 3
340	case share >= 0.1:
341		return 2
342	default:
343		return 1
344	}
345}
346
347// trafficTrend compares the day against this site's own preceding week. The
348// band is wide because a personal site's daily traffic is noisy enough that
349// anything tighter would call every day unusual.
350func trafficTrend(requests int64, baselineDaily float64) string {
351	if baselineDaily <= 0 {
352		return ""
353	}
354	switch ratio := float64(requests) / baselineDaily; {
355	case ratio >= 1.5:
356		return "up"
357	case ratio <= 0.5:
358		return "down"
359	default:
360		return "flat"
361	}
362}