orchard

  1package main
  2
  3import (
  4	"context"
  5	"database/sql"
  6	"encoding/json"
  7	"fmt"
  8	"io"
  9	"log/slog"
 10	"net/http"
 11	"net/netip"
 12	"strings"
 13	"time"
 14
 15	"analytics.bythewood.me/web"
 16	"github.com/google/uuid"
 17)
 18
 19// The collector is the one cross-origin endpoint and the only one an anonymous
 20// stranger can write to.
 21const (
 22	// The only bound on a table that grows forever.
 23	maxCollectBody = 16 * 1024
 24	// Each distinct name becomes a dashboard card, so an oversized one is
 25	// rejected rather than truncated into a collision with a real event.
 26	maxEventNameLen = 200
 27	// The clamp on every stored string field; these columns are indexed and
 28	// rendered into breakdown tables.
 29	maxFieldLen = 2048
 30)
 31
 32// collectRequest is the wire format the embed script sends.
 33type collectRequest struct {
 34	CollectorID string          `json:"collectorId"`
 35	Event       string          `json:"event"`
 36	Data        json.RawMessage `json:"data"`
 37}
 38
 39// collect records one event, answering 204 with no body. It never says whether
 40// the event was filed as a bot, which would give a bot something to tune against.
 41func (s *site) collect(w http.ResponseWriter, r *http.Request) {
 42	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxCollectBody))
 43	if err != nil || len(body) == 0 {
 44		corsStatus(w, r, http.StatusBadRequest)
 45		return
 46	}
 47
 48	var req collectRequest
 49	if err := json.Unmarshal(body, &req); err != nil {
 50		corsStatus(w, r, http.StatusBadRequest)
 51		return
 52	}
 53	if req.CollectorID == "" || req.Event == "" {
 54		corsStatus(w, r, http.StatusBadRequest)
 55		return
 56	}
 57	if len([]rune(req.Event)) > maxEventNameLen {
 58		corsStatus(w, r, http.StatusBadRequest)
 59		return
 60	}
 61	propertyID, err := uuid.Parse(req.CollectorID)
 62	if err != nil {
 63		corsStatus(w, r, http.StatusBadRequest)
 64		return
 65	}
 66
 67	ctx := r.Context()
 68
 69	// An unknown id answers 404, the only signal that a snippet was pasted with
 70	// the wrong one.
 71	var found []byte
 72	switch err := s.db.QueryRowContext(ctx,
 73		"SELECT id FROM properties WHERE id = ?", propertyID[:]).Scan(&found); {
 74	case err == sql.ErrNoRows:
 75		corsStatus(w, r, http.StatusNotFound)
 76		return
 77	case err != nil:
 78		slog.Info(fmt.Sprintf("collect: property lookup: %v", err))
 79		corsStatus(w, r, http.StatusInternalServerError)
 80		return
 81	}
 82
 83	data := map[string]any{}
 84	if len(req.Data) > 0 {
 85		// A non-object data field is ignored; the event is still worth recording.
 86		_ = json.Unmarshal(req.Data, &data)
 87	}
 88
 89	if ref, ok := data["referrer"].(string); ok {
 90		data["referrer"] = normalizeReferrer(ref)
 91	}
 92
 93	// Every later event in a session comes from the same address, so geo is
 94	// looked up once.
 95	if req.Event == "session_start" {
 96		s.enrichGeo(r, data)
 97	}
 98
 99	uaString, _ := data["user_agent"].(string)
100	if uaString == "" {
101		uaString = r.Header.Get("User-Agent")
102	}
103
104	if uaString != "" {
105		parsed := s.ua.Parse(uaString)
106		putIfNotEmpty(data, "platform", parsed.Platform)
107		putIfNotEmpty(data, "browser", parsed.Browser)
108		putIfNotEmpty(data, "device", parsed.Device)
109
110		if parsed.IsBot {
111			data["is_bot"] = true
112			putIfNotEmpty(data, "bot_name", parsed.BotName)
113			s.insertBotEvent(ctx, propertyID, req.Event, uaString, parsed.BotName, data)
114			corsStatus(w, r, http.StatusNoContent)
115			return
116		}
117	}
118
119	s.insertEvent(ctx, propertyID, req.Event, uaString, data)
120	corsStatus(w, r, http.StatusNoContent)
121}
122
123// enrichGeo writes country, region, city and coordinates onto a session_start.
124// web.ClientIP prefers CF-Connecting-IP; the last X-Forwarded-For entry behind
125// the tunnel is cloudflared's own bridge address, and resolves without erroring.
126func (s *site) enrichGeo(r *http.Request, data map[string]any) {
127	addr, err := netip.ParseAddr(web.ClientIP(r))
128	if err != nil || addr.IsLoopback() {
129		return
130	}
131	geo, ok := s.geoip.Lookup(addr)
132	if !ok {
133		return
134	}
135	putIfNotEmpty(data, "country", geo.Country)
136	putIfNotEmpty(data, "region", geo.Region)
137	putIfNotEmpty(data, "city", geo.City)
138	if geo.HasLoc {
139		data["loc"] = []any{geo.Lat, geo.Lon}
140	}
141}
142
143func putIfNotEmpty(m map[string]any, key, value string) {
144	if value != "" {
145		m[key] = value
146	}
147}
148
149// normalizeReferrer reduces a referrer to a bare hostname without "www.", so
150// the breakdown is one row per site rather than per URL.
151func normalizeReferrer(ref string) string {
152	host := ref
153	if i := strings.Index(host, "://"); i >= 0 {
154		host = host[i+3:]
155	}
156	if i := strings.Index(host, "/"); i >= 0 {
157		host = host[:i]
158	}
159	return strings.TrimPrefix(strings.ToLower(host), "www.")
160}
161
162// insertEvent writes a human event, lifting the hot fields out of the payload
163// into typed columns and leaving whatever the site sent of its own in extra.
164func (s *site) insertEvent(ctx context.Context, propertyID uuid.UUID, event, userAgent string, data map[string]any) {
165	// Each take removes the key, so what is left in data is the caller's own
166	// fields and goes to extra.
167	userID := takeString(data, "user_id")
168	url := takeString(data, "url")
169	title := takeString(data, "title")
170	referrer := takeString(data, "referrer")
171	delete(data, "user_agent")
172	platform := takeString(data, "platform")
173	browser := takeString(data, "browser")
174	device := takeString(data, "device")
175	screenWidth := takeInt(data, "screen_width")
176	screenHeight := takeInt(data, "screen_height")
177	country := takeString(data, "country")
178	region := takeString(data, "region")
179	city := takeString(data, "city")
180	lat, lon := takeLoc(data)
181	utmSource := takeString(data, "utm_source")
182	utmMedium := takeString(data, "utm_medium")
183	utmCampaign := takeString(data, "utm_campaign")
184	utmTerm := takeString(data, "utm_term")
185	utmContent := takeString(data, "utm_content")
186	// The wire field is time_on_page, the column time_on_page_ms. Renaming
187	// either end breaks the other.
188	timeOnPage := takeInt(data, "time_on_page")
189
190	_, err := s.db.ExecContext(ctx, `INSERT INTO events (
191	    property_id, event, created_at, user_id, url, title, referrer, user_agent,
192	    platform, browser, device, screen_width, screen_height, country, region, city,
193	    lat, lon, utm_source, utm_medium, utm_campaign, utm_term, utm_content,
194	    time_on_page_ms, extra
195	  ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
196		propertyID[:], event, time.Now().UnixMilli(), userID, url, title, referrer,
197		nullString(userAgent), platform, browser, device, screenWidth, screenHeight,
198		country, region, city, lat, lon, utmSource, utmMedium, utmCampaign, utmTerm,
199		utmContent, timeOnPage, encodeExtra(data))
200	if err != nil {
201		slog.Info(fmt.Sprintf("collect: insert event: %v", err))
202	}
203}
204
205// insertBotEvent writes to the separate bot table, so no human aggregation has
206// to remember an is_bot filter.
207func (s *site) insertBotEvent(ctx context.Context, propertyID uuid.UUID, event, userAgent, botName string, data map[string]any) {
208	_, err := s.db.ExecContext(ctx, `INSERT INTO bot_events (
209	    property_id, event, created_at, bot_name, url, user_agent, country, extra
210	  ) VALUES (?,?,?,?,?,?,?,?)`,
211		propertyID[:], event, time.Now().UnixMilli(), nullString(botName),
212		nullString(stringField(data, "url")), nullString(userAgent),
213		nullString(stringField(data, "country")), encodeExtra(data))
214	if err != nil {
215		slog.Info(fmt.Sprintf("collect: insert bot event: %v", err))
216	}
217}
218
219// encodeExtra serialises whatever is left of the payload. SetEscapeHTML(false)
220// because this is stored, not rendered; the default writes \u003c for a "<".
221func encodeExtra(data map[string]any) string {
222	if len(data) == 0 {
223		return "{}"
224	}
225	var sb strings.Builder
226	enc := json.NewEncoder(&sb)
227	enc.SetEscapeHTML(false)
228	if err := enc.Encode(data); err != nil {
229		return "{}"
230	}
231	return strings.TrimRight(sb.String(), "\n")
232}
233
234// takeString removes a key and returns it as a NULL-able clamped string. A
235// non-string value is stringified rather than dropped.
236func takeString(data map[string]any, key string) any {
237	v, ok := data[key]
238	if !ok {
239		return nil
240	}
241	delete(data, key)
242
243	var s string
244	switch t := v.(type) {
245	case string:
246		s = t
247	case nil:
248		return nil
249	default:
250		b, err := json.Marshal(t)
251		if err != nil {
252			return nil
253		}
254		s = string(b)
255	}
256	if s == "" {
257		return nil
258	}
259	return clampRunes(s, maxFieldLen)
260}
261
262func takeInt(data map[string]any, key string) any {
263	v, ok := data[key]
264	if !ok {
265		return nil
266	}
267	delete(data, key)
268	// Every number out of encoding/json is a float64, integers included.
269	if f, ok := v.(float64); ok {
270		return int64(f)
271	}
272	return nil
273}
274
275// takeLoc splits the [lat, lon] pair the geo enrichment writes.
276func takeLoc(data map[string]any) (any, any) {
277	v, ok := data["loc"]
278	if !ok {
279		return nil, nil
280	}
281	delete(data, "loc")
282	arr, ok := v.([]any)
283	if !ok || len(arr) < 2 {
284		return nil, nil
285	}
286	lat, latOK := arr[0].(float64)
287	lon, lonOK := arr[1].(float64)
288	if !latOK || !lonOK {
289		return nil, nil
290	}
291	return lat, lon
292}
293
294func stringField(data map[string]any, key string) string {
295	s, _ := data[key].(string)
296	return s
297}
298
299// nullString stores an empty value as NULL, since every breakdown query filters
300// on IS NOT NULL and an empty string would be a row labelled with nothing.
301func nullString(s string) any {
302	if s == "" {
303		return nil
304	}
305	return s
306}
307
308// clampRunes truncates on character boundaries, not bytes, so a clamped string
309// is never invalid UTF-8.
310func clampRunes(s string, max int) string {
311	if len([]rune(s)) <= max {
312		return s
313	}
314	return string([]rune(s)[:max])
315}
316
317// collectOptions answers the CORS preflight.
318func (s *site) collectOptions(w http.ResponseWriter, r *http.Request) {
319	h := w.Header()
320	h.Set("Allow", "OPTIONS, POST")
321	h.Set("Access-Control-Allow-Methods", "OPTIONS, POST")
322
323	reqHeaders := r.Header.Get("Access-Control-Request-Headers")
324	if reqHeaders == "" {
325		reqHeaders = "Content-Type"
326	}
327	h.Set("Access-Control-Allow-Headers", reqHeaders)
328	h.Set("Access-Control-Allow-Origin", originOrWildcard(r))
329	w.WriteHeader(http.StatusNoContent)
330}
331
332// corsStatus answers with the permissive origin header the collector needs. Any
333// site may be tracked, so there is no allowlist; nothing is read back here and
334// no credentials are accepted.
335func corsStatus(w http.ResponseWriter, r *http.Request, status int) {
336	w.Header().Set("Access-Control-Allow-Origin", originOrWildcard(r))
337	w.WriteHeader(status)
338}
339
340func originOrWildcard(r *http.Request) string {
341	if o := r.Header.Get("Origin"); o != "" {
342		return o
343	}
344	return "*"
345}
346
347// collectorScript serves the embed script at a stable URL, resolving Vite's
348// content hash per request because pasted snippets hardcode this path. The
349// short cache is because the name is stable and the bytes are not.
350func (s *site) collectorScript(w http.ResponseWriter, r *http.Request) {
351	name := s.assets.Script("static_src/collector/index.js")
352	if name == "" {
353		slog.Info("collector entry missing from vite manifest")
354		http.Error(w, "collector unavailable", http.StatusServiceUnavailable)
355		return
356	}
357
358	f, err := s.dist.Open(strings.TrimPrefix(name, "/static/"))
359	if err != nil {
360		slog.Info(fmt.Sprintf("collector open %s: %v", name, err))
361		http.Error(w, "collector unavailable", http.StatusServiceUnavailable)
362		return
363	}
364	defer f.Close()
365
366	w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
367	w.Header().Set("Cache-Control", "public, max-age=300, must-revalidate")
368
369	body, err := io.ReadAll(f)
370	if err != nil {
371		slog.Info(fmt.Sprintf("collector read %s: %v", name, err))
372		http.Error(w, "collector unavailable", http.StatusServiceUnavailable)
373		return
374	}
375	_, _ = w.Write(body)
376}