Self-hostable uptime monitor and status page on Django: HTTP checks, Lighthouse audits, SEO crawls, and email and Discord alerts.
djangodockerhandcodedpythonself-hostedsqlitestatus-pageuptime-monitoringvite
1"""SEO / accessibility / performance / content / security checks.
2
3Every check takes a single `ctx` dict and returns a list of insight dicts.
4Insight shape: {url, issue, item, type, severity}
5"""
6from urllib.parse import urlparse
7
8from .fetcher import same_site
9
10
11TYPE_SEO = "seo"
12TYPE_LINKS = "links"
13TYPE_ACCESSIBILITY = "accessibility"
14TYPE_CONTENT = "content"
15TYPE_PERFORMANCE = "performance"
16TYPE_SECURITY = "security"
17
18SEVERITY_ERROR = "error"
19SEVERITY_WARNING = "warning"
20SEVERITY_INFO = "info"
21
22REDIRECT_CODES = {301, 302, 303, 307, 308}
23
24
25def _insight(url, issue, type_, severity, item=""):
26 return {
27 "url": url,
28 "issue": issue,
29 "item": item,
30 "type": type_,
31 "severity": severity,
32 }
33
34
35def _normalize(s):
36 return " ".join(s.lower().split()) if s else ""
37
38
39def _group_by(pages, field):
40 seen = {}
41 for p in pages:
42 val = p.get(field, "")
43 if val:
44 seen.setdefault(_normalize(val), []).append(p)
45 return seen
46
47
48# ---------- core metadata ----------
49
50def check_title_missing(ctx):
51 return [
52 _insight(p["url"], "Page has no title", TYPE_SEO, SEVERITY_ERROR)
53 for p in ctx["html_pages"]
54 if not p.get("title")
55 ]
56
57
58def check_title_length(ctx):
59 out = []
60 for p in ctx["html_pages"]:
61 t = p.get("title", "")
62 if t and not (30 <= len(t) <= 60):
63 out.append(
64 _insight(
65 p["url"],
66 f"Title length is {len(t)} chars (recommended 30-60)",
67 TYPE_SEO,
68 SEVERITY_WARNING,
69 item=t,
70 )
71 )
72 return out
73
74
75def check_duplicate_titles(ctx):
76 out = []
77 for _, group in _group_by(ctx["html_pages"], "title").items():
78 if len(group) > 1:
79 for p in group:
80 out.append(
81 _insight(
82 p["url"],
83 "Duplicate title",
84 TYPE_SEO,
85 SEVERITY_WARNING,
86 item=p["title"],
87 )
88 )
89 return out
90
91
92def check_description_missing(ctx):
93 return [
94 _insight(p["url"], "Page has no meta description", TYPE_SEO, SEVERITY_ERROR)
95 for p in ctx["html_pages"]
96 if not p.get("description")
97 ]
98
99
100def check_description_length(ctx):
101 out = []
102 for p in ctx["html_pages"]:
103 d = p.get("description", "")
104 if d and not (70 <= len(d) <= 160):
105 out.append(
106 _insight(
107 p["url"],
108 f"Description length is {len(d)} chars (recommended 70-160)",
109 TYPE_SEO,
110 SEVERITY_WARNING,
111 item=d,
112 )
113 )
114 return out
115
116
117def check_duplicate_descriptions(ctx):
118 out = []
119 for _, group in _group_by(ctx["html_pages"], "description").items():
120 if len(group) > 1:
121 for p in group:
122 out.append(
123 _insight(
124 p["url"],
125 "Duplicate meta description",
126 TYPE_SEO,
127 SEVERITY_WARNING,
128 item=p["description"],
129 )
130 )
131 return out
132
133
134def check_h1_missing(ctx):
135 return [
136 _insight(p["url"], "Page has no h1", TYPE_SEO, SEVERITY_ERROR)
137 for p in ctx["html_pages"]
138 if not p.get("headings", {}).get("h1")
139 ]
140
141
142def check_h1_multiple(ctx):
143 out = []
144 for p in ctx["html_pages"]:
145 h1s = p.get("headings", {}).get("h1", [])
146 if len(h1s) > 1:
147 out.append(
148 _insight(
149 p["url"],
150 f"Page has {len(h1s)} h1 tags (expected 1)",
151 TYPE_SEO,
152 SEVERITY_WARNING,
153 item=" | ".join(h1s[:3]),
154 )
155 )
156 return out
157
158
159def check_h1_length(ctx):
160 out = []
161 for p in ctx["html_pages"]:
162 h1s = p.get("headings", {}).get("h1", [])
163 if h1s and not (20 <= len(h1s[0]) <= 70):
164 out.append(
165 _insight(
166 p["url"],
167 f"H1 length is {len(h1s[0])} chars (recommended 20-70)",
168 TYPE_SEO,
169 SEVERITY_WARNING,
170 item=h1s[0],
171 )
172 )
173 return out
174
175
176def check_duplicate_h1s(ctx):
177 out = []
178 buckets = {}
179 for p in ctx["html_pages"]:
180 h1s = p.get("headings", {}).get("h1", [])
181 if h1s:
182 buckets.setdefault(_normalize(h1s[0]), []).append((p["url"], h1s[0]))
183 for _, group in buckets.items():
184 if len(group) > 1:
185 for url, item in group:
186 out.append(_insight(url, "Duplicate h1", TYPE_SEO, SEVERITY_WARNING, item=item))
187 return out
188
189
190def check_heading_hierarchy(ctx):
191 out = []
192 for p in ctx["html_pages"]:
193 h = p.get("headings", {})
194 levels = [lvl for lvl in range(1, 7) if h.get(f"h{lvl}")]
195 for i in range(1, len(levels)):
196 if levels[i] - levels[i - 1] > 1:
197 out.append(
198 _insight(
199 p["url"],
200 f"Heading hierarchy skips from h{levels[i - 1]} to h{levels[i]}",
201 TYPE_SEO,
202 SEVERITY_INFO,
203 )
204 )
205 break
206 return out
207
208
209def check_canonical_missing(ctx):
210 return [
211 _insight(p["url"], "Page has no canonical URL", TYPE_SEO, SEVERITY_WARNING)
212 for p in ctx["html_pages"]
213 if not p.get("canonical")
214 ]
215
216
217def check_canonical_offdomain(ctx):
218 out = []
219 host = ctx["host"]
220 for p in ctx["html_pages"]:
221 c = p.get("canonical", "")
222 if c and not same_site(c, host):
223 out.append(
224 _insight(
225 p["url"],
226 "Canonical URL points off-domain",
227 TYPE_SEO,
228 SEVERITY_WARNING,
229 item=c,
230 )
231 )
232 return out
233
234
235def check_canonical_broken(ctx):
236 out = []
237 status_map = ctx["status_map"]
238 for p in ctx["html_pages"]:
239 c = p.get("canonical", "")
240 if c and c in status_map and status_map[c] != 200:
241 out.append(
242 _insight(
243 p["url"],
244 f"Canonical URL returns {status_map[c]}",
245 TYPE_SEO,
246 SEVERITY_ERROR,
247 item=c,
248 )
249 )
250 return out
251
252
253def check_robots_meta_noindex(ctx):
254 out = []
255 for p in ctx["html_pages"]:
256 rm = (p.get("robots_meta") or "").lower()
257 if "noindex" in rm:
258 out.append(
259 _insight(
260 p["url"],
261 "Page has noindex in meta robots tag",
262 TYPE_SEO,
263 SEVERITY_WARNING,
264 item=p.get("robots_meta", ""),
265 )
266 )
267 return out
268
269
270def check_lang_missing(ctx):
271 return [
272 _insight(p["url"], "HTML lang attribute missing", TYPE_SEO, SEVERITY_WARNING)
273 for p in ctx["html_pages"]
274 if not p.get("lang")
275 ]
276
277
278def check_viewport_missing(ctx):
279 return [
280 _insight(p["url"], "Viewport meta tag missing (mobile)", TYPE_SEO, SEVERITY_WARNING)
281 for p in ctx["html_pages"]
282 if not p.get("viewport")
283 ]
284
285
286def check_og_incomplete(ctx):
287 out = []
288 for p in ctx["html_pages"]:
289 og = p.get("og") or {}
290 missing = [k for k in ("title", "description", "image", "url") if not og.get(k)]
291 if missing:
292 out.append(
293 _insight(
294 p["url"],
295 f"Open Graph tags missing: {', '.join('og:' + m for m in missing)}",
296 TYPE_SEO,
297 SEVERITY_INFO,
298 )
299 )
300 return out
301
302
303def check_twitter_card(ctx):
304 return [
305 _insight(p["url"], "Twitter card meta tag missing", TYPE_SEO, SEVERITY_INFO)
306 for p in ctx["html_pages"]
307 if not (p.get("twitter") or {}).get("card")
308 ]
309
310
311def check_favicon(ctx):
312 return [
313 _insight(p["url"], "Favicon link missing", TYPE_SEO, SEVERITY_INFO)
314 for p in ctx["html_pages"]
315 if not p.get("favicon")
316 ]
317
318
319def check_json_ld_parse_error(ctx):
320 out = []
321 for p in ctx["html_pages"]:
322 for item in p.get("json_ld", []):
323 if item is None:
324 out.append(
325 _insight(
326 p["url"],
327 "JSON-LD structured data failed to parse",
328 TYPE_SEO,
329 SEVERITY_WARNING,
330 )
331 )
332 break
333 return out
334
335
336# ---------- links ----------
337
338def check_broken_internal_links(ctx):
339 out = []
340 reported = set()
341 status_map = ctx["status_map"]
342 host = ctx["host"]
343 for p in ctx["html_pages"]:
344 for link in p.get("links", []):
345 lu = link["url"]
346 if not same_site(lu, host):
347 continue
348 status = status_map.get(lu)
349 if status is None:
350 continue
351 if status != 200 and status not in REDIRECT_CODES:
352 key = (p["url"], lu)
353 if key in reported:
354 continue
355 reported.add(key)
356 label = f"status {status}" if status else "unreachable"
357 out.append(
358 _insight(
359 p["url"],
360 f"Broken internal link ({label})",
361 TYPE_LINKS,
362 SEVERITY_ERROR,
363 item=lu,
364 )
365 )
366 return out
367
368
369def check_broken_external_links(ctx):
370 out = []
371 reported = set()
372 host = ctx["host"]
373 ext = ctx["external_link_status"]
374 for p in ctx["html_pages"]:
375 for link in p.get("links", []):
376 lu = link["url"]
377 if same_site(lu, host):
378 continue
379 if lu not in ext:
380 continue
381 status = ext[lu]
382 if status == 0 or status >= 400:
383 key = (p["url"], lu)
384 if key in reported:
385 continue
386 reported.add(key)
387 label = f"status {status}" if status else "unreachable"
388 out.append(
389 _insight(
390 p["url"],
391 f"Broken external link ({label})",
392 TYPE_LINKS,
393 SEVERITY_WARNING,
394 item=lu,
395 )
396 )
397 return out
398
399
400def check_redirect_chains(ctx):
401 out = []
402 for p in ctx["pages"]:
403 chain = p.get("redirect_chain") or []
404 if len(chain) > 2: # initial + final is fine; more means multiple hops
405 hops = len(chain) - 1
406 out.append(
407 _insight(
408 p["url"],
409 f"Redirect chain has {hops} hops",
410 TYPE_LINKS,
411 SEVERITY_INFO,
412 item=" -> ".join(str(code) for code, _ in chain),
413 )
414 )
415 return out
416
417
418def check_nofollow_internal_links(ctx):
419 out = []
420 reported = set()
421 host = ctx["host"]
422 for p in ctx["html_pages"]:
423 for link in p.get("links", []):
424 lu = link["url"]
425 if not same_site(lu, host):
426 continue
427 if "nofollow" in (link.get("rel") or []):
428 key = (p["url"], lu)
429 if key in reported:
430 continue
431 reported.add(key)
432 out.append(
433 _insight(
434 p["url"],
435 "Internal link has rel=nofollow",
436 TYPE_LINKS,
437 SEVERITY_INFO,
438 item=lu,
439 )
440 )
441 return out
442
443
444# ---------- robots / sitemap ----------
445
446def check_robots_missing(ctx):
447 if not ctx["robots"]["exists"]:
448 return [
449 _insight(
450 ctx["start_url"],
451 "robots.txt missing",
452 TYPE_SEO,
453 SEVERITY_WARNING,
454 item=ctx["robots"]["url"],
455 )
456 ]
457 return []
458
459
460def check_sitemap_missing(ctx):
461 if not ctx["sitemap_urls"]:
462 return [
463 _insight(
464 ctx["start_url"],
465 "sitemap.xml missing or empty",
466 TYPE_SEO,
467 SEVERITY_WARNING,
468 )
469 ]
470 return []
471
472
473def check_sitemap_not_in_robots(ctx):
474 if (
475 ctx["robots"]["exists"]
476 and ctx["sitemap_urls"]
477 and not ctx["robots"].get("references_sitemap")
478 ):
479 return [
480 _insight(
481 ctx["start_url"],
482 "robots.txt does not reference a sitemap",
483 TYPE_SEO,
484 SEVERITY_INFO,
485 )
486 ]
487 return []
488
489
490def check_sitemap_broken_urls(ctx):
491 out = []
492 status_map = ctx["status_map"]
493 for url in ctx["sitemap_urls"]:
494 s = status_map.get(url)
495 if s is not None and s != 200 and s not in REDIRECT_CODES:
496 out.append(
497 _insight(
498 url,
499 f"URL listed in sitemap returns {s}",
500 TYPE_SEO,
501 SEVERITY_ERROR,
502 )
503 )
504 return out
505
506
507def check_pages_missing_from_sitemap(ctx):
508 if not ctx["sitemap_urls"]:
509 return []
510 sitemap_set = set(ctx["sitemap_urls"])
511 out = []
512 for p in ctx["html_pages"]:
513 if p["url"] in sitemap_set:
514 continue
515 # Ignore pages excluded by meta robots
516 if "noindex" in (p.get("robots_meta") or "").lower():
517 continue
518 out.append(
519 _insight(
520 p["url"],
521 "Page not listed in sitemap",
522 TYPE_SEO,
523 SEVERITY_INFO,
524 )
525 )
526 return out
527
528
529# ---------- accessibility ----------
530
531def check_images_missing_alt(ctx):
532 out = []
533 for p in ctx["html_pages"]:
534 missing = [img for img in p.get("images", []) if img.get("alt") is None]
535 if missing:
536 out.append(
537 _insight(
538 p["url"],
539 f"{len(missing)} image(s) missing alt attribute",
540 TYPE_ACCESSIBILITY,
541 SEVERITY_WARNING,
542 item=missing[0].get("src", "")[:160],
543 )
544 )
545 return out
546
547
548def check_empty_anchor_text(ctx):
549 out = []
550 for p in ctx["html_pages"]:
551 empty = [link for link in p.get("links", []) if not link.get("text")]
552 if empty:
553 out.append(
554 _insight(
555 p["url"],
556 f"{len(empty)} link(s) have no visible text",
557 TYPE_ACCESSIBILITY,
558 SEVERITY_INFO,
559 item=empty[0].get("url", "")[:160],
560 )
561 )
562 return out
563
564
565def check_form_inputs_unlabeled(ctx):
566 out = []
567 ignore_types = {"hidden", "submit", "button", "reset", "image"}
568 for p in ctx["html_pages"]:
569 for form in p.get("forms", []):
570 label_fors = set(form.get("label_fors", []))
571 unlabeled = 0
572 for i in form.get("inputs", []):
573 if (i.get("type") or "text").lower() in ignore_types:
574 continue
575 if i.get("aria_label"):
576 continue
577 if i.get("id") and i.get("id") in label_fors:
578 continue
579 unlabeled += 1
580 if unlabeled:
581 out.append(
582 _insight(
583 p["url"],
584 f"{unlabeled} form input(s) without associated label",
585 TYPE_ACCESSIBILITY,
586 SEVERITY_WARNING,
587 item=form.get("action", ""),
588 )
589 )
590 break # one insight per page
591 return out
592
593
594# ---------- content ----------
595
596def check_thin_content(ctx):
597 out = []
598 for p in ctx["html_pages"]:
599 wc = p.get("word_count", 0)
600 if wc < 300:
601 out.append(
602 _insight(
603 p["url"],
604 f"Thin content ({wc} words)",
605 TYPE_CONTENT,
606 SEVERITY_WARNING,
607 )
608 )
609 return out
610
611
612def check_duplicate_content(ctx):
613 out = []
614 buckets = {}
615 for p in ctx["html_pages"]:
616 th = p.get("text_hash")
617 if th:
618 buckets.setdefault(th, []).append(p["url"])
619 for urls in buckets.values():
620 if len(urls) > 1:
621 for u in urls:
622 other = next((x for x in urls if x != u), urls[0])
623 out.append(
624 _insight(
625 u,
626 "Page has duplicate visible content with another page",
627 TYPE_CONTENT,
628 SEVERITY_WARNING,
629 item=other,
630 )
631 )
632 return out
633
634
635# ---------- performance ----------
636
637def check_slow_pages(ctx):
638 out = []
639 for p in ctx["pages"]:
640 if not p.get("is_html"):
641 continue
642 ms = p.get("elapsed_ms", 0)
643 if ms > 1000:
644 out.append(
645 _insight(
646 p["url"],
647 f"Slow response ({ms} ms)",
648 TYPE_PERFORMANCE,
649 SEVERITY_WARNING,
650 )
651 )
652 return out
653
654
655def check_missing_compression(ctx):
656 out = []
657 for p in ctx["html_pages"]:
658 headers = p.get("headers") or {}
659 enc = ""
660 for k, v in headers.items():
661 if k.lower() == "content-encoding":
662 enc = (v or "").lower()
663 break
664 if not enc:
665 out.append(
666 _insight(
667 p["url"],
668 "Response not compressed (no Content-Encoding header)",
669 TYPE_PERFORMANCE,
670 SEVERITY_INFO,
671 )
672 )
673 return out
674
675
676def check_oversized_pages(ctx):
677 out = []
678 for p in ctx["pages"]:
679 size = p.get("bytes", 0)
680 if size > 500_000:
681 out.append(
682 _insight(
683 p["url"],
684 f"Oversized page ({size // 1024} KB)",
685 TYPE_PERFORMANCE,
686 SEVERITY_WARNING,
687 )
688 )
689 return out
690
691
692# ---------- security (per-page; SecurityMixin covers site-level) ----------
693
694def check_mixed_content(ctx):
695 out = []
696 for p in ctx["html_pages"]:
697 if not p["url"].startswith("https://"):
698 continue
699 http_resources = [r for r in p.get("resources", []) if r.startswith("http://")]
700 if http_resources:
701 out.append(
702 _insight(
703 p["url"],
704 f"Mixed content: {len(http_resources)} http:// resource(s) on https:// page",
705 TYPE_SECURITY,
706 SEVERITY_WARNING,
707 item=http_resources[0],
708 )
709 )
710 return out
711
712
713ALL_CHECKS = [
714 # Core metadata
715 check_title_missing,
716 check_title_length,
717 check_duplicate_titles,
718 check_description_missing,
719 check_description_length,
720 check_duplicate_descriptions,
721 check_h1_missing,
722 check_h1_multiple,
723 check_h1_length,
724 check_duplicate_h1s,
725 check_heading_hierarchy,
726 check_canonical_missing,
727 check_canonical_offdomain,
728 check_canonical_broken,
729 check_robots_meta_noindex,
730 check_lang_missing,
731 check_viewport_missing,
732 check_og_incomplete,
733 check_twitter_card,
734 check_favicon,
735 check_json_ld_parse_error,
736 # Links
737 check_broken_internal_links,
738 check_broken_external_links,
739 check_redirect_chains,
740 check_nofollow_internal_links,
741 # Robots / sitemap
742 check_robots_missing,
743 check_sitemap_missing,
744 check_sitemap_not_in_robots,
745 check_sitemap_broken_urls,
746 check_pages_missing_from_sitemap,
747 # Accessibility
748 check_images_missing_alt,
749 check_empty_anchor_text,
750 check_form_inputs_unlabeled,
751 # Content
752 check_thin_content,
753 check_duplicate_content,
754 # Performance
755 check_slow_pages,
756 check_missing_compression,
757 check_oversized_pages,
758 # Security
759 check_mixed_content,
760]