Single-binary self-hosted uptime monitoring and status pages on Rust axum: HTTP probes, Lighthouse audits, SEO crawler, and PDF reports.
axumdockerrustself-hostedsqlitestatus-pageuptime-monitoringvite
1//! SEO / accessibility / performance / content / security checks.
2//!
3//! Each check takes a `Ctx` and returns a list of insight values
4//! (`{url, issue, item, type, severity}` JSON objects). Direct port of
5//! `crawler/checks.py`.
6
7use super::{fetcher::same_site, Page, RobotsCtx};
8use serde_json::{json, Value};
9use std::collections::{HashMap, HashSet};
10
11const TYPE_SEO: &str = "seo";
12const TYPE_LINKS: &str = "links";
13const TYPE_A11Y: &str = "accessibility";
14const TYPE_CONTENT: &str = "content";
15const TYPE_PERF: &str = "performance";
16const TYPE_SEC: &str = "security";
17
18const SEV_ERROR: &str = "error";
19const SEV_WARN: &str = "warning";
20const SEV_INFO: &str = "info";
21
22pub struct Ctx<'a> {
23 pub start_url: &'a str,
24 pub host: &'a str,
25 pub pages: &'a [Page],
26 pub html_pages: &'a [&'a Page],
27 pub status_map: &'a HashMap<String, u16>,
28 pub external_link_status: &'a HashMap<String, u16>,
29 pub sitemap_urls: &'a [String],
30 pub robots: &'a RobotsCtx,
31 /// Server's `Content-Encoding` for `start_url`, lowercased. `None` means
32 /// the response was uncompressed.
33 pub compression: Option<&'a str>,
34}
35
36fn redirect_codes() -> HashSet<u16> {
37 [301, 302, 303, 307, 308].into_iter().collect()
38}
39
40fn insight(url: &str, issue: &str, type_: &str, severity: &str, item: &str) -> Value {
41 json!({
42 "url": url,
43 "issue": issue,
44 "item": item,
45 "type": type_,
46 "severity": severity,
47 })
48}
49
50fn normalize(s: &str) -> String {
51 s.to_lowercase().split_whitespace().collect::<Vec<_>>().join(" ")
52}
53
54fn group_by<'a, F: Fn(&Page) -> String>(
55 pages: &[&'a Page],
56 f: F,
57) -> HashMap<String, Vec<&'a Page>> {
58 let mut out: HashMap<String, Vec<&'a Page>> = HashMap::new();
59 for p in pages {
60 let v = f(p);
61 if !v.is_empty() {
62 out.entry(normalize(&v)).or_default().push(p);
63 }
64 }
65 out
66}
67
68fn html<'a>(p: &'a Page) -> &'a super::parser::ParsedHtml {
69 p.html.as_ref().expect("called on html page")
70}
71
72// ---------- core metadata ----------
73
74fn check_title_missing(ctx: &Ctx) -> Vec<Value> {
75 ctx.html_pages
76 .iter()
77 .filter(|p| html(p).title.is_empty())
78 .map(|p| insight(&p.url, "Page has no title", TYPE_SEO, SEV_ERROR, ""))
79 .collect()
80}
81
82fn check_title_length(ctx: &Ctx) -> Vec<Value> {
83 let mut out = Vec::new();
84 for p in ctx.html_pages {
85 let t = &html(p).title;
86 let n = t.chars().count();
87 if !t.is_empty() && !(30..=60).contains(&n) {
88 out.push(insight(
89 &p.url,
90 &format!("Title length is {n} chars (recommended 30-60)"),
91 TYPE_SEO,
92 SEV_WARN,
93 t,
94 ));
95 }
96 }
97 out
98}
99
100fn check_duplicate_titles(ctx: &Ctx) -> Vec<Value> {
101 let mut out = Vec::new();
102 for (_, group) in group_by(ctx.html_pages, |p| html(p).title.clone()) {
103 if group.len() > 1 {
104 for p in group {
105 out.push(insight(&p.url, "Duplicate title", TYPE_SEO, SEV_WARN, &html(p).title));
106 }
107 }
108 }
109 out
110}
111
112fn check_description_missing(ctx: &Ctx) -> Vec<Value> {
113 ctx.html_pages
114 .iter()
115 .filter(|p| html(p).description.is_empty())
116 .map(|p| insight(&p.url, "Page has no meta description", TYPE_SEO, SEV_ERROR, ""))
117 .collect()
118}
119
120fn check_description_length(ctx: &Ctx) -> Vec<Value> {
121 let mut out = Vec::new();
122 for p in ctx.html_pages {
123 let d = &html(p).description;
124 let n = d.chars().count();
125 if !d.is_empty() && !(70..=160).contains(&n) {
126 out.push(insight(
127 &p.url,
128 &format!("Description length is {n} chars (recommended 70-160)"),
129 TYPE_SEO,
130 SEV_WARN,
131 d,
132 ));
133 }
134 }
135 out
136}
137
138fn check_duplicate_descriptions(ctx: &Ctx) -> Vec<Value> {
139 let mut out = Vec::new();
140 for (_, group) in group_by(ctx.html_pages, |p| html(p).description.clone()) {
141 if group.len() > 1 {
142 for p in group {
143 out.push(insight(
144 &p.url,
145 "Duplicate meta description",
146 TYPE_SEO,
147 SEV_WARN,
148 &html(p).description,
149 ));
150 }
151 }
152 }
153 out
154}
155
156fn h1s<'a>(p: &'a Page) -> &'a [String] {
157 html(p)
158 .headings
159 .get("h1")
160 .map(|v| v.as_slice())
161 .unwrap_or(&[])
162}
163
164fn check_h1_missing(ctx: &Ctx) -> Vec<Value> {
165 ctx.html_pages
166 .iter()
167 .filter(|p| h1s(p).is_empty())
168 .map(|p| insight(&p.url, "Page has no h1", TYPE_SEO, SEV_ERROR, ""))
169 .collect()
170}
171
172fn check_h1_multiple(ctx: &Ctx) -> Vec<Value> {
173 let mut out = Vec::new();
174 for p in ctx.html_pages {
175 let v = h1s(p);
176 if v.len() > 1 {
177 let item = v.iter().take(3).cloned().collect::<Vec<_>>().join(" | ");
178 out.push(insight(
179 &p.url,
180 &format!("Page has {} h1 tags (expected 1)", v.len()),
181 TYPE_SEO,
182 SEV_WARN,
183 &item,
184 ));
185 }
186 }
187 out
188}
189
190fn check_h1_length(ctx: &Ctx) -> Vec<Value> {
191 let mut out = Vec::new();
192 for p in ctx.html_pages {
193 let v = h1s(p);
194 if let Some(first) = v.first() {
195 let n = first.chars().count();
196 if !(20..=70).contains(&n) {
197 out.push(insight(
198 &p.url,
199 &format!("H1 length is {n} chars (recommended 20-70)"),
200 TYPE_SEO,
201 SEV_WARN,
202 first,
203 ));
204 }
205 }
206 }
207 out
208}
209
210fn check_duplicate_h1s(ctx: &Ctx) -> Vec<Value> {
211 let mut buckets: HashMap<String, Vec<(String, String)>> = HashMap::new();
212 for p in ctx.html_pages {
213 if let Some(first) = h1s(p).first() {
214 buckets.entry(normalize(first)).or_default().push((p.url.clone(), first.clone()));
215 }
216 }
217 let mut out = Vec::new();
218 for (_, group) in buckets {
219 if group.len() > 1 {
220 for (url, item) in group {
221 out.push(insight(&url, "Duplicate h1", TYPE_SEO, SEV_WARN, &item));
222 }
223 }
224 }
225 out
226}
227
228fn check_heading_hierarchy(ctx: &Ctx) -> Vec<Value> {
229 let mut out = Vec::new();
230 for p in ctx.html_pages {
231 let h = &html(p).headings;
232 let levels: Vec<u8> = (1..=6u8)
233 .filter(|lvl| {
234 h.get(&format!("h{lvl}"))
235 .map(|v| !v.is_empty())
236 .unwrap_or(false)
237 })
238 .collect();
239 for i in 1..levels.len() {
240 if levels[i] - levels[i - 1] > 1 {
241 out.push(insight(
242 &p.url,
243 &format!("Heading hierarchy skips from h{} to h{}", levels[i - 1], levels[i]),
244 TYPE_SEO,
245 SEV_INFO,
246 "",
247 ));
248 break;
249 }
250 }
251 }
252 out
253}
254
255fn check_canonical_missing(ctx: &Ctx) -> Vec<Value> {
256 ctx.html_pages
257 .iter()
258 .filter(|p| html(p).canonical.is_empty())
259 .map(|p| insight(&p.url, "Page has no canonical URL", TYPE_SEO, SEV_WARN, ""))
260 .collect()
261}
262
263fn check_canonical_offdomain(ctx: &Ctx) -> Vec<Value> {
264 let mut out = Vec::new();
265 for p in ctx.html_pages {
266 let c = &html(p).canonical;
267 if !c.is_empty() && !same_site(c, ctx.host) {
268 out.push(insight(&p.url, "Canonical URL points off-domain", TYPE_SEO, SEV_WARN, c));
269 }
270 }
271 out
272}
273
274fn check_canonical_broken(ctx: &Ctx) -> Vec<Value> {
275 let mut out = Vec::new();
276 for p in ctx.html_pages {
277 let c = &html(p).canonical;
278 if !c.is_empty() {
279 if let Some(s) = ctx.status_map.get(c) {
280 if *s != 200 {
281 out.push(insight(
282 &p.url,
283 &format!("Canonical URL returns {s}"),
284 TYPE_SEO,
285 SEV_ERROR,
286 c,
287 ));
288 }
289 }
290 }
291 }
292 out
293}
294
295fn check_robots_meta_noindex(ctx: &Ctx) -> Vec<Value> {
296 let mut out = Vec::new();
297 for p in ctx.html_pages {
298 let rm = &html(p).robots_meta;
299 if rm.to_lowercase().contains("noindex") {
300 out.push(insight(
301 &p.url,
302 "Page has noindex in meta robots tag",
303 TYPE_SEO,
304 SEV_WARN,
305 rm,
306 ));
307 }
308 }
309 out
310}
311
312fn check_lang_missing(ctx: &Ctx) -> Vec<Value> {
313 ctx.html_pages
314 .iter()
315 .filter(|p| html(p).lang.is_empty())
316 .map(|p| insight(&p.url, "HTML lang attribute missing", TYPE_SEO, SEV_WARN, ""))
317 .collect()
318}
319
320fn check_viewport_missing(ctx: &Ctx) -> Vec<Value> {
321 ctx.html_pages
322 .iter()
323 .filter(|p| html(p).viewport.is_empty())
324 .map(|p| insight(&p.url, "Viewport meta tag missing (mobile)", TYPE_SEO, SEV_WARN, ""))
325 .collect()
326}
327
328fn check_og_incomplete(ctx: &Ctx) -> Vec<Value> {
329 let mut out = Vec::new();
330 for p in ctx.html_pages {
331 let og = &html(p).og;
332 let mut missing: Vec<&str> = Vec::new();
333 if og.title.is_empty() { missing.push("og:title"); }
334 if og.description.is_empty() { missing.push("og:description"); }
335 if og.image.is_empty() { missing.push("og:image"); }
336 if og.url.is_empty() { missing.push("og:url"); }
337 if !missing.is_empty() {
338 out.push(insight(
339 &p.url,
340 &format!("Open Graph tags missing: {}", missing.join(", ")),
341 TYPE_SEO,
342 SEV_INFO,
343 "",
344 ));
345 }
346 }
347 out
348}
349
350fn check_twitter_card(ctx: &Ctx) -> Vec<Value> {
351 ctx.html_pages
352 .iter()
353 .filter(|p| html(p).twitter.card.is_empty())
354 .map(|p| insight(&p.url, "Twitter card meta tag missing", TYPE_SEO, SEV_INFO, ""))
355 .collect()
356}
357
358fn check_favicon(ctx: &Ctx) -> Vec<Value> {
359 ctx.html_pages
360 .iter()
361 .filter(|p| html(p).favicon.is_empty())
362 .map(|p| insight(&p.url, "Favicon link missing", TYPE_SEO, SEV_INFO, ""))
363 .collect()
364}
365
366fn check_json_ld_parse_error(ctx: &Ctx) -> Vec<Value> {
367 let mut out = Vec::new();
368 for p in ctx.html_pages {
369 for v in &html(p).json_ld {
370 if v.is_null() {
371 out.push(insight(
372 &p.url,
373 "JSON-LD structured data failed to parse",
374 TYPE_SEO,
375 SEV_WARN,
376 "",
377 ));
378 break;
379 }
380 }
381 }
382 out
383}
384
385// ---------- links ----------
386
387fn check_broken_internal_links(ctx: &Ctx) -> Vec<Value> {
388 let mut out = Vec::new();
389 let mut reported: HashSet<(String, String)> = HashSet::new();
390 let redirects = redirect_codes();
391 for p in ctx.html_pages {
392 for link in &html(p).links {
393 if !same_site(&link.url, ctx.host) {
394 continue;
395 }
396 let Some(s) = ctx.status_map.get(&link.url) else { continue };
397 if *s != 200 && !redirects.contains(s) {
398 let key = (p.url.clone(), link.url.clone());
399 if !reported.insert(key) {
400 continue;
401 }
402 let label = if *s == 0 { "unreachable".to_string() } else { format!("status {s}") };
403 out.push(insight(
404 &p.url,
405 &format!("Broken internal link ({label})"),
406 TYPE_LINKS,
407 SEV_ERROR,
408 &link.url,
409 ));
410 }
411 }
412 }
413 out
414}
415
416fn check_broken_external_links(ctx: &Ctx) -> Vec<Value> {
417 let mut out = Vec::new();
418 let mut reported: HashSet<(String, String)> = HashSet::new();
419 for p in ctx.html_pages {
420 for link in &html(p).links {
421 if same_site(&link.url, ctx.host) {
422 continue;
423 }
424 let Some(s) = ctx.external_link_status.get(&link.url) else { continue };
425 if *s == 0 || *s >= 400 {
426 let key = (p.url.clone(), link.url.clone());
427 if !reported.insert(key) {
428 continue;
429 }
430 let label = if *s == 0 { "unreachable".to_string() } else { format!("status {s}") };
431 out.push(insight(
432 &p.url,
433 &format!("Broken external link ({label})"),
434 TYPE_LINKS,
435 SEV_WARN,
436 &link.url,
437 ));
438 }
439 }
440 }
441 out
442}
443
444fn check_redirect_chains(ctx: &Ctx) -> Vec<Value> {
445 let mut out = Vec::new();
446 for p in ctx.pages {
447 if p.redirect_chain.len() > 2 {
448 let hops = p.redirect_chain.len() - 1;
449 let codes = p
450 .redirect_chain
451 .iter()
452 .map(|(c, _)| c.to_string())
453 .collect::<Vec<_>>()
454 .join(" -> ");
455 out.push(insight(
456 &p.url,
457 &format!("Redirect chain has {hops} hops"),
458 TYPE_LINKS,
459 SEV_INFO,
460 &codes,
461 ));
462 }
463 }
464 out
465}
466
467fn check_nofollow_internal_links(ctx: &Ctx) -> Vec<Value> {
468 let mut out = Vec::new();
469 let mut reported: HashSet<(String, String)> = HashSet::new();
470 for p in ctx.html_pages {
471 for link in &html(p).links {
472 if !same_site(&link.url, ctx.host) {
473 continue;
474 }
475 if link.rel.iter().any(|r| r == "nofollow") {
476 let key = (p.url.clone(), link.url.clone());
477 if !reported.insert(key) {
478 continue;
479 }
480 out.push(insight(
481 &p.url,
482 "Internal link has rel=nofollow",
483 TYPE_LINKS,
484 SEV_INFO,
485 &link.url,
486 ));
487 }
488 }
489 }
490 out
491}
492
493// ---------- robots / sitemap ----------
494
495fn check_robots_missing(ctx: &Ctx) -> Vec<Value> {
496 if !ctx.robots.exists {
497 vec![insight(
498 ctx.start_url,
499 "robots.txt missing",
500 TYPE_SEO,
501 SEV_WARN,
502 &ctx.robots.url,
503 )]
504 } else {
505 Vec::new()
506 }
507}
508
509fn check_sitemap_missing(ctx: &Ctx) -> Vec<Value> {
510 if ctx.sitemap_urls.is_empty() {
511 vec![insight(
512 ctx.start_url,
513 "sitemap.xml missing or empty",
514 TYPE_SEO,
515 SEV_WARN,
516 "",
517 )]
518 } else {
519 Vec::new()
520 }
521}
522
523fn check_sitemap_not_in_robots(ctx: &Ctx) -> Vec<Value> {
524 if ctx.robots.exists && !ctx.sitemap_urls.is_empty() && !ctx.robots.references_sitemap {
525 vec![insight(
526 ctx.start_url,
527 "robots.txt does not reference a sitemap",
528 TYPE_SEO,
529 SEV_INFO,
530 "",
531 )]
532 } else {
533 Vec::new()
534 }
535}
536
537fn check_sitemap_broken_urls(ctx: &Ctx) -> Vec<Value> {
538 let redirects = redirect_codes();
539 let mut out = Vec::new();
540 for url in ctx.sitemap_urls {
541 if let Some(s) = ctx.status_map.get(url) {
542 if *s != 200 && !redirects.contains(s) {
543 out.push(insight(
544 url,
545 &format!("URL listed in sitemap returns {s}"),
546 TYPE_SEO,
547 SEV_ERROR,
548 "",
549 ));
550 }
551 }
552 }
553 out
554}
555
556fn check_pages_missing_from_sitemap(ctx: &Ctx) -> Vec<Value> {
557 if ctx.sitemap_urls.is_empty() {
558 return Vec::new();
559 }
560 let set: HashSet<&str> = ctx.sitemap_urls.iter().map(|s| s.as_str()).collect();
561 let mut out = Vec::new();
562 for p in ctx.html_pages {
563 if set.contains(p.url.as_str()) {
564 continue;
565 }
566 if html(p).robots_meta.to_lowercase().contains("noindex") {
567 continue;
568 }
569 out.push(insight(&p.url, "Page not listed in sitemap", TYPE_SEO, SEV_INFO, ""));
570 }
571 out
572}
573
574// ---------- accessibility ----------
575
576fn check_images_missing_alt(ctx: &Ctx) -> Vec<Value> {
577 let mut out = Vec::new();
578 for p in ctx.html_pages {
579 let missing: Vec<_> = html(p).images.iter().filter(|i| i.alt.is_none()).collect();
580 if !missing.is_empty() {
581 let item: String = missing[0]
582 .src
583 .chars()
584 .take(160)
585 .collect();
586 out.push(insight(
587 &p.url,
588 &format!("{} image(s) missing alt attribute", missing.len()),
589 TYPE_A11Y,
590 SEV_WARN,
591 &item,
592 ));
593 }
594 }
595 out
596}
597
598fn check_empty_anchor_text(ctx: &Ctx) -> Vec<Value> {
599 let mut out = Vec::new();
600 for p in ctx.html_pages {
601 let empty: Vec<_> = html(p).links.iter().filter(|l| l.text.is_empty()).collect();
602 if !empty.is_empty() {
603 let item: String = empty[0].url.chars().take(160).collect();
604 out.push(insight(
605 &p.url,
606 &format!("{} link(s) have no visible text", empty.len()),
607 TYPE_A11Y,
608 SEV_INFO,
609 &item,
610 ));
611 }
612 }
613 out
614}
615
616fn check_form_inputs_unlabeled(ctx: &Ctx) -> Vec<Value> {
617 let ignore: HashSet<&str> = ["hidden", "submit", "button", "reset", "image"].into_iter().collect();
618 let mut out = Vec::new();
619 'outer: for p in ctx.html_pages {
620 for form in &html(p).forms {
621 let label_set: HashSet<&str> = form.label_fors.iter().map(|s| s.as_str()).collect();
622 let mut unlabeled = 0;
623 for i in &form.inputs {
624 if ignore.contains(i.r#type.to_lowercase().as_str()) {
625 continue;
626 }
627 if i.aria_label.is_some() {
628 continue;
629 }
630 if let Some(id) = &i.id {
631 if label_set.contains(id.as_str()) {
632 continue;
633 }
634 }
635 unlabeled += 1;
636 }
637 if unlabeled > 0 {
638 out.push(insight(
639 &p.url,
640 &format!("{unlabeled} form input(s) without associated label"),
641 TYPE_A11Y,
642 SEV_WARN,
643 &form.action,
644 ));
645 continue 'outer;
646 }
647 }
648 }
649 out
650}
651
652// ---------- content ----------
653
654fn check_thin_content(ctx: &Ctx) -> Vec<Value> {
655 let mut out = Vec::new();
656 for p in ctx.html_pages {
657 let wc = html(p).word_count;
658 if wc < 300 {
659 out.push(insight(
660 &p.url,
661 &format!("Thin content ({wc} words)"),
662 TYPE_CONTENT,
663 SEV_WARN,
664 "",
665 ));
666 }
667 }
668 out
669}
670
671fn check_duplicate_content(ctx: &Ctx) -> Vec<Value> {
672 let mut buckets: HashMap<String, Vec<String>> = HashMap::new();
673 for p in ctx.html_pages {
674 let th = &html(p).text_hash;
675 if !th.is_empty() {
676 buckets.entry(th.clone()).or_default().push(p.url.clone());
677 }
678 }
679 let mut out = Vec::new();
680 for urls in buckets.values() {
681 if urls.len() > 1 {
682 for u in urls {
683 let other = urls.iter().find(|x| *x != u).unwrap_or(&urls[0]);
684 out.push(insight(
685 u,
686 "Page has duplicate visible content with another page",
687 TYPE_CONTENT,
688 SEV_WARN,
689 other,
690 ));
691 }
692 }
693 }
694 out
695}
696
697// ---------- performance ----------
698
699fn check_slow_pages(ctx: &Ctx) -> Vec<Value> {
700 let mut out = Vec::new();
701 for p in ctx.pages {
702 if !p.is_html {
703 continue;
704 }
705 if p.elapsed_ms > 1000 {
706 out.push(insight(
707 &p.url,
708 &format!("Slow response ({} ms)", p.elapsed_ms),
709 TYPE_PERF,
710 SEV_WARN,
711 "",
712 ));
713 }
714 }
715 out
716}
717
718fn check_missing_compression(ctx: &Ctx) -> Vec<Value> {
719 // Probed at the start URL with a non-decompressing client (the main
720 // crawler client has reqwest's gzip/brotli features on, which strip
721 // `Content-Encoding` after auto-decompression — so we can't read it from
722 // the per-page headers). Compression is a server-wide config in practice,
723 // so one probe is enough.
724 if ctx.compression.is_none() {
725 vec![insight(
726 ctx.start_url,
727 "Response not compressed (no Content-Encoding header)",
728 TYPE_PERF,
729 SEV_INFO,
730 "",
731 )]
732 } else {
733 Vec::new()
734 }
735}
736
737fn check_oversized_pages(ctx: &Ctx) -> Vec<Value> {
738 let mut out = Vec::new();
739 for p in ctx.pages {
740 if p.bytes > 500_000 {
741 out.push(insight(
742 &p.url,
743 &format!("Oversized page ({} KB)", p.bytes / 1024),
744 TYPE_PERF,
745 SEV_WARN,
746 "",
747 ));
748 }
749 }
750 out
751}
752
753// ---------- security ----------
754
755fn check_mixed_content(ctx: &Ctx) -> Vec<Value> {
756 let mut out = Vec::new();
757 for p in ctx.html_pages {
758 if !p.url.starts_with("https://") {
759 continue;
760 }
761 let http_resources: Vec<&String> = html(p)
762 .resources
763 .iter()
764 .filter(|r| r.starts_with("http://"))
765 .collect();
766 if !http_resources.is_empty() {
767 out.push(insight(
768 &p.url,
769 &format!(
770 "Mixed content: {} http:// resource(s) on https:// page",
771 http_resources.len()
772 ),
773 TYPE_SEC,
774 SEV_WARN,
775 http_resources[0],
776 ));
777 }
778 }
779 out
780}
781
782pub fn run_all(ctx: &Ctx) -> Vec<Value> {
783 let checks: &[(&'static str, fn(&Ctx) -> Vec<Value>)] = &[
784 ("title_missing", check_title_missing),
785 ("title_length", check_title_length),
786 ("duplicate_titles", check_duplicate_titles),
787 ("description_missing", check_description_missing),
788 ("description_length", check_description_length),
789 ("duplicate_descriptions", check_duplicate_descriptions),
790 ("h1_missing", check_h1_missing),
791 ("h1_multiple", check_h1_multiple),
792 ("h1_length", check_h1_length),
793 ("duplicate_h1s", check_duplicate_h1s),
794 ("heading_hierarchy", check_heading_hierarchy),
795 ("canonical_missing", check_canonical_missing),
796 ("canonical_offdomain", check_canonical_offdomain),
797 ("canonical_broken", check_canonical_broken),
798 ("robots_meta_noindex", check_robots_meta_noindex),
799 ("lang_missing", check_lang_missing),
800 ("viewport_missing", check_viewport_missing),
801 ("og_incomplete", check_og_incomplete),
802 ("twitter_card", check_twitter_card),
803 ("favicon", check_favicon),
804 ("json_ld_parse_error", check_json_ld_parse_error),
805 ("broken_internal_links", check_broken_internal_links),
806 ("broken_external_links", check_broken_external_links),
807 ("redirect_chains", check_redirect_chains),
808 ("nofollow_internal_links", check_nofollow_internal_links),
809 ("robots_missing", check_robots_missing),
810 ("sitemap_missing", check_sitemap_missing),
811 ("sitemap_not_in_robots", check_sitemap_not_in_robots),
812 ("sitemap_broken_urls", check_sitemap_broken_urls),
813 ("pages_missing_from_sitemap", check_pages_missing_from_sitemap),
814 ("images_missing_alt", check_images_missing_alt),
815 ("empty_anchor_text", check_empty_anchor_text),
816 ("form_inputs_unlabeled", check_form_inputs_unlabeled),
817 ("thin_content", check_thin_content),
818 ("duplicate_content", check_duplicate_content),
819 ("slow_pages", check_slow_pages),
820 ("missing_compression", check_missing_compression),
821 ("oversized_pages", check_oversized_pages),
822 ("mixed_content", check_mixed_content),
823 ];
824 let mut out = Vec::new();
825 for (name, fn_) in checks {
826 let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| fn_(ctx)));
827 match r {
828 Ok(v) => out.extend(v),
829 Err(_) => tracing::warn!("[crawler] check {name} panicked"),
830 }
831 }
832 out
833}