Single-binary self-hosted uptime monitoring and status pages on Rust axum: HTTP probes, Lighthouse audits, SEO crawler, and PDF reports.
axumdockerrustself-hostedsqlitestatus-pageuptime-monitoringvite
1use anyhow::Result;
2use scraper::{Html, Selector};
3use serde::Serialize;
4use sha2::{Digest, Sha256};
5use std::collections::{BTreeMap, HashSet};
6use url::Url;
7
8#[derive(Debug, Clone, Serialize)]
9pub struct ParsedHtml {
10 pub title: String,
11 pub description: String,
12 pub canonical: String,
13 pub robots_meta: String,
14 pub viewport: String,
15 pub lang: String,
16 pub og: Og,
17 pub twitter: Twitter,
18 pub headings: BTreeMap<String, Vec<String>>,
19 pub links: Vec<Link>,
20 pub images: Vec<Image>,
21 pub resources: Vec<String>,
22 pub json_ld: Vec<serde_json::Value>,
23 pub favicon: String,
24 pub forms: Vec<Form>,
25 pub word_count: usize,
26 pub text_hash: String,
27}
28
29#[derive(Debug, Clone, Default, Serialize)]
30pub struct Og {
31 pub title: String,
32 pub description: String,
33 pub image: String,
34 pub url: String,
35}
36
37#[derive(Debug, Clone, Default, Serialize)]
38pub struct Twitter {
39 pub card: String,
40 pub title: String,
41 pub description: String,
42}
43
44#[derive(Debug, Clone, Serialize)]
45pub struct Link {
46 pub url: String,
47 pub text: String,
48 pub rel: Vec<String>,
49}
50
51#[derive(Debug, Clone, Serialize)]
52pub struct Image {
53 pub src: String,
54 /// `None` = attribute absent (a11y violation); `Some("")` = explicitly empty (decorative).
55 pub alt: Option<String>,
56}
57
58#[derive(Debug, Clone, Serialize)]
59pub struct Form {
60 pub action: String,
61 pub inputs: Vec<FormInput>,
62 pub label_fors: Vec<String>,
63}
64
65#[derive(Debug, Clone, Serialize)]
66pub struct FormInput {
67 pub r#type: String,
68 pub name: Option<String>,
69 pub id: Option<String>,
70 pub aria_label: Option<String>,
71}
72
73fn meta_by(doc: &Html, attr: &str, val: &str) -> String {
74 // CSS selectors don't support arbitrary attribute matching with spaces, so
75 // build the selector string by hand and let scraper parse it.
76 let sel = Selector::parse(&format!(r#"meta[{attr}="{val}"]"#)).ok();
77 if let Some(s) = sel {
78 if let Some(el) = doc.select(&s).next() {
79 return el.value().attr("content").unwrap_or("").trim().to_string();
80 }
81 }
82 String::new()
83}
84
85fn join(base: &Url, rel: &str) -> String {
86 base.join(rel)
87 .map(|u| u.to_string())
88 .unwrap_or_else(|_| rel.to_string())
89}
90
91pub fn parse_html(body: &[u8], page_url: &str) -> Result<ParsedHtml> {
92 let body_str = String::from_utf8_lossy(body);
93 let doc = Html::parse_document(&body_str);
94 let base = Url::parse(page_url)?;
95
96 let title = {
97 let s = Selector::parse("title").unwrap();
98 doc.select(&s)
99 .next()
100 .map(|t| t.text().collect::<String>().trim().to_string())
101 .unwrap_or_default()
102 };
103
104 let description = meta_by(&doc, "name", "description");
105 let robots_meta = meta_by(&doc, "name", "robots");
106 let viewport = meta_by(&doc, "name", "viewport");
107
108 let canonical = {
109 let s = Selector::parse(r#"link[rel="canonical"]"#).unwrap();
110 doc.select(&s)
111 .next()
112 .and_then(|el| el.value().attr("href"))
113 .map(|h| join(&base, h.trim()))
114 .unwrap_or_default()
115 };
116
117 let og = Og {
118 title: meta_by(&doc, "property", "og:title"),
119 description: meta_by(&doc, "property", "og:description"),
120 image: meta_by(&doc, "property", "og:image"),
121 url: meta_by(&doc, "property", "og:url"),
122 };
123
124 let twitter = Twitter {
125 card: meta_by(&doc, "name", "twitter:card"),
126 title: meta_by(&doc, "name", "twitter:title"),
127 description: meta_by(&doc, "name", "twitter:description"),
128 };
129
130 let lang = {
131 let s = Selector::parse("html").unwrap();
132 doc.select(&s)
133 .next()
134 .and_then(|el| el.value().attr("lang"))
135 .unwrap_or("")
136 .trim()
137 .to_string()
138 };
139
140 let mut headings: BTreeMap<String, Vec<String>> = BTreeMap::new();
141 for level in 1..=6u8 {
142 let key = format!("h{level}");
143 let s = Selector::parse(&key).unwrap();
144 let v: Vec<String> = doc
145 .select(&s)
146 .map(|el| el.text().collect::<Vec<_>>().join(" ").split_whitespace().collect::<Vec<_>>().join(" "))
147 .collect();
148 headings.insert(key, v);
149 }
150
151 let mut links: Vec<Link> = Vec::new();
152 let s = Selector::parse("a[href]").unwrap();
153 for a in doc.select(&s) {
154 let href = a.value().attr("href").unwrap_or("").trim();
155 if href.is_empty()
156 || href.starts_with("javascript:")
157 || href.starts_with("mailto:")
158 || href.starts_with("tel:")
159 || href.starts_with('#')
160 {
161 continue;
162 }
163 let text = a
164 .text()
165 .collect::<Vec<_>>()
166 .join(" ")
167 .split_whitespace()
168 .collect::<Vec<_>>()
169 .join(" ");
170 let rel: Vec<String> = a
171 .value()
172 .attr("rel")
173 .map(|s| s.split_whitespace().map(|t| t.to_string()).collect())
174 .unwrap_or_default();
175 links.push(Link {
176 url: join(&base, href),
177 text,
178 rel,
179 });
180 }
181
182 let mut images: Vec<Image> = Vec::new();
183 let s = Selector::parse("img").unwrap();
184 for img in doc.select(&s) {
185 let src = img.value().attr("src").unwrap_or("").trim();
186 let alt = img.value().attr("alt").map(|s| s.to_string());
187 images.push(Image {
188 src: if src.is_empty() { String::new() } else { join(&base, src) },
189 alt,
190 });
191 }
192
193 let mut resources: Vec<String> = Vec::new();
194 let s = Selector::parse("script, link, img, iframe, source").unwrap();
195 for el in doc.select(&s) {
196 let src = el
197 .value()
198 .attr("src")
199 .or_else(|| el.value().attr("href"))
200 .unwrap_or("")
201 .trim();
202 if !src.is_empty() {
203 resources.push(join(&base, src));
204 }
205 }
206
207 let mut json_ld: Vec<serde_json::Value> = Vec::new();
208 let s = Selector::parse(r#"script[type="application/ld+json"]"#).unwrap();
209 for sc in doc.select(&s) {
210 let raw = sc.text().collect::<String>();
211 if raw.trim().is_empty() {
212 continue;
213 }
214 match serde_json::from_str::<serde_json::Value>(&raw) {
215 Ok(v) => json_ld.push(v),
216 Err(_) => json_ld.push(serde_json::Value::Null),
217 }
218 }
219
220 let mut favicon = String::new();
221 let s = Selector::parse("link[rel]").unwrap();
222 for el in doc.select(&s) {
223 let rels = el.value().attr("rel").unwrap_or("");
224 if rels.split_whitespace().any(|r| r.to_lowercase().contains("icon")) {
225 let href = el.value().attr("href").unwrap_or("").trim();
226 if !href.is_empty() {
227 favicon = join(&base, href);
228 break;
229 }
230 }
231 }
232
233 let mut forms: Vec<Form> = Vec::new();
234 let s_form = Selector::parse("form").unwrap();
235 let s_input = Selector::parse("input, textarea, select").unwrap();
236 let s_label = Selector::parse("label[for]").unwrap();
237 for form in doc.select(&s_form) {
238 let mut inputs: Vec<FormInput> = Vec::new();
239 for i in form.select(&s_input) {
240 let v = i.value();
241 inputs.push(FormInput {
242 r#type: v.attr("type").unwrap_or("text").to_string(),
243 name: v.attr("name").map(String::from),
244 id: v.attr("id").map(String::from),
245 aria_label: v.attr("aria-label").map(String::from),
246 });
247 }
248 let mut label_fors: HashSet<String> = HashSet::new();
249 for lb in form.select(&s_label) {
250 if let Some(f) = lb.value().attr("for") {
251 label_fors.insert(f.to_string());
252 }
253 }
254 let action = form
255 .value()
256 .attr("action")
257 .map(|a| {
258 if a.is_empty() {
259 page_url.to_string()
260 } else {
261 join(&base, a)
262 }
263 })
264 .unwrap_or_else(|| page_url.to_string());
265 forms.push(Form {
266 action,
267 inputs,
268 label_fors: label_fors.into_iter().collect(),
269 });
270 }
271
272 // Visible text: drop script/style/noscript before extracting.
273 let strip_re = regex::Regex::new(
274 r"(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>",
275 )
276 .unwrap();
277 let stripped = strip_re.replace_all(&body_str, "");
278 let tag_re = regex::Regex::new(r"(?is)<[^>]+>").unwrap();
279 let text_only = tag_re.replace_all(&stripped, " ");
280 let text = text_only.split_whitespace().collect::<Vec<_>>().join(" ");
281 let word_count = text.split_whitespace().count();
282 let mut h = Sha256::new();
283 h.update(text.as_bytes());
284 let text_hash = format!("{:x}", h.finalize());
285
286 Ok(ParsedHtml {
287 title,
288 description,
289 canonical,
290 robots_meta,
291 viewport,
292 lang,
293 og,
294 twitter,
295 headings,
296 links,
297 images,
298 resources,
299 json_ld,
300 favicon,
301 forms,
302 word_count,
303 text_hash,
304 })
305}