status-rust

  1use reqwest::Client;
  2use std::collections::HashMap;
  3use std::time::{Duration, Instant};
  4use url::Url;
  5
  6pub const PAGE_CAP: usize = 500;
  7pub const CONCURRENCY: usize = 4;
  8pub const EXTERNAL_LINK_CAP: usize = 500;
  9/// Per-page HTML body cap. Real HTML pages are well under this; anything
 10/// larger is a mislabeled download and buffering it whole (times 4-way
 11/// concurrency) could exhaust container memory.
 12const MAX_BODY_BYTES: usize = 5 * 1024 * 1024;
 13const REQUEST_TIMEOUT_SECS: u64 = 15;
 14const EXTERNAL_LINK_TIMEOUT_SECS: u64 = 8;
 15pub const CRAWL_DEADLINE_SECS: u64 = 540;
 16const USER_AGENT: &str = "status (+https://status.bythewood.me)";
 17
 18#[derive(Debug, Clone)]
 19pub struct FetchResult {
 20    pub url: String,
 21    pub requested_url: String,
 22    pub status: u16,
 23    pub headers: HashMap<String, String>,
 24    pub body: Vec<u8>,
 25    pub content_type: String,
 26    pub elapsed_ms: i64,
 27    pub redirect_chain: Vec<(u16, String)>,
 28    pub error: String,
 29}
 30
 31pub fn make_client() -> Client {
 32    Client::builder()
 33        .user_agent(USER_AGENT)
 34        .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
 35        .redirect(reqwest::redirect::Policy::limited(10))
 36        .build()
 37        .expect("client builds")
 38}
 39
 40// Companion client used only to probe the server's `Content-Encoding` header.
 41// The main client has reqwest's `gzip` and `brotli` features on, which
 42// auto-decompress responses *and* strip `Content-Encoding` from the headers,
 43// so we can't tell from a normal fetch whether the server compressed the
 44// response. This client disables auto-decompression so the header survives.
 45pub fn make_probe_client() -> Client {
 46    Client::builder()
 47        .user_agent(USER_AGENT)
 48        .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS))
 49        .redirect(reqwest::redirect::Policy::limited(10))
 50        .gzip(false)
 51        .brotli(false)
 52        .build()
 53        .expect("probe client builds")
 54}
 55
 56/// Probe `url` with a non-decompressing client and return the server's
 57/// `Content-Encoding` (lowercased). Returns `None` if the server didn't
 58/// compress, the encoding was `identity`, or the request failed.
 59pub async fn probe_compression(client: &Client, url: &str) -> Option<String> {
 60    let resp = client
 61        .get(url)
 62        .header("Accept-Encoding", "gzip, br, zstd, deflate")
 63        .send()
 64        .await
 65        .ok()?;
 66    let enc = resp
 67        .headers()
 68        .get(reqwest::header::CONTENT_ENCODING)
 69        .and_then(|v| v.to_str().ok())?
 70        .trim()
 71        .to_lowercase();
 72    if enc.is_empty() || enc == "identity" {
 73        None
 74    } else {
 75        Some(enc)
 76    }
 77}
 78
 79pub async fn fetch(client: &Client, url: &str) -> FetchResult {
 80    let started = Instant::now();
 81    match client.get(url).send().await {
 82        Ok(mut resp) => {
 83            let final_url = resp.url().to_string();
 84            let status = resp.status().as_u16();
 85            let mut headers = HashMap::new();
 86            for (k, v) in resp.headers().iter() {
 87                if let Ok(s) = v.to_str() {
 88                    headers.insert(k.as_str().to_string(), s.to_string());
 89                }
 90            }
 91            let content_type = headers
 92                .iter()
 93                .find(|(k, _)| k.eq_ignore_ascii_case("content-type"))
 94                .map(|(_, v)| v.to_lowercase())
 95                .unwrap_or_default();
 96            let body = if content_type.contains("text/html") {
 97                let mut buf: Vec<u8> = Vec::new();
 98                loop {
 99                    match resp.chunk().await {
100                        Ok(Some(chunk)) => {
101                            let room = MAX_BODY_BYTES - buf.len();
102                            if chunk.len() >= room {
103                                buf.extend_from_slice(&chunk[..room]);
104                                tracing::warn!(
105                                    "[crawler] body cap {MAX_BODY_BYTES} hit for {url}"
106                                );
107                                break;
108                            }
109                            buf.extend_from_slice(&chunk);
110                        }
111                        Ok(None) => break,
112                        Err(_) => break,
113                    }
114                }
115                buf
116            } else {
117                Vec::new()
118            };
119            let elapsed_ms = started.elapsed().as_millis() as i64;
120            // Reqwest doesn't expose the redirect chain, so we approximate
121            // with [final_status, final_url]. Mid-chain hops are lost.
122            let redirect_chain = vec![(status, final_url.clone())];
123            FetchResult {
124                url: final_url,
125                requested_url: url.to_string(),
126                status,
127                headers,
128                body,
129                content_type,
130                elapsed_ms,
131                redirect_chain,
132                error: String::new(),
133            }
134        }
135        Err(e) => FetchResult {
136            url: url.to_string(),
137            requested_url: url.to_string(),
138            status: 0,
139            headers: HashMap::new(),
140            body: Vec::new(),
141            content_type: String::new(),
142            elapsed_ms: started.elapsed().as_millis() as i64,
143            redirect_chain: Vec::new(),
144            error: e.to_string(),
145        },
146    }
147}
148
149pub async fn head_status(client: &Client, url: &str) -> u16 {
150    let timeout = Duration::from_secs(EXTERNAL_LINK_TIMEOUT_SECS);
151    match client.head(url).timeout(timeout).send().await {
152        Ok(r) => {
153            let s = r.status().as_u16();
154            if matches!(s, 403 | 405 | 501) {
155                client
156                    .get(url)
157                    .timeout(timeout)
158                    .send()
159                    .await
160                    .map(|r| r.status().as_u16())
161                    .unwrap_or(0)
162            } else {
163                s
164            }
165        }
166        Err(_) => 0,
167    }
168}
169
170/// Robots.txt evaluator. Uses the `robotstxt` crate. Treats parse errors
171/// or missing files as "allow everything" so a broken robots.txt doesn't
172/// tank the crawl.
173pub struct Robots {
174    text: Option<String>,
175}
176
177impl Robots {
178    pub fn allowed(&self, url: &str) -> bool {
179        let Some(text) = &self.text else { return true };
180        let mut matcher = robotstxt::DefaultMatcher::default();
181        matcher.one_agent_allowed_by_robots(text, "*", url)
182    }
183    fn empty() -> Self {
184        Self { text: None }
185    }
186}
187
188pub async fn load_robots(client: &Client, base_origin: &str) -> (Robots, String, Option<String>) {
189    let robots_url = format!("{base_origin}/robots.txt");
190    let mut robots = Robots::empty();
191    let mut raw: Option<String> = None;
192    if let Ok(r) = client.get(&robots_url).send().await {
193        if r.status().as_u16() == 200 {
194            if let Ok(text) = r.text().await {
195                robots.text = Some(text.clone());
196                raw = Some(text);
197            }
198        }
199    }
200    (robots, robots_url, raw)
201}
202
203pub async fn load_sitemap(
204    client: &Client,
205    base_origin: &str,
206    robots_text: Option<&str>,
207) -> Vec<String> {
208    let mut candidates: Vec<String> = Vec::new();
209    if let Some(text) = robots_text {
210        for line in text.lines() {
211            let line = line.trim();
212            if let Some(rest) = line.to_lowercase().strip_prefix("sitemap:") {
213                // Reuse the original to preserve casing of the URL.
214                let original_after = &line[line.len() - rest.len()..];
215                candidates.push(original_after.trim().to_string());
216            }
217        }
218    }
219    if candidates.is_empty() {
220        candidates.push(format!("{base_origin}/sitemap.xml"));
221    }
222
223    let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
224    let mut urls: Vec<String> = Vec::new();
225    let mut to_fetch: Vec<String> = candidates;
226    while let Some(smurl) = to_fetch.pop() {
227        if seen.len() >= 20 {
228            break;
229        }
230        if seen.contains(&smurl) {
231            continue;
232        }
233        seen.insert(smurl.clone());
234        let r = match client.get(&smurl).send().await {
235            Ok(r) => r,
236            Err(_) => continue,
237        };
238        if r.status().as_u16() != 200 {
239            continue;
240        }
241        let body = match r.bytes().await {
242            Ok(b) => b,
243            Err(_) => continue,
244        };
245        for loc in parse_sitemap_xml(&body) {
246            let lower = loc.to_lowercase();
247            if lower.ends_with(".xml") || lower.contains("sitemap") {
248                to_fetch.push(loc);
249            } else {
250                urls.push(loc);
251            }
252        }
253    }
254    urls
255}
256
257/// Pull `<loc>` text from a sitemap XML body. Lightweight regex-based
258/// extraction; sitemaps are well-formed enough that we don't need a full
259/// XML parser, and adding one just for this would be overkill.
260fn parse_sitemap_xml(body: &[u8]) -> Vec<String> {
261    let s = String::from_utf8_lossy(body);
262    let re = regex::Regex::new(r"(?is)<loc>\s*([^<]+?)\s*</loc>").expect("regex");
263    re.captures_iter(&s)
264        .map(|c| c[1].trim().to_string())
265        .collect()
266}
267
268pub fn same_site(url: &str, host: &str) -> bool {
269    let Ok(u) = Url::parse(url) else { return false };
270    let h_lower = host.to_lowercase();
271    let Some(u_host) = u.host_str() else { return false };
272    let u_lower = u_host.to_lowercase();
273    if u_lower == h_lower {
274        return true;
275    }
276    if u_lower == format!("www.{h_lower}") || h_lower == format!("www.{u_lower}") {
277        return true;
278    }
279    false
280}
281
282/// Hosts that aggressively block non-browser HTTP clients, returning 403/404
283/// to anything that looks like a crawler regardless of the actual link state.
284/// Treating their responses as broken-link signals produces only false
285/// positives, so we skip them entirely from the external HEAD probe and the
286/// broken-external-links check.
287pub fn is_crawler_hostile(url: &str) -> bool {
288    let Ok(u) = Url::parse(url) else { return false };
289    let Some(host) = u.host_str() else { return false };
290    let host = host.to_lowercase();
291    matches!(
292        host.as_str(),
293        "linkedin.com" | "www.linkedin.com"
294    ) || host.ends_with(".linkedin.com")
295}