status-rust

  1//! In-process SEO crawler. Fetches up to PAGE_CAP pages from the same host,
  2//! collects metadata, and runs a fixed set of checks. Designed to be invoked
  3//! from the scheduler with a progress callback.
  4mod checks;
  5mod fetcher;
  6mod parser;
  7
  8use anyhow::Result;
  9use serde_json::Value;
 10use std::collections::{HashMap, HashSet, VecDeque};
 11use std::sync::Arc;
 12use std::time::Instant;
 13use url::Url;
 14
 15pub use fetcher::PAGE_CAP;
 16use fetcher::{
 17    fetch, head_status, load_robots, load_sitemap, make_client, make_probe_client,
 18    is_crawler_hostile, probe_compression, same_site, FetchResult, CRAWL_DEADLINE_SECS,
 19    CONCURRENCY, EXTERNAL_LINK_CAP,
 20};
 21use parser::parse_html;
 22
 23#[derive(Debug, Clone, serde::Serialize)]
 24pub struct Page {
 25    pub url: String,
 26    pub requested_url: String,
 27    pub status: u16,
 28    pub content_type: String,
 29    pub elapsed_ms: i64,
 30    pub bytes: usize,
 31    pub headers: HashMap<String, String>,
 32    pub redirect_chain: Vec<(u16, String)>,
 33    pub error: String,
 34    pub is_html: bool,
 35    #[serde(flatten)]
 36    pub html: Option<parser::ParsedHtml>,
 37}
 38
 39#[derive(Debug, serde::Serialize)]
 40pub struct CrawlResult {
 41    pub start_url: String,
 42    pub host: String,
 43    pub pages: Vec<Page>,
 44    pub external_link_status: HashMap<String, u16>,
 45    pub sitemap_urls: Vec<String>,
 46    pub robots: RobotsCtx,
 47    /// Server's `Content-Encoding` for the start URL (e.g. "gzip", "br",
 48    /// "zstd"). `None` means the server returned the response uncompressed.
 49    /// Probed with a separate non-decompressing client because reqwest's auto-
 50    /// decompression strips the header.
 51    pub compression: Option<String>,
 52}
 53
 54#[derive(Debug, serde::Serialize)]
 55pub struct RobotsCtx {
 56    pub url: String,
 57    pub exists: bool,
 58    pub raw: Option<String>,
 59    pub references_sitemap: bool,
 60}
 61
 62fn normalize(url: &str) -> String {
 63    if let Ok(mut u) = Url::parse(url) {
 64        u.set_fragment(None);
 65        let s = u.to_string();
 66        let trimmed = s.trim_end_matches('/');
 67        if trimmed.is_empty() { s } else { trimmed.to_string() }
 68    } else {
 69        url.to_string()
 70    }
 71}
 72
 73/// Crawl `start_url`, run all checks, return the flat insight list.
 74pub async fn run_seo_spider<F>(start_url: &str, mut progress_cb: F) -> Result<Vec<Value>>
 75where
 76    F: FnMut(usize) + Send + 'static,
 77{
 78    let start = Instant::now();
 79    tracing::info!("[crawler] starting {start_url}");
 80    let result = crawl(start_url, &mut progress_cb).await?;
 81    let insights = run_checks(&result);
 82    tracing::info!(
 83        "[crawler] done {start_url} - {} pages, {} insights, {:.1}s",
 84        result.pages.len(),
 85        insights.len(),
 86        start.elapsed().as_secs_f64()
 87    );
 88    Ok(insights)
 89}
 90
 91async fn crawl<F>(start_url: &str, progress_cb: &mut F) -> Result<CrawlResult>
 92where
 93    F: FnMut(usize) + Send,
 94{
 95    let parsed = Url::parse(start_url)?;
 96    let host = parsed.host_str().unwrap_or("").to_string();
 97    let base_origin = format!("{}://{}", parsed.scheme(), parsed.host_str().unwrap_or(""));
 98
 99    let client = make_client();
100    let probe_client = make_probe_client();
101    let compression = probe_compression(&probe_client, start_url).await;
102    let (robots, robots_url, robots_text) = load_robots(&client, &base_origin).await;
103    let robots = Arc::new(robots);
104    let sitemap_urls = load_sitemap(&client, &base_origin, robots_text.as_deref()).await;
105
106    let references_sitemap = robots_text
107        .as_deref()
108        .map(|t| {
109            t.lines()
110                .any(|l| l.trim().to_lowercase().starts_with("sitemap:"))
111        })
112        .unwrap_or(false);
113
114    let mut seen: HashSet<String> = HashSet::new();
115    let mut queue: VecDeque<String> = VecDeque::new();
116    let mut pages: Vec<Page> = Vec::new();
117    let mut fetched: HashSet<String> = HashSet::new();
118    let deadline = Instant::now() + std::time::Duration::from_secs(CRAWL_DEADLINE_SECS);
119
120    let enqueue = |url: String, seen: &mut HashSet<String>, queue: &mut VecDeque<String>| {
121        let n = normalize(&url);
122        if !seen.contains(&n) {
123            seen.insert(n);
124            queue.push_back(url);
125        }
126    };
127
128    enqueue(start_url.to_string(), &mut seen, &mut queue);
129    for url in sitemap_urls.iter().take(PAGE_CAP) {
130        if same_site(url, &host) {
131            enqueue(url.clone(), &mut seen, &mut queue);
132        }
133    }
134
135    while !queue.is_empty() && pages.len() < PAGE_CAP && Instant::now() < deadline {
136        // Pull a batch up to CONCURRENCY, respecting robots.
137        let mut batch: Vec<String> = Vec::new();
138        while let Some(url) = queue.pop_front() {
139            if !robots.allowed(&url) {
140                continue;
141            }
142            batch.push(url);
143            if batch.len() >= CONCURRENCY || pages.len() + batch.len() >= PAGE_CAP {
144                break;
145            }
146        }
147        if batch.is_empty() {
148            break;
149        }
150
151        let futs = batch
152            .into_iter()
153            .map(|u| {
154                let client = client.clone();
155                async move { fetch(&client, &u).await }
156            })
157            .collect::<Vec<_>>();
158        let results: Vec<FetchResult> = futures_util::future::join_all(futs).await;
159
160        for r in results {
161            let final_key = normalize(&r.url);
162            if fetched.contains(&final_key) {
163                seen.insert(final_key);
164                continue;
165            }
166            fetched.insert(final_key.clone());
167            let is_html = r.status == 200 && r.content_type.contains("text/html");
168            let mut page = Page {
169                url: r.url.clone(),
170                requested_url: r.requested_url.clone(),
171                status: r.status,
172                content_type: r.content_type.clone(),
173                elapsed_ms: r.elapsed_ms,
174                bytes: r.body.len(),
175                headers: r.headers.clone(),
176                redirect_chain: r.redirect_chain.clone(),
177                error: r.error.clone(),
178                is_html,
179                html: None,
180            };
181            if is_html {
182                match parse_html(&r.body, &r.url) {
183                    Ok(parsed) => {
184                        for link in &parsed.links {
185                            if same_site(&link.url, &host) {
186                                let n = normalize(&link.url);
187                                if !seen.contains(&n) {
188                                    seen.insert(n);
189                                    queue.push_back(link.url.clone());
190                                }
191                            }
192                        }
193                        page.html = Some(parsed);
194                    }
195                    Err(e) => {
196                        tracing::warn!("[crawler] parse failed for {}: {e}", r.url);
197                        page.is_html = false;
198                    }
199                }
200            }
201            seen.insert(normalize(&r.url));
202            pages.push(page);
203        }
204        progress_cb(pages.len());
205    }
206
207    if Instant::now() >= deadline {
208        tracing::warn!(
209            "[crawler] hit deadline for {start_url} after {} pages",
210            pages.len()
211        );
212    }
213
214    // External link HEAD check.
215    let mut external_links: HashSet<String> = HashSet::new();
216    for p in &pages {
217        if !p.is_html {
218            continue;
219        }
220        if let Some(html) = &p.html {
221            for link in &html.links {
222                if same_site(&link.url, &host) || is_crawler_hostile(&link.url) {
223                    continue;
224                }
225                external_links.insert(link.url.clone());
226            }
227        }
228    }
229    let mut external_link_status: HashMap<String, u16> = HashMap::new();
230    if !external_links.is_empty() && Instant::now() < deadline {
231        // Cap the probe list: a 500-page site can easily surface thousands of
232        // distinct external links, and at 4-way concurrency with an 8s
233        // timeout each, an unbounded list could run for hours.
234        let mut urls: Vec<String> = external_links.into_iter().collect();
235        if urls.len() > EXTERNAL_LINK_CAP {
236            urls.sort();
237            urls.truncate(EXTERNAL_LINK_CAP);
238            tracing::warn!(
239                "[crawler] capping external link probes at {EXTERNAL_LINK_CAP} for {start_url}"
240            );
241        }
242        for chunk in urls.chunks(CONCURRENCY) {
243            // The page-fetch loop already respects the deadline; the external
244            // probes must too, or a slow tail keeps the crawl "running" long
245            // past its budget (until the scheduler wedge-reset kills it).
246            if Instant::now() >= deadline {
247                tracing::warn!(
248                    "[crawler] hit deadline during external link probes for {start_url}"
249                );
250                break;
251            }
252            let futs = chunk.iter().cloned().map(|u| {
253                let client = client.clone();
254                async move {
255                    let s = head_status(&client, &u).await;
256                    (u, s)
257                }
258            });
259            let results = futures_util::future::join_all(futs).await;
260            for (u, s) in results {
261                external_link_status.insert(u, s);
262            }
263        }
264    }
265
266    Ok(CrawlResult {
267        start_url: start_url.to_string(),
268        host,
269        pages,
270        external_link_status,
271        sitemap_urls,
272        robots: RobotsCtx {
273            url: robots_url,
274            exists: robots_text.is_some(),
275            raw: robots_text,
276            references_sitemap,
277        },
278        compression,
279    })
280}
281
282fn run_checks(result: &CrawlResult) -> Vec<Value> {
283    let html_pages: Vec<&Page> = result.pages.iter().filter(|p| p.is_html).collect();
284    let mut status_map: HashMap<String, u16> = HashMap::new();
285    for p in &result.pages {
286        status_map.insert(p.url.clone(), p.status);
287    }
288    let ctx = checks::Ctx {
289        start_url: &result.start_url,
290        host: &result.host,
291        pages: &result.pages,
292        html_pages: &html_pages,
293        status_map: &status_map,
294        external_link_status: &result.external_link_status,
295        sitemap_urls: &result.sitemap_urls,
296        robots: &result.robots,
297        compression: result.compression.as_deref(),
298    };
299    checks::run_all(&ctx)
300}