Yahoo v10: crumb-gated quoteSummary now authenticates

@@ -340,6 +340,35 @@ version = "0.9.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"[[package]]name = "cookie"version = "0.18.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"dependencies = [ "percent-encoding", "time", "version_check",][[package]]name = "cookie_store"version = "0.22.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"dependencies = [ "cookie", "document-features", "idna", "log", "publicsuffix", "serde", "serde_derive", "serde_json", "time", "url",][[package]]name = "core-foundation-sys"version = "0.8.7"

@@ -436,6 +465,15 @@ dependencies = [ "zeroize",][[package]]name = "deranged"version = "0.5.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"dependencies = [ "powerfmt",][[package]]name = "derive_arbitrary"version = "1.4.2"

@@ -470,6 +508,15 @@ dependencies = [ "syn",][[package]]name = "document-features"version = "0.2.12"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"dependencies = [ "litrs",][[package]]name = "dotenvy"version = "0.15.7"

@@ -1102,6 +1149,12 @@ version = "0.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"[[package]]name = "litrs"version = "1.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"[[package]]name = "lock_api"version = "0.4.14"

@@ -1233,6 +1286,12 @@ dependencies = [ "zeroize",][[package]]name = "num-conv"version = "0.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"[[package]]name = "num-integer"version = "0.1.46"

@@ -1379,6 +1438,12 @@ dependencies = [ "zerovec",][[package]]name = "powerfmt"version = "0.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"[[package]]name = "ppv-lite86"version = "0.2.21"

@@ -1397,6 +1462,22 @@ dependencies = [ "unicode-ident",][[package]]name = "psl-types"version = "2.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"[[package]]name = "publicsuffix"version = "2.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"dependencies = [ "idna", "psl-types",][[package]]name = "quick-xml"version = "0.37.5"

@@ -1578,6 +1659,8 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"dependencies = [ "base64", "bytes", "cookie", "cookie_store", "futures-core", "http", "http-body",

@@ -2155,6 +2238,37 @@ dependencies = [ "cfg-if",][[package]]name = "time"version = "0.3.47"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c"dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", "serde_core", "time-core", "time-macros",][[package]]name = "time-core"version = "0.1.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"[[package]]name = "time-macros"version = "0.2.27"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215"dependencies = [ "num-conv", "time-core",][[package]]name = "tinystr"version = "0.8.3"

@@ -20,7 +20,7 @@ thiserror = "2"tracing = "0.1"tracing-subscriber = { version = "0.3", features = ["env-filter"] }dotenvy = "0.15"reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip", "brotli", "json"] }reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "gzip", "brotli", "json", "cookies"] }async-trait = "0.1"futures-util = "0.3"async-stream = "0.3"

@@ -12,6 +12,12 @@ pub fn build_client(config: &Config) -> reqwest::Client {    reqwest::Client::builder()        .user_agent(config.user_agent.clone())        .timeout(Duration::from_secs(25))        // Yahoo's v10 `quoteSummary` endpoint is crumb-gated and the crumb        // round-trip requires session cookies — the GET to fc.yahoo.com sets        // one, which the subsequent /v1/test/getcrumb call must echo. With        // cookies enabled here, [`YahooProvider::ensure_crumb`] does the        // dance and the cookies replay automatically on later requests.        .cookie_store(true)        .build()        .expect("reqwest client builds")}

@@ -11,6 +11,8 @@//! reuses it to validate and describe a symbol the add-symbol flow (Phase 9)//! is about to register.use std::sync::Mutex;use anyhow::{anyhow, Result};use async_trait::async_trait;use reqwest::{header::RETRY_AFTER, StatusCode};

@@ -24,11 +26,75 @@ use crate::providers::{/// Near-real-time quotes from Yahoo Finance.pub struct YahooProvider {    client: reqwest::Client,    /// Cached crumb token for Yahoo's v10 `quoteSummary` endpoint, which is    /// crumb-gated and refuses (401) without it. Lazy: only the first    /// `quoteSummary` call pays the round-trip, then every subsequent call    /// in the process replays the cached token until Yahoo invalidates it.    crumb: Mutex<Option<String>>,}impl YahooProvider {    pub fn new(client: reqwest::Client) -> Self {        Self { client }        Self {            client,            crumb: Mutex::new(None),        }    }    /// Return a Yahoo `crumb` token, fetching one if the cache is empty.    /// The dance is two requests: a primer to `fc.yahoo.com` that drops a    /// session cookie (handled by reqwest's cookie jar — see `http.rs`),    /// then a GET to `/v1/test/getcrumb` that replays the cookie and    /// returns the crumb as a plain-text body. A failure here propagates    /// as a regular `RateLimited` / transport error so the caller can feed    /// the endpoint guard the same way as a v10 call would.    async fn ensure_crumb(&self) -> Result<String> {        if let Some(c) = self.crumb.lock().expect("crumb cache").clone() {            return Ok(c);        }        // Primer: any 2xx body is fine, we only want the Set-Cookie.        let _ = self            .client            .get("https://fc.yahoo.com/")            .send()            .await?            .error_for_status();        // The crumb endpoint returns the token as its raw body.        let resp = self            .client            .get("https://query1.finance.yahoo.com/v1/test/getcrumb")            .send()            .await?;        let status = resp.status();        if matches!(            status,            StatusCode::TOO_MANY_REQUESTS                | StatusCode::SERVICE_UNAVAILABLE                | StatusCode::UNAUTHORIZED                | StatusCode::FORBIDDEN        ) {            let retry_after_secs = resp                .headers()                .get(RETRY_AFTER)                .and_then(|v| v.to_str().ok())                .and_then(|s| s.trim().parse::<i64>().ok());            return Err(anyhow::Error::new(RateLimited {                status: status.as_u16(),                retry_after_secs,            }));        }        let crumb = resp.error_for_status()?.text().await?.trim().to_string();        if crumb.is_empty() {            return Err(anyhow!("yahoo returned an empty crumb"));        }        *self.crumb.lock().expect("crumb cache") = Some(crumb.clone());        Ok(crumb)    }    /// Forget the cached crumb. Called on a 401 from a v10 call so the next    /// attempt fetches a fresh one (Yahoo crumbs rotate occasionally).    fn invalidate_crumb(&self) {        *self.crumb.lock().expect("crumb cache") = None;    }}

@@ -294,20 +360,67 @@ impl YahooProvider {    /// returned [`FundMetadata`] may carry only a subset of fields populated    /// — Yahoo's coverage is uneven across small ETFs.    pub async fn fund_metadata(&self, ticker: &str) -> Result<Option<FundMetadata>> {        let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();        // The five modules that between them carry every Phase 28 field. A        // module Yahoo does not recognise for this symbol is silently        // omitted from the response (rather than failing the whole request).        let Some(result) = self            .quote_summary(                ticker,                "fundProfile,defaultKeyStatistics,summaryDetail,price,assetProfile",            )            .await?        else {            return Ok(None);        };        Ok(Some(parse_fund_metadata(result)))    }    /// Shared v10 `quoteSummary` fetch: ensures the crumb is cached, builds    /// the URL with `&crumb=...`, parses gating responses (429 / 503 / 401 /    /// 403) as the typed [`RateLimited`], and treats Yahoo's "unknown    /// symbol" replies (404 or a `quoteSummary.error` body) as a clean    /// `Ok(None)`. A bare 401 on a previously-good crumb is retried once    /// with a fresh one — Yahoo rotates crumbs and this masks the rotation    /// from the endpoint guard.    async fn quote_summary(        &self,        ticker: &str,        modules: &str,    ) -> Result<Option<QuoteSummaryResult>> {        let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();        // First attempt with the cached crumb (may fetch one on the first        // call of the process).        match self.quote_summary_once(&sym, modules).await {            Ok(v) => Ok(v),            Err(e) => {                // A 401/403 on a request we sent a crumb with means the                // crumb expired. Drop it and try one more time so the                // caller sees a clean answer instead of a guard trip.                let retry = e.downcast_ref::<RateLimited>().is_some_and(|r| {                    matches!(r.status, 401 | 403)                });                if retry {                    self.invalidate_crumb();                    return self.quote_summary_once(&sym, modules).await;                }                Err(e)            }        }    }    async fn quote_summary_once(        &self,        sym: &str,        modules: &str,    ) -> Result<Option<QuoteSummaryResult>> {        let crumb = self.ensure_crumb().await?;        let url = format!(            "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{sym}\             ?modules=fundProfile,defaultKeyStatistics,summaryDetail,price,assetProfile"             ?modules={modules}&crumb={c}",            c = urlencoding::encode(&crumb),        );        let resp = self.client.get(&url).send().await?;        let status = resp.status();        // The v10 endpoint is occasionally crumb-gated — Yahoo returns        // either a plain 401 / 403 or, confusingly, a 429 from the same gate.        // Treat all of them as a rate-limit signal so the guard trips and        // we defer the sweep rather than spinning on a broken upstream.        if matches!(            status,            StatusCode::TOO_MANY_REQUESTS

@@ -333,14 +446,10 @@ impl YahooProvider {        if env.quote_summary.error.is_some() {            return Ok(None);        }        let Some(result) = env        Ok(env            .quote_summary            .result            .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) })        else {            return Ok(None);        };        Ok(Some(parse_fund_metadata(result)))            .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) }))    }    /// Fetch the next-expected earnings date for `ticker` from Yahoo's

@@ -356,43 +465,7 @@ impl YahooProvider {    /// 503 / 401 / 403) surface as the typed [`RateLimited`], same defensive    /// set as `fund_metadata`.    pub async fn earnings_calendar(&self, ticker: &str) -> Result<Option<i64>> {        let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();        let url = format!(            "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{sym}\             ?modules=calendarEvents"        );        let resp = self.client.get(&url).send().await?;        let status = resp.status();        if matches!(            status,            StatusCode::TOO_MANY_REQUESTS                | StatusCode::SERVICE_UNAVAILABLE                | StatusCode::UNAUTHORIZED                | StatusCode::FORBIDDEN        ) {            let retry_after_secs = resp                .headers()                .get(RETRY_AFTER)                .and_then(|v| v.to_str().ok())                .and_then(|s| s.trim().parse::<i64>().ok());            return Err(anyhow::Error::new(RateLimited {                status: status.as_u16(),                retry_after_secs,            }));        }        if status == StatusCode::NOT_FOUND {            return Ok(None);        }        let resp = resp.error_for_status()?;        let env: QuoteSummaryEnvelope = resp.json().await?;        if env.quote_summary.error.is_some() {            return Ok(None);        }        let Some(result) = env            .quote_summary            .result            .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) })        else {        let Some(result) = self.quote_summary(ticker, "calendarEvents").await? else {            return Ok(None);        };        // `earningsDate` is an array; Yahoo populates 1 or 2 entries — the

@@ -426,43 +499,7 @@ impl YahooProvider {    /// 401 / 403) surface as the typed [`RateLimited`] so the endpoint    /// guard trips at once.    pub async fn asset_profile(&self, ticker: &str) -> Result<Option<AssetProfile>> {        let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();        let url = format!(            "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{sym}\             ?modules=assetProfile"        );        let resp = self.client.get(&url).send().await?;        let status = resp.status();        if matches!(            status,            StatusCode::TOO_MANY_REQUESTS                | StatusCode::SERVICE_UNAVAILABLE                | StatusCode::UNAUTHORIZED                | StatusCode::FORBIDDEN        ) {            let retry_after_secs = resp                .headers()                .get(RETRY_AFTER)                .and_then(|v| v.to_str().ok())                .and_then(|s| s.trim().parse::<i64>().ok());            return Err(anyhow::Error::new(RateLimited {                status: status.as_u16(),                retry_after_secs,            }));        }        if status == StatusCode::NOT_FOUND {            return Ok(None);        }        let resp = resp.error_for_status()?;        let env: QuoteSummaryEnvelope = resp.json().await?;        if env.quote_summary.error.is_some() {            return Ok(None);        }        let Some(result) = env            .quote_summary            .result            .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) })        else {        let Some(result) = self.quote_summary(ticker, "assetProfile").await? else {            return Ok(None);        };        let ap = result.asset_profile.unwrap_or_default();