Single-binary self-hosted market watcher for stocks, ETFs, indexes, and futures: live charts, key stats, fundamentals, SEC filings, and SSE streaming.
axumdockerfinancerustself-hostedsqlitestocksvite
1//! Live quotes and intraday bars from Yahoo Finance's v8 chart endpoint.
2//!
3//! `https://query1.finance.yahoo.com/v8/finance/chart/<symbol>` needs no API
4//! key — just an ordinary browser User-Agent, which the shared client already
5//! sends. One call with `interval=15m&range=1d` returns the day's 15-minute
6//! bars *and* a `meta` block carrying the live quote, so a single request
7//! feeds both the `quotes` snapshot and `intraday_bars`.
8//!
9//! The same endpoint also identifies a symbol — its name, instrument type,
10//! exchange and currency all sit in `meta` — so [`YahooProvider::lookup`]
11//! reuses it to validate and describe a symbol the add-symbol flow (Phase 9)
12//! is about to register.
13
14use std::sync::Mutex;
15
16use anyhow::{anyhow, Result};
17use async_trait::async_trait;
18use reqwest::{header::RETRY_AFTER, StatusCode};
19use serde::Deserialize;
20
21use crate::providers::{
22 AssetProfile, DailyBar, DividendEvent, FundMetadata, HistoryProvider, IntradayBar, Quote,
23 QuoteData, QuoteProvider, RateLimited,
24};
25
26/// Near-real-time quotes from Yahoo Finance.
27pub struct YahooProvider {
28 client: reqwest::Client,
29 /// Cached crumb token for Yahoo's v10 `quoteSummary` endpoint, which is
30 /// crumb-gated and refuses (401) without it. Lazy: only the first
31 /// `quoteSummary` call pays the round-trip, then every subsequent call
32 /// in the process replays the cached token until Yahoo invalidates it.
33 crumb: Mutex<Option<String>>,
34}
35
36impl YahooProvider {
37 pub fn new(client: reqwest::Client) -> Self {
38 Self {
39 client,
40 crumb: Mutex::new(None),
41 }
42 }
43
44 /// Return a Yahoo `crumb` token, fetching one if the cache is empty.
45 /// The dance is two requests: a primer to `fc.yahoo.com` that drops a
46 /// session cookie (handled by reqwest's cookie jar — see `http.rs`),
47 /// then a GET to `/v1/test/getcrumb` that replays the cookie and
48 /// returns the crumb as a plain-text body. A failure here propagates
49 /// as a regular `RateLimited` / transport error so the caller can feed
50 /// the endpoint guard the same way as a v10 call would.
51 async fn ensure_crumb(&self) -> Result<String> {
52 if let Some(c) = self.crumb.lock().expect("crumb cache").clone() {
53 return Ok(c);
54 }
55 // Primer: any 2xx body is fine, we only want the Set-Cookie.
56 let _ = self
57 .client
58 .get("https://fc.yahoo.com/")
59 .send()
60 .await?
61 .error_for_status();
62 // The crumb endpoint returns the token as its raw body.
63 let resp = self
64 .client
65 .get("https://query1.finance.yahoo.com/v1/test/getcrumb")
66 .send()
67 .await?;
68 let status = resp.status();
69 if matches!(
70 status,
71 StatusCode::TOO_MANY_REQUESTS
72 | StatusCode::SERVICE_UNAVAILABLE
73 | StatusCode::UNAUTHORIZED
74 | StatusCode::FORBIDDEN
75 ) {
76 let retry_after_secs = resp
77 .headers()
78 .get(RETRY_AFTER)
79 .and_then(|v| v.to_str().ok())
80 .and_then(|s| s.trim().parse::<i64>().ok());
81 return Err(anyhow::Error::new(RateLimited {
82 status: status.as_u16(),
83 retry_after_secs,
84 }));
85 }
86 let crumb = resp.error_for_status()?.text().await?.trim().to_string();
87 if crumb.is_empty() {
88 return Err(anyhow!("yahoo returned an empty crumb"));
89 }
90 *self.crumb.lock().expect("crumb cache") = Some(crumb.clone());
91 Ok(crumb)
92 }
93
94 /// Forget the cached crumb. Called on a 401 from a v10 call so the next
95 /// attempt fetches a fresh one (Yahoo crumbs rotate occasionally).
96 fn invalidate_crumb(&self) {
97 *self.crumb.lock().expect("crumb cache") = None;
98 }
99}
100
101/// Map a canonical ticker to Yahoo's symbol scheme.
102/// - most indexes already match Yahoo (`^DJI`, `^NDX`, `^RUT`, `^VIX`)
103/// - two differ: our Stooq-style `^SPX` / `^NDQ` are `^GSPC` / `^IXIC` on Yahoo
104/// - stocks and ETFs are the plain ticker with `.` rewritten to `-`
105/// (`BRK.B` -> `BRK-B`)
106fn yahoo_symbol(ticker: &str) -> String {
107 match ticker {
108 "^SPX" => "^GSPC".to_string(),
109 "^NDQ" => "^IXIC".to_string(),
110 t if t.starts_with('^') => t.to_string(),
111 t => t.replace('.', "-"),
112 }
113}
114
115// ── identity of a looked-up symbol ─────────────────────────────────────────
116
117/// Identifying metadata for a symbol, derived from Yahoo's chart `meta`. The
118/// add-symbol flow uses it to register a new symbol with a real name and kind
119/// rather than a bare ticker.
120#[derive(Debug, Clone)]
121pub struct SymbolInfo {
122 pub name: String,
123 /// One of `stock` | `etf` | `index` | `future`.
124 pub kind: String,
125 pub exchange: Option<String>,
126 pub currency: String,
127}
128
129/// The outcome of [`YahooProvider::lookup`]. An `Err` from `lookup` is a
130/// genuine transport / rate-limit failure (and should feed the endpoint
131/// guard); these three variants are all successful answers from Yahoo.
132#[derive(Debug)]
133pub enum SymbolLookup {
134 /// Yahoo knows this symbol: its identity, plus the quote and intraday bars
135 /// the same request returned.
136 Found { info: SymbolInfo, data: QuoteData },
137 /// Yahoo has no such symbol.
138 Unknown,
139 /// Yahoo knows the symbol but it is an instrument type this app does not
140 /// model yet (a currency pair, a cryptocurrency, ...). Carries the raw type.
141 Unsupported(String),
142}
143
144// ── the slice of the v8 chart JSON we read ─────────────────────────────────
145
146#[derive(Deserialize)]
147struct ChartEnvelope {
148 chart: Chart,
149}
150
151#[derive(Deserialize)]
152struct Chart {
153 result: Option<Vec<ChartResult>>,
154 /// Non-null on a logical failure (e.g. an unknown symbol).
155 error: Option<serde_json::Value>,
156}
157
158#[derive(Deserialize)]
159struct ChartResult {
160 meta: Meta,
161 /// Bar-start times, Unix seconds. Absent when the day has no bars yet.
162 timestamp: Option<Vec<i64>>,
163 indicators: Indicators,
164 /// `events.dividends` carries declared payouts when the request asked for
165 /// `events=div` (Phase 26). Absent on a routine quote fetch.
166 events: Option<ChartEvents>,
167}
168
169/// The events block of a Yahoo chart payload. Each value of `dividends` is
170/// keyed by the event's Unix-second timestamp (a JSON string, which is why
171/// the outer type is a map).
172#[derive(Default, Deserialize)]
173struct ChartEvents {
174 #[serde(default)]
175 dividends: std::collections::HashMap<String, ChartDividend>,
176}
177
178#[derive(Deserialize)]
179struct ChartDividend {
180 /// Per-share amount.
181 amount: f64,
182 /// Ex-dividend date as a Unix second. Yahoo also echoes the timestamp as
183 /// the map key, but the inner field is the canonical one to read off.
184 date: i64,
185}
186
187#[derive(Deserialize)]
188#[serde(rename_all = "camelCase")]
189struct Meta {
190 regular_market_price: Option<f64>,
191 previous_close: Option<f64>,
192 chart_previous_close: Option<f64>,
193 regular_market_open: Option<f64>,
194 regular_market_day_high: Option<f64>,
195 regular_market_day_low: Option<f64>,
196 regular_market_volume: Option<i64>,
197 /// The source's own timestamp for the quote, Unix seconds.
198 regular_market_time: Option<i64>,
199 /// Seconds east of UTC for the exchange's timezone (e.g. -14400 for ET in
200 /// summer). Daily-bar timestamps are bucketed by the exchange's local day,
201 /// so the daily-history parser adds this offset before taking the date.
202 gmtoffset: Option<i64>,
203 market_state: Option<String>,
204 /// Identity fields — read only by `lookup`. `EQUITY` | `ETF` | `INDEX` |
205 /// `MUTUALFUND` | `FUTURE` | `CURRENCY` | `CRYPTOCURRENCY` | ...
206 instrument_type: Option<String>,
207 short_name: Option<String>,
208 long_name: Option<String>,
209 currency: Option<String>,
210 exchange_name: Option<String>,
211 full_exchange_name: Option<String>,
212}
213
214#[derive(Deserialize)]
215struct Indicators {
216 quote: Vec<OhlcvArrays>,
217}
218
219/// Column-oriented OHLCV: one parallel array per field, indexed by bar. A cell
220/// can be null when a bar has no print, so every element is optional.
221#[derive(Default, Deserialize)]
222struct OhlcvArrays {
223 #[serde(default)]
224 open: Vec<Option<f64>>,
225 #[serde(default)]
226 high: Vec<Option<f64>>,
227 #[serde(default)]
228 low: Vec<Option<f64>>,
229 #[serde(default)]
230 close: Vec<Option<f64>>,
231 #[serde(default)]
232 volume: Vec<Option<i64>>,
233}
234
235/// One row in a market-movers list (top gainers / losers / most active), from the
236/// predefined-screener endpoint. A plain snapshot for the dashboard, cached as
237/// JSON in `meta` (hence `Deserialize` too), not a stored table row.
238#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
239pub struct Mover {
240 pub symbol: String,
241 pub name: String,
242 pub price: Option<f64>,
243 /// Day % move, already a percent (Yahoo returns e.g. 20.08, not 0.2008).
244 pub change_pct: Option<f64>,
245 pub volume: Option<i64>,
246}
247
248#[derive(Deserialize)]
249struct ScreenerEnvelope {
250 finance: ScreenerFinance,
251}
252#[derive(Deserialize)]
253struct ScreenerFinance {
254 #[serde(default)]
255 result: Vec<ScreenerResult>,
256}
257#[derive(Deserialize)]
258struct ScreenerResult {
259 #[serde(default)]
260 quotes: Vec<ScreenerQuote>,
261}
262#[derive(Deserialize)]
263#[serde(rename_all = "camelCase")]
264struct ScreenerQuote {
265 symbol: String,
266 short_name: Option<String>,
267 long_name: Option<String>,
268 display_name: Option<String>,
269 regular_market_price: Option<f64>,
270 regular_market_change_percent: Option<f64>,
271 regular_market_volume: Option<i64>,
272}
273
274impl YahooProvider {
275 /// Fetch a predefined market-movers screener (`day_gainers` / `day_losers` /
276 /// `most_actives`), `count` rows. One unauthenticated GET (the predefined
277 /// screeners are not crumb-gated). A 429/503/401/403 surfaces as the typed
278 /// [`RateLimited`] so the endpoint guard trips its breaker at once, exactly
279 /// like the chart calls; anything else parses to the rows (empty on a shape
280 /// we don't recognise).
281 pub async fn fetch_movers(&self, scr_id: &str, count: u32) -> Result<Vec<Mover>> {
282 let url = format!(
283 "https://query1.finance.yahoo.com/v1/finance/screener/predefined/saved\
284 ?count={count}&scrIds={scr_id}"
285 );
286 let resp = self.client.get(&url).send().await?;
287 let status = resp.status();
288 if matches!(
289 status,
290 StatusCode::TOO_MANY_REQUESTS
291 | StatusCode::SERVICE_UNAVAILABLE
292 | StatusCode::UNAUTHORIZED
293 | StatusCode::FORBIDDEN
294 ) {
295 let retry_after_secs = resp
296 .headers()
297 .get(RETRY_AFTER)
298 .and_then(|v| v.to_str().ok())
299 .and_then(|s| s.trim().parse::<i64>().ok());
300 return Err(anyhow::Error::new(RateLimited {
301 status: status.as_u16(),
302 retry_after_secs,
303 }));
304 }
305 let resp = resp.error_for_status()?;
306 let env: ScreenerEnvelope = resp.json().await?;
307 let quotes = env
308 .finance
309 .result
310 .into_iter()
311 .next()
312 .map(|r| r.quotes)
313 .unwrap_or_default();
314 Ok(quotes
315 .into_iter()
316 .map(|q| {
317 let name = q
318 .short_name
319 .or(q.display_name)
320 .or(q.long_name)
321 .unwrap_or_else(|| q.symbol.clone());
322 Mover {
323 symbol: q.symbol,
324 name,
325 price: q.regular_market_price,
326 change_pct: q.regular_market_change_percent,
327 volume: q.regular_market_volume,
328 }
329 })
330 .collect())
331 }
332
333 /// Fetch and parse the v8 chart payload for `ticker`.
334 ///
335 /// `Ok(Some(_))` is a real chart result; `Ok(None)` means Yahoo answered
336 /// cleanly that it has no such symbol (a 404, or a `chart.error` body) —
337 /// a definitive "unknown", not a failure. `Err` is a transport error or an
338 /// explicit rate-limit signal, surfaced as the typed [`RateLimited`] so the
339 /// endpoint guard trips its breaker at once.
340 async fn fetch_chart(&self, ticker: &str) -> Result<Option<ChartResult>> {
341 self.fetch_chart_range(ticker, "1d").await
342 }
343
344 /// Fetch the v8 chart payload for `ticker` over an explicit intraday `range`
345 /// (e.g. `1d` for the routine quote, `5d` to backfill the whole trading week
346 /// for the end-of-week dashboard view). `interval=15m` is held constant, so
347 /// `5d` returns five trading days of 15-minute bars in one request.
348 async fn fetch_chart_range(&self, ticker: &str, range: &str) -> Result<Option<ChartResult>> {
349 // `^` is not a bare path character; percent-encode the symbol.
350 let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();
351 let url = format!(
352 "https://query1.finance.yahoo.com/v8/finance/chart/{sym}\
353 ?interval=15m&range={range}&includePrePost=true"
354 );
355 self.request_chart(&url).await
356 }
357
358 /// Fetch a symbol's deep daily OHLCV history from the same v8 chart
359 /// endpoint, one bar per trading day. `since` (a `YYYY-MM-DD` date) selects
360 /// an incremental window via Yahoo's `period1`/`period2` epoch params;
361 /// `None` asks for the full `range=max` history. A single call returns the
362 /// entire deep history (or the window), which is why Yahoo replaced the
363 /// per-symbol Stooq fetch the app used through Phase 1.
364 ///
365 /// Error / "unknown symbol" semantics match [`Self::fetch_chart`].
366 async fn fetch_daily(&self, ticker: &str, since: Option<&str>) -> Result<Option<ChartResult>> {
367 let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();
368 let chart_url = |window: &str| {
369 format!("https://query1.finance.yahoo.com/v8/finance/chart/{sym}?interval=1d&{window}")
370 };
371 match since {
372 Some(d) => {
373 // Re-fetch from the day before `since` to "now + 1 day" (epoch
374 // seconds). The day-before back-step and the day-ahead end
375 // cover same-day re-runs and exchange-timezone edges; the
376 // daily upsert makes any overlap free.
377 let p1 = chrono::NaiveDate::parse_from_str(d, "%Y-%m-%d")
378 .ok()
379 .and_then(|dt| dt.pred_opt())
380 .and_then(|dt| dt.and_hms_opt(0, 0, 0))
381 .map(|dt| dt.and_utc().timestamp())
382 .unwrap_or(0);
383 let p2 = chrono::Utc::now().timestamp() + 86_400;
384 self.request_chart(&chart_url(&format!("period1={p1}&period2={p2}")))
385 .await
386 }
387 None => {
388 // Deep backfill: ask for the full history. Yahoo honours
389 // `interval=1d` at `range=max` for most symbols (full daily
390 // history, e.g. ^SPX back to the 1700s), but silently
391 // downsamples it to monthly / quarterly bars for some index and
392 // futures symbols (^RUT, ^VIX, the `=F` futures) even though a
393 // daily interval was requested. When that happens, refetch a
394 // bounded 10-year window, which Yahoo *does* serve at daily
395 // granularity — far better for a daily chart than coarse
396 // max-range bars. (Costs one extra request for those few
397 // symbols, once per deep backfill.)
398 let res = self.request_chart(&chart_url("range=max")).await?;
399 match &res {
400 Some(r) if is_downsampled(r) => {
401 self.request_chart(&chart_url("range=10y")).await
402 }
403 _ => Ok(res),
404 }
405 }
406 }
407 }
408
409 /// GET a v8 chart URL and parse it into at most one [`ChartResult`].
410 ///
411 /// `Ok(Some(_))` is a real result; `Ok(None)` means Yahoo answered cleanly
412 /// that it has no such symbol (a 404, or a `chart.error` body) — a
413 /// definitive "unknown", not a failure. `Err` is a transport error or an
414 /// explicit rate-limit signal, surfaced as the typed [`RateLimited`] so the
415 /// endpoint guard trips its breaker at once.
416 async fn request_chart(&self, url: &str) -> Result<Option<ChartResult>> {
417 let resp = self.client.get(url).send().await?;
418 let status = resp.status();
419
420 if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::SERVICE_UNAVAILABLE {
421 let retry_after_secs = resp
422 .headers()
423 .get(RETRY_AFTER)
424 .and_then(|v| v.to_str().ok())
425 .and_then(|s| s.trim().parse::<i64>().ok());
426 return Err(anyhow::Error::new(RateLimited {
427 status: status.as_u16(),
428 retry_after_secs,
429 }));
430 }
431 // Yahoo answers an unknown symbol with 404 and a `chart.error` body —
432 // a definitive "no such symbol", not a transport failure.
433 if status == StatusCode::NOT_FOUND {
434 return Ok(None);
435 }
436
437 let resp = resp.error_for_status()?;
438 let env: ChartEnvelope = resp.json().await?;
439 if env.chart.error.is_some() {
440 return Ok(None);
441 }
442 Ok(env
443 .chart
444 .result
445 .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) }))
446 }
447
448 /// Fetch the declared dividend history for `ticker` (Phase 26).
449 ///
450 /// The same v8 chart endpoint that serves quotes carries an
451 /// `events.dividends` series when the request asks for `events=div`. Ask
452 /// for a five-year window at daily granularity: that is plenty for the
453 /// page's prior-year + YTD totals and a long history list, while keeping
454 /// the payload modest (the candle stream itself is discarded here — only
455 /// the events block is parsed). Returns the payouts oldest first.
456 ///
457 /// Error semantics mirror [`Self::quote`]: a 429/503 surfaces as
458 /// [`RateLimited`] so the endpoint guard trips at once; an unknown symbol
459 /// (404 or `chart.error`) returns an empty vec, not an error, since the
460 /// guard should not treat it as a transport failure.
461 pub async fn dividends(&self, ticker: &str) -> Result<Vec<DividendEvent>> {
462 let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();
463 let url = format!(
464 "https://query1.finance.yahoo.com/v8/finance/chart/{sym}\
465 ?interval=1d&range=5y&events=div"
466 );
467 let resp = self.client.get(&url).send().await?;
468 let status = resp.status();
469 if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::SERVICE_UNAVAILABLE {
470 let retry_after_secs = resp
471 .headers()
472 .get(RETRY_AFTER)
473 .and_then(|v| v.to_str().ok())
474 .and_then(|s| s.trim().parse::<i64>().ok());
475 return Err(anyhow::Error::new(RateLimited {
476 status: status.as_u16(),
477 retry_after_secs,
478 }));
479 }
480 if status == StatusCode::NOT_FOUND {
481 return Ok(Vec::new());
482 }
483 let resp = resp.error_for_status()?;
484 let env: ChartEnvelope = resp.json().await?;
485 if env.chart.error.is_some() {
486 return Ok(Vec::new());
487 }
488 let Some(result) = env
489 .chart
490 .result
491 .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) })
492 else {
493 return Ok(Vec::new());
494 };
495 let mut out: Vec<DividendEvent> = result
496 .events
497 .unwrap_or_default()
498 .dividends
499 .into_values()
500 .filter_map(|d| {
501 // A non-positive amount or a nonsense timestamp is filtered;
502 // Yahoo has occasionally emitted a literal 0 placeholder.
503 if d.amount <= 0.0 {
504 return None;
505 }
506 let ex_date = chrono::DateTime::from_timestamp(d.date, 0)?
507 .format("%Y-%m-%d")
508 .to_string();
509 Some(DividendEvent {
510 ex_date,
511 amount: d.amount,
512 })
513 })
514 .collect();
515 out.sort_by(|a, b| a.ex_date.cmp(&b.ex_date));
516 Ok(out)
517 }
518
519 /// Fetch the Yahoo `quoteSummary` ETF metadata snapshot for `ticker`
520 /// (Phase 28). One request to `v10/finance/quoteSummary` pulls the five
521 /// modules that together carry every figure the Phase 28 ETF page needs
522 /// beyond what SEC N-PORT already provides — expense ratio, distribution
523 /// yield, latest NAV, inception, category, fund family, and the issuer's
524 /// strategy paragraph.
525 ///
526 /// Returns `Ok(None)` when Yahoo answers cleanly that it has no such
527 /// symbol (a 404 or a `quoteSummary.error` body) — a definitive empty,
528 /// not a guard failure. Yahoo's gating responses (`429`, `503`, and `401
529 /// "Invalid Crumb"` which the gate sometimes returns as either) surface
530 /// as the typed [`RateLimited`] so the endpoint guard trips at once. The
531 /// returned [`FundMetadata`] may carry only a subset of fields populated
532 /// — Yahoo's coverage is uneven across small ETFs.
533 pub async fn fund_metadata(&self, ticker: &str) -> Result<Option<FundMetadata>> {
534 // The five modules that between them carry every Phase 28 field. A
535 // module Yahoo does not recognise for this symbol is silently
536 // omitted from the response (rather than failing the whole request).
537 let Some(result) = self
538 .quote_summary(
539 ticker,
540 "fundProfile,defaultKeyStatistics,summaryDetail,price,assetProfile",
541 )
542 .await?
543 else {
544 return Ok(None);
545 };
546 Ok(Some(parse_fund_metadata(result)))
547 }
548
549 /// Fetch just the ETF's latest NAV (net asset value per share) via the v10
550 /// `quoteSummary` `summaryDetail` / `price` modules — the two that carry
551 /// `navPrice`. The Phase 4 daily NAV refresh calls this so the price-vs-NAV
552 /// premium behind the ETF quality read's tracking factor stays current,
553 /// without re-pulling the static fields the 30-day `fund_metadata` sweep
554 /// owns. `Ok(None)` is a clean empty (unknown symbol or no NAV reported);
555 /// gating responses (429 / 503 / 401 / 403) surface as the typed
556 /// [`RateLimited`], the same defensive set as `fund_metadata`.
557 pub async fn fund_nav(&self, ticker: &str) -> Result<Option<f64>> {
558 let Some(result) = self.quote_summary(ticker, "summaryDetail,price").await? else {
559 return Ok(None);
560 };
561 let sd = result.summary_detail.unwrap_or_default();
562 let price = result.price.unwrap_or_default();
563 Ok(sd.nav_price.or(price.nav_price).map(|v| v.0))
564 }
565
566 /// Shared v10 `quoteSummary` fetch: ensures the crumb is cached, builds
567 /// the URL with `&crumb=...`, parses gating responses (429 / 503 / 401 /
568 /// 403) as the typed [`RateLimited`], and treats Yahoo's "unknown
569 /// symbol" replies (404 or a `quoteSummary.error` body) as a clean
570 /// `Ok(None)`. A bare 401 on a previously-good crumb is retried once
571 /// with a fresh one — Yahoo rotates crumbs and this masks the rotation
572 /// from the endpoint guard.
573 async fn quote_summary(
574 &self,
575 ticker: &str,
576 modules: &str,
577 ) -> Result<Option<QuoteSummaryResult>> {
578 let sym = urlencoding::encode(&yahoo_symbol(ticker)).into_owned();
579 // First attempt with the cached crumb (may fetch one on the first
580 // call of the process).
581 match self.quote_summary_once(&sym, modules).await {
582 Ok(v) => Ok(v),
583 Err(e) => {
584 // A 401/403 on a request we sent a crumb with means the
585 // crumb expired. Drop it and try one more time so the
586 // caller sees a clean answer instead of a guard trip.
587 let retry = e.downcast_ref::<RateLimited>().is_some_and(|r| {
588 matches!(r.status, 401 | 403)
589 });
590 if retry {
591 self.invalidate_crumb();
592 return self.quote_summary_once(&sym, modules).await;
593 }
594 Err(e)
595 }
596 }
597 }
598
599 async fn quote_summary_once(
600 &self,
601 sym: &str,
602 modules: &str,
603 ) -> Result<Option<QuoteSummaryResult>> {
604 let crumb = self.ensure_crumb().await?;
605 let url = format!(
606 "https://query1.finance.yahoo.com/v10/finance/quoteSummary/{sym}\
607 ?modules={modules}&crumb={c}",
608 c = urlencoding::encode(&crumb),
609 );
610 let resp = self.client.get(&url).send().await?;
611 let status = resp.status();
612 if matches!(
613 status,
614 StatusCode::TOO_MANY_REQUESTS
615 | StatusCode::SERVICE_UNAVAILABLE
616 | StatusCode::UNAUTHORIZED
617 | StatusCode::FORBIDDEN
618 ) {
619 let retry_after_secs = resp
620 .headers()
621 .get(RETRY_AFTER)
622 .and_then(|v| v.to_str().ok())
623 .and_then(|s| s.trim().parse::<i64>().ok());
624 return Err(anyhow::Error::new(RateLimited {
625 status: status.as_u16(),
626 retry_after_secs,
627 }));
628 }
629 if status == StatusCode::NOT_FOUND {
630 return Ok(None);
631 }
632 let resp = resp.error_for_status()?;
633 let env: QuoteSummaryEnvelope = resp.json().await?;
634 if env.quote_summary.error.is_some() {
635 return Ok(None);
636 }
637 Ok(env
638 .quote_summary
639 .result
640 .and_then(|mut r| if r.is_empty() { None } else { Some(r.remove(0)) }))
641 }
642
643 /// Fetch the next-expected earnings date for `ticker` from Yahoo's
644 /// `quoteSummary.calendarEvents` module (Phase 25). One request to the
645 /// same v10 endpoint that already serves `fund_metadata`, asking only for
646 /// the calendar module — Yahoo's smallest reply on this endpoint.
647 ///
648 /// Returns `Ok(Some(epoch_ms))` when Yahoo has an upcoming earnings date,
649 /// `Ok(None)` when it knows the symbol but carries no date (Yahoo's
650 /// coverage is uneven on small caps, and a closely-watched name with a
651 /// just-passed print also briefly reads empty), or `Ok(None)` for an
652 /// unknown symbol (404 or `quoteSummary.error`). Gating responses (429 /
653 /// 503 / 401 / 403) surface as the typed [`RateLimited`], same defensive
654 /// set as `fund_metadata`.
655 pub async fn earnings_calendar(&self, ticker: &str) -> Result<Option<i64>> {
656 let Some(result) = self.quote_summary(ticker, "calendarEvents").await? else {
657 return Ok(None);
658 };
659 // `earningsDate` is an array; Yahoo populates 1 or 2 entries — the
660 // confirmed date, or a confirmed/estimated pair. The earliest one
661 // is the upcoming print. Future events only: a date in the past
662 // means Yahoo has not yet rolled it forward, so we ignore it.
663 let now_secs = chrono::Utc::now().timestamp();
664 let next_secs = result
665 .calendar_events
666 .and_then(|c| c.earnings)
667 .and_then(|e| {
668 e.earnings_date
669 .into_iter()
670 .filter_map(|d| Some(d.0 as i64))
671 .filter(|s| *s >= now_secs)
672 .min()
673 });
674 Ok(next_secs.map(|s| s * 1000))
675 }
676
677 /// Fetch a stock's sector and industry classification from Yahoo's
678 /// `quoteSummary.assetProfile` module (Phase 15). One request to the same
679 /// v10 endpoint that serves `fund_metadata` and `earnings_calendar`,
680 /// asking only for the `assetProfile` module — Yahoo's smallest reply
681 /// for this concern.
682 ///
683 /// Returns `Ok(Some(profile))` with whichever of `sector` / `industry`
684 /// Yahoo carries (a small cap with a partial profile leaves the absent
685 /// field `None`); `Ok(None)` when Yahoo cleanly does not know the
686 /// symbol (404 or `quoteSummary.error`); gating responses (429 / 503 /
687 /// 401 / 403) surface as the typed [`RateLimited`] so the endpoint
688 /// guard trips at once.
689 pub async fn asset_profile(&self, ticker: &str) -> Result<Option<AssetProfile>> {
690 let Some(result) = self.quote_summary(ticker, "assetProfile").await? else {
691 return Ok(None);
692 };
693 let ap = result.asset_profile.unwrap_or_default();
694 let trim = |s: Option<String>| s.map(|x| x.trim().to_string()).filter(|x| !x.is_empty());
695 Ok(Some(AssetProfile {
696 sector: trim(ap.sector),
697 industry: trim(ap.industry),
698 }))
699 }
700
701 /// Identify a symbol: validate it exists on Yahoo and return its name,
702 /// kind, exchange and currency, alongside the quote the same request
703 /// carried. Used by the Phase 9 add-symbol flow.
704 pub async fn lookup(&self, ticker: &str) -> Result<SymbolLookup> {
705 let Some(result) = self.fetch_chart(ticker).await? else {
706 return Ok(SymbolLookup::Unknown);
707 };
708 // Derive identity from `meta` before the result is consumed for the
709 // quote. An instrument type we do not model is a clean rejection.
710 let Some(info) = symbol_info(ticker, &result.meta) else {
711 let raw = result.meta.instrument_type.clone().unwrap_or_default();
712 return Ok(SymbolLookup::Unsupported(raw));
713 };
714 let data = chart_to_quote_data(ticker, result)?;
715 Ok(SymbolLookup::Found { info, data })
716 }
717}
718
719// ── v10 quoteSummary response (Phase 28) ───────────────────────────────────
720//
721// Yahoo wraps most numeric fields as `{"raw": ..., "fmt": "..."}`. A small
722// `RawF64` / `RawI64` carrier lets one serde derive handle every field of
723// that shape; an absent field, an unparsable one, or one missing the inner
724// `raw` is None — Yahoo's coverage is uneven and a partial snapshot is
725// still useful, so the parser keeps what it has rather than failing whole.
726
727#[derive(Deserialize)]
728struct QuoteSummaryEnvelope {
729 #[serde(rename = "quoteSummary")]
730 quote_summary: QuoteSummary,
731}
732
733#[derive(Deserialize)]
734struct QuoteSummary {
735 result: Option<Vec<QuoteSummaryResult>>,
736 /// Non-null on a logical failure (e.g. an unknown symbol — Yahoo returns
737 /// `200 OK` with an `error` body, not a 404).
738 error: Option<serde_json::Value>,
739}
740
741/// One module bag from the `quoteSummary` response. Every module is optional
742/// — Yahoo silently drops a module it does not recognise for this symbol
743/// rather than failing the whole request.
744#[derive(Default, Deserialize)]
745#[serde(rename_all = "camelCase", default)]
746struct QuoteSummaryResult {
747 fund_profile: Option<FundProfileModule>,
748 default_key_statistics: Option<DefaultKeyStatisticsModule>,
749 summary_detail: Option<SummaryDetailModule>,
750 price: Option<PriceModule>,
751 asset_profile: Option<AssetProfileModule>,
752 /// `calendarEvents` (Phase 25) — the upcoming earnings date and ex-div
753 /// date a v10 request can carry alongside the fund modules.
754 calendar_events: Option<CalendarEventsModule>,
755}
756
757#[derive(Default, Deserialize)]
758#[serde(rename_all = "camelCase", default)]
759struct CalendarEventsModule {
760 earnings: Option<CalendarEarnings>,
761}
762
763#[derive(Default, Deserialize)]
764#[serde(rename_all = "camelCase", default)]
765struct CalendarEarnings {
766 /// Yahoo emits 1 or 2 `RawF64` entries (Unix seconds), the confirmed
767 /// upcoming date or a confirmed/estimated pair.
768 earnings_date: Vec<RawF64>,
769}
770
771#[derive(Default, Deserialize)]
772#[serde(rename_all = "camelCase", default)]
773struct FundProfileModule {
774 family: Option<String>,
775 category_name: Option<String>,
776 fees_expenses_investment: Option<FeesExpensesInvestment>,
777}
778
779#[derive(Default, Deserialize)]
780#[serde(rename_all = "camelCase", default)]
781struct FeesExpensesInvestment {
782 annual_report_expense_ratio: Option<RawF64>,
783}
784
785#[derive(Default, Deserialize)]
786#[serde(rename_all = "camelCase", default)]
787struct DefaultKeyStatisticsModule {
788 /// Yahoo timestamps inception in either the unix-seconds `{raw, fmt}`
789 /// shape or, on some funds, an `fmt`-only ISO-ish string. Accept both.
790 fund_inception_date: Option<RawF64>,
791 #[serde(rename = "yield")]
792 yield_pct: Option<RawF64>,
793}
794
795#[derive(Default, Deserialize)]
796#[serde(rename_all = "camelCase", default)]
797struct SummaryDetailModule {
798 #[serde(rename = "yield")]
799 yield_pct: Option<RawF64>,
800 trailing_annual_dividend_yield: Option<RawF64>,
801 nav_price: Option<RawF64>,
802}
803
804#[derive(Default, Deserialize)]
805#[serde(rename_all = "camelCase", default)]
806struct PriceModule {
807 nav_price: Option<RawF64>,
808 first_trade_date_milliseconds: Option<RawF64>,
809}
810
811#[derive(Default, Deserialize)]
812#[serde(rename_all = "camelCase", default)]
813struct AssetProfileModule {
814 long_business_summary: Option<String>,
815 /// GICS-style sector ("Technology"). Stocks only; Yahoo leaves this
816 /// blank or omits the module entirely on ETFs / indexes / funds.
817 sector: Option<String>,
818 /// GICS-style industry ("Consumer Electronics"). See `sector`.
819 industry: Option<String>,
820}
821
822/// Yahoo's `{ "raw": ..., "fmt": "..." }` numeric carrier. Deserialises from
823/// either form: the wrapped object, a bare number, or a string that parses
824/// as a number. Missing or unparsable -> `None`.
825#[derive(Debug, Clone, Copy)]
826struct RawF64(f64);
827
828impl<'de> serde::Deserialize<'de> for RawF64 {
829 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Self, D::Error> {
830 #[derive(Deserialize)]
831 struct Wrap {
832 raw: Option<f64>,
833 }
834 // `untagged` lets serde try each variant in order; the first one to
835 // deserialise cleanly wins.
836 #[derive(Deserialize)]
837 #[serde(untagged)]
838 enum Any {
839 Wrapped(Wrap),
840 Bare(f64),
841 Str(String),
842 }
843 let any = Any::deserialize(d)?;
844 let v = match any {
845 Any::Wrapped(w) => w.raw,
846 Any::Bare(v) => Some(v),
847 Any::Str(s) => s.trim().parse().ok(),
848 };
849 v.map(RawF64).ok_or_else(|| {
850 // Yahoo sometimes serves `{}` for a missing field; surface as a
851 // deserialiser error so serde's outer Option<RawF64> on each
852 // field captures it as None rather than failing the whole parse.
853 serde::de::Error::custom("missing raw")
854 })
855 }
856}
857
858/// Build a [`FundMetadata`] from one parsed `quoteSummary` result. Every
859/// field is best-effort: a missing module or field just leaves its slot
860/// `None` rather than rejecting the row.
861fn parse_fund_metadata(r: QuoteSummaryResult) -> FundMetadata {
862 let fp = r.fund_profile.unwrap_or_default();
863 let dks = r.default_key_statistics.unwrap_or_default();
864 let sd = r.summary_detail.unwrap_or_default();
865 let price = r.price.unwrap_or_default();
866 let ap = r.asset_profile.unwrap_or_default();
867
868 let expense_ratio = fp
869 .fees_expenses_investment
870 .and_then(|f| f.annual_report_expense_ratio)
871 .map(|v| v.0);
872 // `summaryDetail.yield` is the live figure; `defaultKeyStatistics.yield`
873 // is the same number on most funds but missing on some, so fall back.
874 let yield_pct = sd
875 .yield_pct
876 .or(dks.yield_pct)
877 .map(|v| v.0);
878 let trailing_yield_pct = sd.trailing_annual_dividend_yield.map(|v| v.0);
879 let nav_price = sd.nav_price.or(price.nav_price).map(|v| v.0);
880 // Inception comes in two shapes: a unix-seconds carrier (older API),
881 // or — on some funds — a unix-ms carrier from `price`. Normalise to a
882 // `YYYY-MM-DD` date in UTC; the inception itself is day-precision.
883 let inception_date = dks
884 .fund_inception_date
885 .map(|v| v.0 as i64)
886 .or_else(|| {
887 price
888 .first_trade_date_milliseconds
889 // Heuristic: a value > 10^11 is ms, else seconds. ETF
890 // inceptions are post-1989 so both shapes are plausible.
891 .map(|v| {
892 let n = v.0 as i64;
893 if n.abs() > 100_000_000_000 { n / 1000 } else { n }
894 })
895 })
896 .and_then(|secs| chrono::DateTime::from_timestamp(secs, 0))
897 .map(|dt| dt.format("%Y-%m-%d").to_string());
898
899 let non_empty = |s: Option<String>| s.filter(|x| !x.trim().is_empty());
900 FundMetadata {
901 expense_ratio,
902 yield_pct,
903 trailing_yield_pct,
904 nav_price,
905 inception_date,
906 category: non_empty(fp.category_name),
907 fund_family: non_empty(fp.family),
908 strategy_summary: non_empty(ap.long_business_summary),
909 }
910}
911
912/// Build a `SymbolInfo` from a chart `meta`. `None` for an instrument type
913/// this app does not model yet (currencies, crypto, ...).
914fn symbol_info(ticker: &str, meta: &Meta) -> Option<SymbolInfo> {
915 let kind = match meta
916 .instrument_type
917 .as_deref()
918 .map(str::to_uppercase)
919 .as_deref()
920 {
921 Some("EQUITY") => "stock",
922 Some("ETF") | Some("MUTUALFUND") => "etf",
923 Some("INDEX") => "index",
924 Some("FUTURE") => "future",
925 // No instrument type at all: fall back to the ticker's shape — a `^`
926 // prefix is an index, a Yahoo `=F` suffix a future, else a stock.
927 None => {
928 if ticker.starts_with('^') {
929 "index"
930 } else if ticker.ends_with("=F") {
931 "future"
932 } else {
933 "stock"
934 }
935 }
936 // A type we do not model yet (CURRENCY, CRYPTOCURRENCY, ...).
937 Some(_) => return None,
938 };
939 let non_empty = |s: &String| !s.trim().is_empty();
940 let name = meta
941 .long_name
942 .clone()
943 .or_else(|| meta.short_name.clone())
944 .filter(non_empty)
945 .unwrap_or_else(|| ticker.to_string());
946 let exchange = meta
947 .full_exchange_name
948 .clone()
949 .or_else(|| meta.exchange_name.clone())
950 .filter(non_empty);
951 let currency = meta
952 .currency
953 .clone()
954 .filter(non_empty)
955 .unwrap_or_else(|| "USD".to_string());
956 Some(SymbolInfo {
957 name,
958 kind: kind.to_string(),
959 exchange,
960 currency,
961 })
962}
963
964/// Turn a parsed chart result into a `QuoteData` (live quote + intraday bars).
965fn chart_to_quote_data(ticker: &str, result: ChartResult) -> Result<QuoteData> {
966 let meta = result.meta;
967 let price = meta
968 .regular_market_price
969 .ok_or_else(|| anyhow!("yahoo quote for {ticker} carried no price"))?;
970 let quote = Quote {
971 price,
972 prev_close: meta.previous_close.or(meta.chart_previous_close),
973 open: meta.regular_market_open,
974 day_high: meta.regular_market_day_high,
975 day_low: meta.regular_market_day_low,
976 volume: meta.regular_market_volume,
977 market_state: meta.market_state,
978 source_time: meta.regular_market_time.map(|t| t * 1000),
979 };
980
981 // The day's intraday bars, zipping the timestamp column against the OHLCV
982 // columns. A bar missing any OHLC cell is skipped.
983 let mut bars = Vec::new();
984 if let (Some(ts), Some(o)) = (result.timestamp, result.indicators.quote.into_iter().next()) {
985 for (i, &t) in ts.iter().enumerate() {
986 let cell = |v: &[Option<f64>]| v.get(i).copied().flatten();
987 let (Some(open), Some(high), Some(low), Some(close)) =
988 (cell(&o.open), cell(&o.high), cell(&o.low), cell(&o.close))
989 else {
990 continue;
991 };
992 let volume = o.volume.get(i).copied().flatten().unwrap_or(0);
993 bars.push(IntradayBar {
994 ts: t * 1000,
995 open,
996 high,
997 low,
998 close,
999 volume,
1000 });
1001 }
1002 }
1003
1004 Ok(QuoteData { quote, bars })
1005}
1006
1007/// True when a supposedly-daily chart response is actually coarser than daily.
1008///
1009/// Yahoo silently downsamples `interval=1d&range=max` to monthly / quarterly
1010/// bars for some index and futures symbols (^RUT, ^VIX, the `=F` futures),
1011/// ignoring the requested interval. Detect it from the *median* spacing of the
1012/// returned timestamps: the *median* gap of a genuine daily series is 1 day
1013/// (consecutive trading days; only weekends and holidays stretch it to 3-4),
1014/// while a weekly series medians ~7 days and a monthly one ~30. A median over
1015/// four days therefore means the series is coarser than daily. Fewer than three
1016/// bars carries no usable spacing, so it is treated as fine.
1017fn is_downsampled(result: &ChartResult) -> bool {
1018 let Some(ts) = result.timestamp.as_ref() else {
1019 return false;
1020 };
1021 if ts.len() < 3 {
1022 return false;
1023 }
1024 let mut gaps: Vec<i64> = ts.windows(2).map(|w| w[1] - w[0]).collect();
1025 gaps.sort_unstable();
1026 let median = gaps[gaps.len() / 2];
1027 median > 4 * 86_400
1028}
1029
1030/// Turn a parsed `interval=1d` chart result into daily bars, oldest first.
1031///
1032/// Yahoo timestamps each daily bar at the start of the trading day in UTC
1033/// seconds; adding the exchange's `gmtoffset` before formatting yields the
1034/// local trading date (so a bar that starts 14:30 UTC reads as the right ET
1035/// day). A bar missing any of open/high/low/close is skipped.
1036fn chart_to_daily(result: ChartResult) -> Vec<DailyBar> {
1037 let off = result.meta.gmtoffset.unwrap_or(0);
1038 let (Some(ts), Some(o)) = (result.timestamp, result.indicators.quote.into_iter().next()) else {
1039 return Vec::new();
1040 };
1041 let mut out = Vec::with_capacity(ts.len());
1042 for (i, &t) in ts.iter().enumerate() {
1043 let cell = |v: &[Option<f64>]| v.get(i).copied().flatten();
1044 let (Some(open), Some(high), Some(low), Some(close)) =
1045 (cell(&o.open), cell(&o.high), cell(&o.low), cell(&o.close))
1046 else {
1047 continue;
1048 };
1049 let Some(d) = chrono::DateTime::from_timestamp(t + off, 0)
1050 .map(|dt| dt.format("%Y-%m-%d").to_string())
1051 else {
1052 continue;
1053 };
1054 let volume = o.volume.get(i).copied().flatten().unwrap_or(0);
1055 out.push(DailyBar {
1056 d,
1057 open,
1058 high,
1059 low,
1060 close,
1061 volume,
1062 });
1063 }
1064 out
1065}
1066
1067#[async_trait]
1068impl QuoteProvider for YahooProvider {
1069 async fn quote(&self, ticker: &str) -> Result<QuoteData> {
1070 match self.fetch_chart(ticker).await? {
1071 Some(result) => chart_to_quote_data(ticker, result),
1072 None => Err(anyhow!("yahoo returned no chart result for {ticker}")),
1073 }
1074 }
1075}
1076
1077impl YahooProvider {
1078 /// Fetch a wider intraday window (e.g. `range=5d`) of 15-minute bars in one
1079 /// request, used to backfill the whole trading week for the end-of-week
1080 /// dashboard view. Same shape as [`Self::quote`] (live quote + bars); callers
1081 /// store only the bars.
1082 pub async fn intraday_window(&self, ticker: &str, range: &str) -> Result<QuoteData> {
1083 match self.fetch_chart_range(ticker, range).await? {
1084 Some(result) => chart_to_quote_data(ticker, result),
1085 None => Err(anyhow!("yahoo returned no chart result for {ticker}")),
1086 }
1087 }
1088}
1089
1090#[async_trait]
1091impl HistoryProvider for YahooProvider {
1092 fn name(&self) -> &'static str {
1093 "yahoo"
1094 }
1095
1096 async fn daily(&self, ticker: &str, since: Option<&str>) -> Result<Vec<DailyBar>> {
1097 // An unknown / historyless symbol returns an empty vec (a clean empty,
1098 // not a guard failure) — same contract the Stooq provider had.
1099 match self.fetch_daily(ticker, since).await? {
1100 Some(result) => Ok(chart_to_daily(result)),
1101 None => Ok(Vec::new()),
1102 }
1103 }
1104}