Single-binary self-hosted market watcher for stocks, ETFs, indexes, and futures: live charts, key stats, fundamentals, SEC filings, and SSE streaming.
axumdockerfinancerustself-hostedsqlitestocksvite
1//! Company fundamentals and filing history from SEC EDGAR.
2//!
3//! Three endpoints, no API key (SEC asks only for an identifying User-Agent,
4//! which `build_sec_client` sets):
5//!
6//! - `https://www.sec.gov/files/company_tickers.json`: the whole-market
7//! ticker -> CIK map, fetched once to fill in `symbols.cik`.
8//! - `https://data.sec.gov/api/xbrl/companyfacts/CIK##########.json`: every
9//! XBRL fact a company has reported.
10//! - `https://data.sec.gov/submissions/CIK##########.json`: its filing
11//! history.
12//!
13//! The XBRL `companyfacts` payload is the awkward one. A company reports the
14//! same metric under different us-gaap *concepts* across accounting eras (e.g.
15//! revenue moved to `RevenueFromContractWithCustomerExcludingAssessedTax`), and
16//! each value carries year-to-date *and* discrete-period durations. `facts`
17//! normalises that: it merges the candidate concepts for each of our metrics
18//! and keeps only the clean full-year and discrete-quarter figures (see
19//! `classify`).
20
21use std::collections::HashMap;
22
23use anyhow::{anyhow, Result};
24use async_trait::async_trait;
25use chrono::{Datelike, NaiveDate};
26use reqwest::{header::RETRY_AFTER, StatusCode};
27use serde::Deserialize;
28
29use quick_xml::events::{BytesStart, Event};
30use quick_xml::Reader;
31
32use crate::providers::{
33 Fact, FilingRecord, FundFilings, FundHolding, FundId, FundShape, FundamentalsProvider,
34 OwnershipFiling, OwnershipPerson, PortfolioData, RateLimited,
35};
36
37/// Fundamentals and filings from SEC EDGAR.
38pub struct SecProvider {
39 /// A client whose User-Agent carries our contact email (see
40 /// `http::build_sec_client`).
41 client: reqwest::Client,
42}
43
44impl SecProvider {
45 pub fn new(client: reqwest::Client) -> Self {
46 Self { client }
47 }
48
49 /// Send one GET. `Ok(None)` means HTTP 404: a valid "this company has no
50 /// such resource" answer (some symbols simply have no XBRL facts), not a
51 /// failure, so it must not feed the circuit breaker. A 429/503 surfaces as
52 /// the typed `RateLimited` the guard trips on at once.
53 async fn get(&self, url: &str) -> Result<Option<reqwest::Response>> {
54 let resp = self.client.get(url).send().await?;
55 let status = resp.status();
56 if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::SERVICE_UNAVAILABLE {
57 let retry_after_secs = resp
58 .headers()
59 .get(RETRY_AFTER)
60 .and_then(|v| v.to_str().ok())
61 .and_then(|s| s.trim().parse::<i64>().ok());
62 return Err(anyhow::Error::new(RateLimited {
63 status: status.as_u16(),
64 retry_after_secs,
65 }));
66 }
67 if status == StatusCode::NOT_FOUND {
68 return Ok(None);
69 }
70 Ok(Some(resp.error_for_status()?))
71 }
72}
73
74/// Normalise a ticker to bare uppercase alphanumerics so our universe's
75/// `BRK.B` matches EDGAR's `BRK-B` and Stooq-style symbols line up too.
76pub fn normalize_ticker(ticker: &str) -> String {
77 ticker
78 .chars()
79 .filter(|c| c.is_ascii_alphanumeric())
80 .collect::<String>()
81 .to_uppercase()
82}
83
84// ── company_tickers.json ───────────────────────────────────────────────────
85
86#[derive(Deserialize)]
87struct TickerEntry {
88 cik_str: i64,
89 ticker: String,
90}
91
92// ── companyfacts ───────────────────────────────────────────────────────────
93
94#[derive(Deserialize)]
95struct CompanyFacts {
96 #[serde(default)]
97 facts: FactNamespaces,
98}
99
100#[derive(Default, Deserialize)]
101struct FactNamespaces {
102 /// The us-gaap taxonomy carries every metric we read. `dei` (entity
103 /// identifiers) is ignored.
104 #[serde(rename = "us-gaap", default)]
105 us_gaap: HashMap<String, Concept>,
106}
107
108#[derive(Deserialize)]
109struct Concept {
110 /// Keyed by XBRL unit (`USD`, `USD/shares`, `shares`, ...).
111 #[serde(default)]
112 units: HashMap<String, Vec<UnitEntry>>,
113}
114
115/// One reported value of a concept. `start` is present only for *duration*
116/// facts (income-statement items); *instantaneous* facts (balance-sheet items)
117/// carry only `end`.
118///
119/// The `fy` (fiscal year) field is deliberately not read: companyfacts tags a
120/// fact with the fiscal year of the *filing* it was drawn from, so a prior
121/// year shown as a comparative in a later 10-K carries that later filing's
122/// `fy`. The fiscal year is taken from the `end` date instead (see `classify`).
123#[derive(Deserialize)]
124struct UnitEntry {
125 start: Option<String>,
126 end: String,
127 val: f64,
128 /// Fiscal period: `FY`, `Q1`, `Q2`, `Q3` (`Q4` appears rarely).
129 fp: Option<String>,
130 form: Option<String>,
131 filed: Option<String>,
132}
133
134/// Our canonical metric -> the us-gaap concepts that can carry it. All listed
135/// concepts are merged, so a company that changed concepts across eras still
136/// gets a continuous series; a later filing's restated value wins ties (see
137/// the `filed` comparison in `facts`).
138const METRIC_CONCEPTS: &[(&str, &[&str])] = &[
139 (
140 "revenue",
141 &[
142 "RevenueFromContractWithCustomerExcludingAssessedTax",
143 "Revenues",
144 "RevenueFromContractWithCustomerIncludingAssessedTax",
145 "SalesRevenueNet",
146 ],
147 ),
148 ("net_income", &["NetIncomeLoss", "ProfitLoss"]),
149 (
150 "eps_diluted",
151 &["EarningsPerShareDiluted", "EarningsPerShareBasicAndDiluted"],
152 ),
153 (
154 "shares_diluted",
155 &["WeightedAverageNumberOfDilutedSharesOutstanding"],
156 ),
157 (
158 "dividends_per_share",
159 &[
160 "CommonStockDividendsPerShareDeclared",
161 "CommonStockDividendsPerShareCashPaid",
162 ],
163 ),
164 ("assets", &["Assets"]),
165 ("liabilities", &["Liabilities"]),
166 (
167 "equity",
168 &[
169 "StockholdersEquity",
170 "StockholdersEquityIncludingPortionAttributableToNoncontrollingInterest",
171 ],
172 ),
173 ("assets_current", &["AssetsCurrent"]),
174 ("liabilities_current", &["LiabilitiesCurrent"]),
175];
176
177/// How many fiscal years of annual and quarterly history to keep. Older
178/// figures are dropped at parse time so `fundamentals` stays small.
179const ANNUAL_YEARS: i64 = 6;
180const QUARTERLY_YEARS: i64 = 3;
181
182/// `Q3` -> `3`. `None` for anything that is not a `Q1`..`Q4` label.
183fn quarter_num(fp: &str) -> Option<i64> {
184 match fp {
185 "Q1" => Some(1),
186 "Q2" => Some(2),
187 "Q3" => Some(3),
188 "Q4" => Some(4),
189 _ => None,
190 }
191}
192
193/// The fiscal year a period ending on `end` belongs to, for a company whose
194/// fiscal year ends in month `fye_month`.
195///
196/// A period that ends *after* the fiscal-year-end month falls in the next
197/// fiscal year: e.g. an October-to-December quarter of a company with a
198/// September fiscal-year end is Q1 of the *following* fiscal year, even though
199/// it ends in the same calendar year as that year-end.
200fn fiscal_year_of(end: NaiveDate, fye_month: u32) -> i64 {
201 let y = end.year() as i64;
202 if end.month() > fye_month {
203 y + 1
204 } else {
205 y
206 }
207}
208
209/// The calendar month a company's fiscal year ends, taken as the most common
210/// end month across its annual (full-year duration) facts. Defaults to 12 (a
211/// calendar fiscal year) when none can be determined.
212fn fiscal_year_end_month(body: &CompanyFacts) -> u32 {
213 let mut counts = [0u32; 13]; // indices 1..=12
214 for concept in body.facts.us_gaap.values() {
215 for entries in concept.units.values() {
216 for e in entries {
217 if e.fp.as_deref() != Some("FY") {
218 continue;
219 }
220 let (Some(start), Ok(end)) = (
221 e.start.as_deref(),
222 NaiveDate::parse_from_str(&e.end, "%Y-%m-%d"),
223 ) else {
224 continue;
225 };
226 let Ok(start) = NaiveDate::parse_from_str(start, "%Y-%m-%d") else {
227 continue;
228 };
229 if (330..=400).contains(&(end - start).num_days()) {
230 counts[end.month() as usize] += 1;
231 }
232 }
233 }
234 }
235 counts
236 .iter()
237 .enumerate()
238 .skip(1)
239 .max_by_key(|(_, &c)| c)
240 .filter(|(_, &c)| c > 0)
241 .map(|(m, _)| m as u32)
242 .unwrap_or(12)
243}
244
245/// Classify one XBRL value into the fiscal period it cleanly represents, or
246/// `None` to drop it. Returns `(period_label, fiscal_year, fiscal_qtr)`.
247///
248/// The fiscal year comes from the `end` date and the company's fiscal-year-end
249/// month (see `fiscal_year_of`), never the `fy` field (see `UnitEntry`). XBRL
250/// is otherwise noisy: a concept carries discrete-quarter values, full-year
251/// values, *and* year-to-date roll-ups (6- and 9-month spans). This keeps only:
252/// - **full years**: a duration fact spanning ~a year with `fp == FY`;
253/// - **discrete quarters**: a duration fact spanning ~a quarter;
254/// - **year-end balances**: an instantaneous fact with `fp == FY`.
255///
256/// Quarterly *balance-sheet* (instantaneous) figures are deliberately not
257/// collected: a 10-Q tags its prior-year-end comparative with the filing's own
258/// quarter, which would mislabel a year-end snapshot as a quarter.
259fn classify(e: &UnitEntry, fye_month: u32) -> Option<(String, i64, Option<i64>)> {
260 let end = NaiveDate::parse_from_str(&e.end, "%Y-%m-%d").ok()?;
261 let fiscal_year = fiscal_year_of(end, fye_month);
262 let fp = e.fp.as_deref().unwrap_or("");
263
264 if let Some(start) = e.start.as_deref() {
265 // Duration fact (an income-statement flow). The span length tells a
266 // discrete quarter from a full year and from a year-to-date roll-up.
267 let start = NaiveDate::parse_from_str(start, "%Y-%m-%d").ok()?;
268 let days = (end - start).num_days();
269 if (330..=400).contains(&days) && fp == "FY" {
270 Some((format!("FY{fiscal_year}"), fiscal_year, None))
271 } else if (80..=100).contains(&days) {
272 let q = quarter_num(fp)?;
273 Some((format!("Q{q}-{fiscal_year}"), fiscal_year, Some(q)))
274 } else {
275 None
276 }
277 } else {
278 // Instantaneous fact (a balance-sheet snapshot): keep only the fiscal
279 // year-end, sourced from an annual report.
280 let form = e.form.as_deref().unwrap_or("");
281 let annual_form = form.is_empty()
282 || form.starts_with("10-K")
283 || form.starts_with("20-F")
284 || form.starts_with("40-F");
285 if fp == "FY" && annual_form {
286 Some((format!("FY{fiscal_year}"), fiscal_year, None))
287 } else {
288 None
289 }
290 }
291}
292
293/// One candidate us-gaap concept's classified series for a single metric, used
294/// while choosing which concept to pin in `facts`. `rank` is the concept's
295/// index in the metric's candidate list (lower = more preferred); `facts` maps
296/// each period label to the chosen fact for that period.
297struct ConceptSeries {
298 rank: usize,
299 facts: HashMap<String, Fact>,
300}
301
302impl ConceptSeries {
303 /// The newest `period_end` across the series, for picking the concept whose
304 /// data reaches furthest forward. Only built for non-empty series.
305 fn newest_end(&self) -> &str {
306 self.facts
307 .values()
308 .map(|f| f.period_end.as_str())
309 .max()
310 .unwrap_or("")
311 }
312}
313
314// ── submissions ────────────────────────────────────────────────────────────
315
316#[derive(Deserialize)]
317struct Submissions {
318 #[serde(default)]
319 filings: SubmissionFilings,
320}
321
322#[derive(Default, Deserialize)]
323struct SubmissionFilings {
324 #[serde(default)]
325 recent: RecentFilings,
326}
327
328/// EDGAR returns the filing history column-oriented: one parallel array per
329/// field, indexed by filing.
330#[derive(Default, Deserialize)]
331#[serde(rename_all = "camelCase")]
332struct RecentFilings {
333 #[serde(default)]
334 accession_number: Vec<String>,
335 #[serde(default)]
336 filing_date: Vec<String>,
337 #[serde(default)]
338 report_date: Vec<String>,
339 #[serde(default)]
340 form: Vec<String>,
341 #[serde(default)]
342 primary_document: Vec<String>,
343 #[serde(default)]
344 primary_doc_description: Vec<String>,
345 /// 8-K item codes, one comma-separated string per filing (empty otherwise).
346 #[serde(default)]
347 items: Vec<String>,
348}
349
350/// The filing forms worth showing a market-watcher: periodic reports (10-K,
351/// 10-Q, 8-K and the foreign-filer equivalents) and the annual proxy. The
352/// long tail (insider Form 4s, 13G/D ownership stakes, S-8 registrations) is
353/// dropped as noise.
354fn is_material_form(form: &str) -> bool {
355 const PREFIXES: [&str; 7] = ["10-K", "10-Q", "8-K", "20-F", "40-F", "6-K", "DEF 14A"];
356 PREFIXES.iter().any(|p| form.starts_with(p))
357}
358
359/// How many material filings to keep per company.
360const MAX_FILINGS: usize = 40;
361
362#[async_trait]
363impl FundamentalsProvider for SecProvider {
364 fn name(&self) -> &'static str {
365 "sec"
366 }
367
368 async fn cik_map(&self) -> Result<HashMap<String, String>> {
369 let url = "https://www.sec.gov/files/company_tickers.json";
370 let resp = self
371 .get(url)
372 .await?
373 .ok_or_else(|| anyhow!("sec company_tickers.json not found"))?;
374 // The file is a JSON object keyed by a row index; only the values matter.
375 let entries: HashMap<String, TickerEntry> = resp.json().await?;
376 let mut map = HashMap::with_capacity(entries.len());
377 for entry in entries.into_values() {
378 map.insert(
379 normalize_ticker(&entry.ticker),
380 format!("{:010}", entry.cik_str),
381 );
382 }
383 Ok(map)
384 }
385
386 async fn facts(&self, cik: &str) -> Result<Vec<Fact>> {
387 let url = format!("https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json");
388 let Some(resp) = self.get(&url).await? else {
389 return Ok(Vec::new()); // 404: company has no XBRL facts
390 };
391 let body: CompanyFacts = resp.json().await?;
392 Ok(select_facts(&body, chrono::Utc::now().year() as i64))
393 }
394
395 async fn filings(&self, cik: &str) -> Result<Vec<FilingRecord>> {
396 let url = format!("https://data.sec.gov/submissions/CIK{cik}.json");
397 let Some(resp) = self.get(&url).await? else {
398 return Ok(Vec::new()); // 404: no submission history
399 };
400 let body: Submissions = resp.json().await?;
401 Ok(select_filings(body, cik))
402 }
403}
404
405/// The pure core of [`SecProvider::facts`]: turn a parsed `companyfacts` body
406/// into our normalised facts, given the current year (passed in for test
407/// determinism). Pins one us-gaap concept per metric so the whole series shares
408/// one definition — see the loop below. Extracted so the concept-pinning logic
409/// is unit-testable without a network round trip.
410fn select_facts(body: &CompanyFacts, this_year: i64) -> Vec<Fact> {
411 let fye_month = fiscal_year_end_month(body);
412
413 // For each metric, pin ONE us-gaap concept and read the whole series
414 // from it. The candidate concepts for a metric are NOT interchangeable
415 // (e.g. `Revenues` can bundle items the contract-revenue tag excludes),
416 // so picking the newest-filed value per period across all candidates can
417 // silently source FY2024 from one concept and FY2023 from another, and
418 // the year-over-year growth then compares two different definitions. By
419 // emitting only the pinned concept's facts, every period of a metric
420 // shares one definition: a year the company did not report under the
421 // pinned concept is left absent (growth reads "no data") rather than
422 // computed against a mismatched figure. Restatements within the pinned
423 // concept are still collapsed to the latest `filed`. See `ConceptSeries`.
424 let mut out: Vec<Fact> = Vec::new();
425 for (metric, concepts) in METRIC_CONCEPTS {
426 let mut candidates: Vec<ConceptSeries> = Vec::new();
427 for (rank, concept) in concepts.iter().enumerate() {
428 let Some(concept_data) = body.facts.us_gaap.get(*concept) else {
429 continue;
430 };
431 // This concept's in-window series, deduped by period (a later
432 // filing's restated value wins, by `filed`).
433 let mut series: HashMap<String, Fact> = HashMap::new();
434 for (unit, entries) in &concept_data.units {
435 for e in entries {
436 let Some((period, fiscal_year, fiscal_qtr)) = classify(e, fye_month)
437 else {
438 continue;
439 };
440 // Drop anything older than the retention window.
441 let keep_since = if fiscal_qtr.is_some() {
442 this_year - QUARTERLY_YEARS
443 } else {
444 this_year - ANNUAL_YEARS
445 };
446 if fiscal_year < keep_since {
447 continue;
448 }
449 let newer = series.get(&period).map_or(true, |prev: &Fact| {
450 e.filed.as_deref().unwrap_or("")
451 > prev.filed_at.as_deref().unwrap_or("")
452 });
453 if newer {
454 series.insert(
455 period.clone(),
456 Fact {
457 metric: metric.to_string(),
458 period,
459 fiscal_year,
460 fiscal_qtr,
461 period_end: e.end.clone(),
462 value: e.val,
463 unit: Some(unit.clone()),
464 form: e.form.clone(),
465 filed_at: e.filed.clone(),
466 },
467 );
468 }
469 }
470 }
471 if !series.is_empty() {
472 candidates.push(ConceptSeries { rank, facts: series });
473 }
474 }
475 // Pin the best candidate: the concept whose series reaches the most
476 // recent period, then the one covering the most periods, then the
477 // earliest-listed (most-preferred) concept. Emit only its facts.
478 if let Some(best) = candidates.into_iter().max_by(|a, b| {
479 a.newest_end()
480 .cmp(b.newest_end())
481 .then_with(|| a.facts.len().cmp(&b.facts.len()))
482 .then_with(|| b.rank.cmp(&a.rank)) // lower rank (preferred) wins ties
483 }) {
484 out.extend(best.facts.into_values());
485 }
486 }
487
488 out
489}
490
491/// The pure core of [`SecProvider::filings`]: pick the material filings out of a
492/// parsed submissions body, newest first, capped at `MAX_FILINGS`. Extracted to
493/// mirror `select_facts` (the network fetch lives in the method).
494fn select_filings(body: Submissions, cik: &str) -> Vec<FilingRecord> {
495 let r = body.filings.recent;
496
497 // EDGAR pads the CIK to 10 digits; the Archives path uses it unpadded.
498 let cik_int = cik.trim_start_matches('0');
499
500 let mut out = Vec::new();
501 for i in 0..r.accession_number.len() {
502 let form = r.form.get(i).cloned().unwrap_or_default();
503 if !is_material_form(&form) {
504 continue;
505 }
506 let accession = r.accession_number[i].clone();
507 let filed_at = r.filing_date.get(i).cloned().unwrap_or_default();
508 if accession.is_empty() || filed_at.is_empty() {
509 continue;
510 }
511 let nodash = accession.replace('-', "");
512 let primary_doc = r
513 .primary_document
514 .get(i)
515 .filter(|s| !s.is_empty())
516 .cloned();
517 // Link straight to the primary document when EDGAR names one;
518 // otherwise to the filing index page.
519 let url = match &primary_doc {
520 Some(doc) => {
521 format!("https://www.sec.gov/Archives/edgar/data/{cik_int}/{nodash}/{doc}")
522 }
523 None => format!(
524 "https://www.sec.gov/Archives/edgar/data/{cik_int}/{nodash}/{accession}-index.htm"
525 ),
526 };
527 out.push(FilingRecord {
528 accession,
529 form,
530 filed_at,
531 period_of_report: r.report_date.get(i).filter(|s| !s.is_empty()).cloned(),
532 primary_doc,
533 url,
534 description: r
535 .primary_doc_description
536 .get(i)
537 .filter(|s| !s.is_empty())
538 .cloned(),
539 items: r.items.get(i).filter(|s| !s.is_empty()).cloned(),
540 });
541 if out.len() >= MAX_FILINGS {
542 break;
543 }
544 }
545 out
546}
547
548// ── ETF fund profiles: N-PORT holdings, AUM, filing history (Phase 18) ─────
549//
550// An ETF files as a registered fund, so its portfolio is not in the XBRL
551// `companyfacts` above — it is in quarterly N-PORT filings, one large XML per
552// fund. These methods are inherent to `SecProvider` rather than behind a
553// trait: N-PORT is wholly SEC-specific, with no second source to abstract
554// over. Each method makes exactly one HTTP request so the scheduler can keep
555// wrapping every call in the endpoint guard, as it does for `facts`/`filings`.
556
557/// Fund trusts that file with the SEC but are absent from
558/// `company_tickers_mf.json` (which is keyed on the series/class structure of
559/// open-end funds): the unit investment trusts (SPY, DIA) and the physical-
560/// commodity grantor trusts (GLD, SLV). Mapped straight to their registrant
561/// CIK, with no series id since each trust is a single fund.
562const FUND_FALLBACK: &[(&str, i64)] = &[
563 ("SPY", 884394),
564 ("DIA", 1041130),
565 ("GLD", 1222333),
566 ("SLV", 1330568),
567];
568
569/// How many of a fund's holdings to keep — the largest by weight. A bond
570/// aggregate fund holds thousands of positions; the page shows only the top.
571const TOP_HOLDINGS: usize = 25;
572
573/// How many of a fund's filings to keep for the page's filing list.
574const MAX_FUND_FILINGS: usize = 40;
575
576/// `company_tickers_mf.json`: a `fields` header plus row tuples of
577/// `(cik, seriesId, classId, symbol)`.
578#[derive(Deserialize)]
579struct MfFile {
580 data: Vec<(i64, String, String, String)>,
581}
582
583/// One filing parsed from a browse-edgar Atom feed.
584#[derive(Default)]
585struct AtomEntry {
586 form: String,
587 accession: String,
588 filed: String,
589 /// EDGAR filing-index page URL.
590 href: String,
591}
592
593/// One holding accumulated while streaming through an `<invstOrSec>` block.
594/// `issuer_cat` and `country` are captured for the sector / geography mixes
595/// (Phase 28) but not surfaced on [`FundHolding`] itself — only the
596/// aggregations are kept past parse time.
597#[derive(Default)]
598struct HoldingAcc {
599 name: String,
600 title: String,
601 pct: Option<f64>,
602 value: Option<f64>,
603 asset_cat: Option<String>,
604 /// N-PORT `<issuerCat>`, e.g. `CORP` / `GOVT` / `MUN` / `RF`. Meaningful
605 /// on bond and multi-sector funds; an equity ETF rolls up almost wholly
606 /// to `CORP`, so the sector panel is hidden in that degenerate case.
607 issuer_cat: Option<String>,
608 /// N-PORT `<invCountry>`, the issuer's ISO-3166-1 alpha-2 country code.
609 country: Option<String>,
610}
611
612impl HoldingAcc {
613 fn into_holding(self) -> FundHolding {
614 // Prefer `title` (the issue title): it is clean mixed-case, where the
615 // issuer `name` often arrives truncated and all-caps. Fall back to
616 // `name` for the rare holding that carried no title.
617 let name = if self.title.is_empty() {
618 self.name
619 } else {
620 self.title
621 };
622 FundHolding {
623 name,
624 pct: self.pct,
625 value_usd: self.value,
626 asset_cat: self.asset_cat,
627 }
628 }
629}
630
631impl SecProvider {
632 /// Ticker -> fund identity, from the SEC mutual-fund ticker file plus the
633 /// hardcoded fallback for fund trusts absent from it. Keys are normalised
634 /// like `cik_map`'s. One bulk request, fetched while some ETF lacks a CIK.
635 pub async fn fund_ticker_map(&self) -> Result<HashMap<String, FundId>> {
636 let url = "https://www.sec.gov/files/company_tickers_mf.json";
637 let resp = self
638 .get(url)
639 .await?
640 .ok_or_else(|| anyhow!("sec company_tickers_mf.json not found"))?;
641 let body: MfFile = resp.json().await?;
642 let mut map = HashMap::with_capacity(body.data.len() + FUND_FALLBACK.len());
643 for (cik, series_id, _class_id, symbol) in body.data {
644 map.entry(normalize_ticker(&symbol)).or_insert(FundId {
645 cik: format!("{cik:010}"),
646 series_id: Some(series_id),
647 });
648 }
649 for (ticker, cik) in FUND_FALLBACK {
650 map.entry(normalize_ticker(ticker)).or_insert(FundId {
651 cik: format!("{cik:010}"),
652 series_id: None,
653 });
654 }
655 Ok(map)
656 }
657
658 /// A fund's filing list, plus what the filing history says about its shape
659 /// (whether to read an N-PORT for holdings, or treat it as a commodity
660 /// trust). One browse-edgar request, keyed on the series id when the
661 /// registrant hosts several funds so a sibling fund's filings never leak in.
662 pub async fn fund_filings(&self, id: &FundId) -> Result<FundFilings> {
663 let key = id.series_id.as_deref().unwrap_or(&id.cik);
664 let url = format!(
665 "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={key}\
666 &type=&dateb=&owner=include&count=100&output=atom"
667 );
668 let resp = self
669 .get(&url)
670 .await?
671 .ok_or_else(|| anyhow!("edgar filing index for {key} not found"))?;
672 let bytes = resp.bytes().await?;
673 let entries = parse_edgar_atom(&bytes)?;
674
675 // Entries arrive newest-first. Read the fund's shape from the whole set
676 // before trimming the list to the material forms shown on the page.
677 let mut nport: Option<String> = None;
678 let mut has_ncen = false;
679 let mut has_10k = false;
680 for e in &entries {
681 if nport.is_none() && e.form.starts_with("NPORT-P") && !e.href.is_empty() {
682 nport = Some(e.href.clone());
683 }
684 has_ncen |= e.form.starts_with("N-CEN");
685 has_10k |= e.form.starts_with("10-K");
686 }
687 let shape = if let Some(nport_href) = nport {
688 FundShape::Portfolio { nport_href }
689 } else if has_10k && !has_ncen {
690 // Files 10-Ks and no fund-census report: a grantor trust holding a
691 // physical commodity rather than a securities portfolio.
692 FundShape::CommodityTrust
693 } else {
694 FundShape::Unknown
695 };
696
697 let filings = entries
698 .into_iter()
699 .filter(|e| is_material_fund_form(&e.form))
700 .take(MAX_FUND_FILINGS)
701 .map(|e| FilingRecord {
702 accession: e.accession,
703 form: e.form,
704 filed_at: e.filed,
705 period_of_report: None,
706 primary_doc: None,
707 url: e.href,
708 description: None,
709 items: None,
710 })
711 .collect();
712
713 Ok(FundFilings { filings, shape })
714 }
715
716 /// Parse one N-PORT filing into a portfolio snapshot: net assets, the
717 /// holdings (top slice by weight), the holding count, and the asset mix.
718 /// `index_href` is the filing's EDGAR index-page URL; the N-PORT XML sits
719 /// beside it in the same Archives directory. One request.
720 pub async fn fund_portfolio(&self, index_href: &str) -> Result<PortfolioData> {
721 // `.../data/{cik}/{nodash}/{accession}-index.htm` -> swap the index
722 // page for `primary_doc.xml` in the same directory.
723 let dir = index_href
724 .rsplit_once('/')
725 .map(|(d, _)| d)
726 .ok_or_else(|| anyhow!("malformed filing href {index_href}"))?;
727 let url = format!("{dir}/primary_doc.xml");
728 let resp = self
729 .get(&url)
730 .await?
731 .ok_or_else(|| anyhow!("N-PORT not found at {url}"))?;
732 let bytes = resp.bytes().await?;
733 parse_nport(&bytes)
734 }
735
736 /// The latest total-assets figure a company has reported, USD. Gives a
737 /// physical-commodity grantor trust (GLD, SLV) an AUM: those file 10-Ks,
738 /// not N-PORT, so `Assets` from their XBRL companyfacts stands in for net
739 /// assets. Unlike `facts`, this takes the single most recent value
740 /// regardless of fiscal period, so a mid-year 10-Q figure beats a stale
741 /// prior year-end. `None` when the company has no `Assets` concept.
742 pub async fn fund_aum(&self, cik: &str) -> Result<Option<f64>> {
743 let url = format!("https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json");
744 let Some(resp) = self.get(&url).await? else {
745 return Ok(None); // 404: no XBRL facts
746 };
747 let body: CompanyFacts = resp.json().await?;
748 let Some(assets) = body.facts.us_gaap.get("Assets") else {
749 return Ok(None);
750 };
751 // Newest by period-end date, ties broken by the later filing.
752 let mut best: Option<&UnitEntry> = None;
753 for entries in assets.units.values() {
754 for e in entries {
755 let newer = best.map_or(true, |b| {
756 (e.end.as_str(), e.filed.as_deref().unwrap_or(""))
757 > (b.end.as_str(), b.filed.as_deref().unwrap_or(""))
758 });
759 if newer {
760 best = Some(e);
761 }
762 }
763 }
764 Ok(best.map(|e| e.val))
765 }
766}
767
768/// Filing forms worth showing on a fund's page: its portfolio reports
769/// (N-PORT), the annual fund census and shareholder reports, the prospectus,
770/// and — for a commodity trust — the 10-K family it files instead.
771fn is_material_fund_form(form: &str) -> bool {
772 const PREFIXES: &[&str] = &[
773 "NPORT-P", "NPORT-EX", "N-CEN", "N-CSR", "485BPOS", "485APOS", "10-K", "10-Q", "8-K",
774 ];
775 PREFIXES.iter().any(|p| form.starts_with(p))
776}
777
778/// Map an N-PORT `assetCat` code to a human asset-class bucket for the mix.
779/// The codes are from the N-PORT technical schema; the long tail is "Other".
780fn asset_bucket(cat: &str) -> &'static str {
781 match cat {
782 "EC" | "EP" => "Equity",
783 "DBT" | "SF" => "Bonds",
784 "STIV" | "RA" => "Cash & equivalents",
785 "COMM" => "Commodities",
786 "RE" => "Real estate",
787 "LON" => "Loans",
788 c if c.starts_with("ABS") => "Bonds",
789 // Every other derivative category code begins `D` (DBT is matched above).
790 c if c.starts_with('D') => "Derivatives",
791 _ => "Other",
792 }
793}
794
795/// Read one attribute off a start tag as a `String`.
796fn attr_val(e: &BytesStart, key: &[u8]) -> Option<String> {
797 e.attributes()
798 .flatten()
799 .find(|a| a.key.as_ref() == key)
800 .and_then(|a| String::from_utf8(a.value.into_owned()).ok())
801}
802
803/// Set the current entry's form from a `<category term="..."/>` tag.
804fn set_form_from_category(e: &BytesStart, cur: &mut Option<AtomEntry>) {
805 if let (Some(c), Some(term)) = (cur.as_mut(), attr_val(e, b"term")) {
806 c.form = term;
807 }
808}
809
810/// Parse a browse-edgar Atom feed into its filing entries, newest first.
811fn parse_edgar_atom(xml: &[u8]) -> Result<Vec<AtomEntry>> {
812 let mut reader = Reader::from_reader(xml);
813 let mut buf = Vec::new();
814 let mut path: Vec<Vec<u8>> = Vec::new();
815 let mut entries = Vec::new();
816 let mut cur: Option<AtomEntry> = None;
817
818 loop {
819 match reader.read_event_into(&mut buf)? {
820 Event::Start(e) => {
821 let name = e.local_name().as_ref().to_vec();
822 if name == b"entry" {
823 cur = Some(AtomEntry::default());
824 }
825 if name == b"category" {
826 set_form_from_category(&e, &mut cur);
827 }
828 path.push(name);
829 }
830 // The form type usually rides a self-closing `<category term=".."/>`.
831 Event::Empty(e) => {
832 if e.local_name().as_ref() == b"category" {
833 set_form_from_category(&e, &mut cur);
834 }
835 }
836 Event::End(e) => {
837 if e.local_name().as_ref() == b"entry" {
838 if let Some(c) = cur.take() {
839 if !c.accession.is_empty() {
840 entries.push(c);
841 }
842 }
843 }
844 path.pop();
845 }
846 Event::Text(t) => {
847 let (Some(tag), Some(c)) = (path.last(), cur.as_mut()) else {
848 continue;
849 };
850 let raw = t.unescape().unwrap_or_default();
851 let txt = raw.trim();
852 if txt.is_empty() {
853 continue;
854 }
855 match tag.as_slice() {
856 b"accession-number" if c.accession.is_empty() => c.accession = txt.to_string(),
857 b"filing-date" if c.filed.is_empty() => c.filed = txt.to_string(),
858 b"filing-href" if c.href.is_empty() => c.href = txt.to_string(),
859 b"filing-type" if c.form.is_empty() => c.form = txt.to_string(),
860 _ => {}
861 }
862 }
863 Event::Eof => break,
864 _ => {}
865 }
866 buf.clear();
867 }
868 Ok(entries)
869}
870
871/// Stream-parse an N-PORT `primary_doc.xml` into a portfolio snapshot. The file
872/// can run to many megabytes for a bond fund's thousands of positions, so this
873/// walks events rather than building a DOM, keeping only the running totals
874/// and, at the end, the largest holdings.
875fn parse_nport(xml: &[u8]) -> Result<PortfolioData> {
876 let mut reader = Reader::from_reader(xml);
877 let mut buf = Vec::new();
878 // Stack of open element local names, so a leaf is read in context.
879 let mut path: Vec<Vec<u8>> = Vec::new();
880 let mut out = PortfolioData::default();
881 let mut all: Vec<FundHolding> = Vec::new();
882 // The holding currently being assembled, set while inside `<invstOrSec>`.
883 let mut cur: Option<HoldingAcc> = None;
884 // Running sector / geography aggregations: each holding's weight summed
885 // into its `issuerCat` / `invCountry` bucket as the parser closes it. A
886 // missing bucket on a holding contributes to an `"Unknown"` slot, which
887 // is dropped at the end.
888 let mut sector_acc: HashMap<String, f64> = HashMap::new();
889 let mut country_acc: HashMap<String, f64> = HashMap::new();
890
891 loop {
892 match reader.read_event_into(&mut buf)? {
893 Event::Start(e) => {
894 let name = e.local_name().as_ref().to_vec();
895 if name == b"invstOrSec" {
896 cur = Some(HoldingAcc::default());
897 }
898 path.push(name);
899 }
900 Event::End(e) => {
901 if e.local_name().as_ref() == b"invstOrSec" {
902 if let Some(h) = cur.take() {
903 let pct = h.pct.unwrap_or(0.0);
904 let sec = issuer_bucket(h.issuer_cat.as_deref());
905 let ctry = country_bucket(h.country.as_deref());
906 *sector_acc.entry(sec.to_string()).or_insert(0.0) += pct;
907 *country_acc.entry(ctry).or_insert(0.0) += pct;
908 all.push(h.into_holding());
909 }
910 }
911 path.pop();
912 }
913 Event::Text(t) => {
914 let Some(tag) = path.last() else {
915 continue;
916 };
917 let raw = t.unescape().unwrap_or_default();
918 let txt = raw.trim();
919 if txt.is_empty() {
920 continue;
921 }
922 let tag = tag.as_slice();
923 // Holding fields, captured only inside an `<invstOrSec>`.
924 // First-wins: the issuer-level value precedes any nested block.
925 if let Some(h) = cur.as_mut() {
926 match tag {
927 b"name" if h.name.is_empty() => h.name = txt.to_string(),
928 b"title" if h.title.is_empty() => h.title = txt.to_string(),
929 b"pctVal" if h.pct.is_none() => h.pct = txt.parse().ok(),
930 b"valUSD" if h.value.is_none() => h.value = txt.parse().ok(),
931 b"assetCat" if h.asset_cat.is_none() => {
932 h.asset_cat = Some(txt.to_string())
933 }
934 b"issuerCat" if h.issuer_cat.is_none() => {
935 h.issuer_cat = Some(txt.to_string())
936 }
937 b"invCountry" if h.country.is_none() => {
938 h.country = Some(txt.to_string())
939 }
940 _ => {}
941 }
942 }
943 // Fund-level fields, scoped by their parent element.
944 let parent = path.iter().rev().nth(1).map(Vec::as_slice);
945 match (parent, tag) {
946 (Some(b"fundInfo"), b"netAssets") if out.net_assets.is_none() => {
947 out.net_assets = txt.parse().ok();
948 }
949 (Some(b"fundInfo"), b"totAssets") if out.total_assets.is_none() => {
950 out.total_assets = txt.parse().ok();
951 }
952 (Some(b"genInfo"), b"repPdDate") if out.report_date.is_none() => {
953 out.report_date = Some(txt.to_string());
954 }
955 _ => {}
956 }
957 }
958 Event::Eof => break,
959 _ => {}
960 }
961 buf.clear();
962 }
963
964 out.holdings_count = all.len() as i64;
965
966 // Asset-class mix: each holding's weight summed into its bucket. Tiny
967 // residual buckets (rounding noise) are dropped.
968 let mut mix: HashMap<&'static str, f64> = HashMap::new();
969 for h in &all {
970 let bucket = h.asset_cat.as_deref().map_or("Other", asset_bucket);
971 *mix.entry(bucket).or_insert(0.0) += h.pct.unwrap_or(0.0);
972 }
973 let mut mix: Vec<(String, f64)> = mix
974 .into_iter()
975 .filter(|(_, p)| *p >= 0.05)
976 .map(|(b, p)| (b.to_string(), p))
977 .collect();
978 mix.sort_by(|a, b| b.1.total_cmp(&a.1));
979 out.asset_mix = mix;
980
981 // Largest holdings first; keep only the top slice.
982 all.sort_by(|a, b| b.pct.unwrap_or(0.0).total_cmp(&a.pct.unwrap_or(0.0)));
983 all.truncate(TOP_HOLDINGS);
984 out.top_holdings = all;
985
986 // Sector / geography mixes (Phase 28). Same shape as `asset_mix`: drop
987 // residual rounding-noise buckets, sort largest first. The sector panel
988 // hides itself in the route when only one bucket survives — that is the
989 // pure-equity-ETF degenerate case where everything rolls up to "Corporate"
990 // and the panel would be a single flat bar carrying no information.
991 let trim_and_sort = |acc: HashMap<String, f64>| -> Vec<(String, f64)> {
992 let mut v: Vec<(String, f64)> = acc
993 .into_iter()
994 .filter(|(_, p)| *p >= 0.05)
995 .collect();
996 v.sort_by(|a, b| b.1.total_cmp(&a.1));
997 v
998 };
999 out.sector_mix = trim_and_sort(sector_acc);
1000 out.geography_mix = trim_and_sort(country_acc);
1001
1002 Ok(out)
1003}
1004
1005/// Map an N-PORT `issuerCat` code to a human-readable sector bucket. The
1006/// codes are corporate / government / municipal / fund / etc. — meaningful
1007/// on bond and multi-sector funds; a pure-equity ETF rolls up almost wholly
1008/// to `Corporate`, which the route detects and hides.
1009fn issuer_bucket(cat: Option<&str>) -> &'static str {
1010 match cat.unwrap_or("") {
1011 "CORP" => "Corporate",
1012 "USGSE" => "US gov't-sponsored",
1013 "USGA" => "US government agency",
1014 "UST" => "US Treasury",
1015 "MUN" => "Municipal",
1016 "PF" => "Private fund",
1017 "RF" => "Registered fund",
1018 "ABS" => "Asset-backed",
1019 "" => "Unknown",
1020 _ => "Other",
1021 }
1022}
1023
1024/// Map an issuer country's ISO-3166-1 alpha-2 code to a display name. Only
1025/// the codes most commonly seen in US-listed ETF portfolios are spelled
1026/// out; the long tail falls through to the raw code, which still reads
1027/// usefully in the panel.
1028fn country_bucket(code: Option<&str>) -> String {
1029 let name = match code.unwrap_or("").to_ascii_uppercase().as_str() {
1030 "US" => "United States",
1031 "CA" => "Canada",
1032 "GB" => "United Kingdom",
1033 "DE" => "Germany",
1034 "FR" => "France",
1035 "CH" => "Switzerland",
1036 "NL" => "Netherlands",
1037 "IE" => "Ireland",
1038 "LU" => "Luxembourg",
1039 "IT" => "Italy",
1040 "ES" => "Spain",
1041 "SE" => "Sweden",
1042 "NO" => "Norway",
1043 "DK" => "Denmark",
1044 "FI" => "Finland",
1045 "BE" => "Belgium",
1046 "AT" => "Austria",
1047 "JP" => "Japan",
1048 "CN" => "China",
1049 "HK" => "Hong Kong",
1050 "TW" => "Taiwan",
1051 "KR" => "South Korea",
1052 "IN" => "India",
1053 "SG" => "Singapore",
1054 "AU" => "Australia",
1055 "NZ" => "New Zealand",
1056 "BR" => "Brazil",
1057 "MX" => "Mexico",
1058 "ZA" => "South Africa",
1059 "IL" => "Israel",
1060 "AE" => "United Arab Emirates",
1061 "BM" => "Bermuda",
1062 "KY" => "Cayman Islands",
1063 "" => "Unknown",
1064 other => return other.to_string(),
1065 };
1066 name.to_string()
1067}
1068
1069// ── company leadership: officers & board from Form 3/4/5 (Phase 14) ────────
1070//
1071// Every director and Section-16 officer of a company must file Form 3 (on
1072// becoming an insider) and Form 4 (on each trade); each ownership XML carries
1073// a structured `reportingOwnerRelationship`. The roster is built by parsing a
1074// window of a company's recent ownership filings. Like the fund methods above,
1075// each of these makes exactly one HTTP request so the scheduler wraps every
1076// call in the endpoint guard.
1077
1078/// Whether `form` is an ownership form — Form 3, 4 or 5, or an amendment.
1079fn is_ownership_form(form: &str) -> bool {
1080 matches!(form.trim_end_matches("/A"), "3" | "4" | "5")
1081}
1082
1083impl SecProvider {
1084 /// A company's recent Form 3/4/5 ownership filings, newest first, read from
1085 /// the same `submissions` JSON that backs `filings`. One request. Only
1086 /// filings whose primary document is an `.xml` are returned: `ownership_doc`
1087 /// parses that XML, and the rare legacy filing with a non-XML primary
1088 /// document has nothing to parse.
1089 pub async fn ownership_index(&self, cik: &str) -> Result<Vec<OwnershipFiling>> {
1090 let url = format!("https://data.sec.gov/submissions/CIK{cik}.json");
1091 let Some(resp) = self.get(&url).await? else {
1092 return Ok(Vec::new()); // 404: no submission history
1093 };
1094 let body: Submissions = resp.json().await?;
1095 let r = body.filings.recent;
1096
1097 let mut out = Vec::new();
1098 for i in 0..r.accession_number.len() {
1099 let form = r.form.get(i).cloned().unwrap_or_default();
1100 if !is_ownership_form(&form) {
1101 continue;
1102 }
1103 let accession = r.accession_number[i].clone();
1104 let filed_at = r.filing_date.get(i).cloned().unwrap_or_default();
1105 let primary_doc = r.primary_document.get(i).cloned().unwrap_or_default();
1106 if accession.is_empty() || filed_at.is_empty() || !primary_doc.ends_with(".xml") {
1107 continue;
1108 }
1109 out.push(OwnershipFiling {
1110 accession,
1111 filed_at,
1112 primary_doc,
1113 });
1114 }
1115 Ok(out)
1116 }
1117
1118 /// Parse one ownership filing's XML into its reporting people. One request.
1119 /// A filing names more than one reporting person only for the rare joint
1120 /// filing; usually the vec holds one. `Ok(Vec::new())` for a 404 or an
1121 /// unparseable document — neither is a circuit-breaker failure.
1122 pub async fn ownership_doc(
1123 &self,
1124 cik: &str,
1125 accession: &str,
1126 primary_doc: &str,
1127 ) -> Result<Vec<OwnershipPerson>> {
1128 // EDGAR pads the CIK to 10 digits; the Archives path uses it unpadded.
1129 let cik_int = cik.trim_start_matches('0');
1130 let nodash = accession.replace('-', "");
1131 // The submissions feed sometimes names the primary document as an
1132 // xsl-styled viewer path (`xslF345X05/foo.xml`); the raw XML this parses
1133 // sits at the bare filename in the accession directory.
1134 let doc = primary_doc.rsplit('/').next().unwrap_or(primary_doc);
1135 let url = format!("https://www.sec.gov/Archives/edgar/data/{cik_int}/{nodash}/{doc}");
1136 let Some(resp) = self.get(&url).await? else {
1137 return Ok(Vec::new()); // 404
1138 };
1139 let bytes = resp.bytes().await?;
1140 Ok(parse_ownership(&bytes))
1141 }
1142}
1143
1144/// Read an ownership-XML boolean. SEC writes these as `1`/`0`, very
1145/// occasionally `true`.
1146fn ownership_flag(txt: &str) -> bool {
1147 matches!(txt.trim(), "1" | "true" | "TRUE" | "Y")
1148}
1149
1150/// One reporting person accumulated while streaming an ownership XML.
1151#[derive(Default)]
1152struct OwnerAcc {
1153 name: String,
1154 is_director: bool,
1155 is_officer: bool,
1156 officer_title: String,
1157}
1158
1159/// Stream-parse an ownership Form 3/4/5 XML into its reporting people. Walks
1160/// the `<reportingOwner>` blocks, reading each owner's name and the
1161/// `reportingOwnerRelationship` flags. A malformed document yields whatever was
1162/// parsed before the error rather than failing the whole leadership sweep.
1163fn parse_ownership(xml: &[u8]) -> Vec<OwnershipPerson> {
1164 let mut reader = Reader::from_reader(xml);
1165 let mut buf = Vec::new();
1166 // Stack of open element local names, so a leaf is read in context.
1167 let mut path: Vec<Vec<u8>> = Vec::new();
1168 let mut owners: Vec<OwnerAcc> = Vec::new();
1169 // The reporting person currently being assembled, set inside a
1170 // `<reportingOwner>` block.
1171 let mut cur: Option<OwnerAcc> = None;
1172
1173 loop {
1174 match reader.read_event_into(&mut buf) {
1175 Ok(Event::Start(e)) => {
1176 let name = e.local_name().as_ref().to_vec();
1177 if name == b"reportingOwner" {
1178 cur = Some(OwnerAcc::default());
1179 }
1180 path.push(name);
1181 }
1182 Ok(Event::End(e)) => {
1183 if e.local_name().as_ref() == b"reportingOwner" {
1184 if let Some(o) = cur.take() {
1185 owners.push(o);
1186 }
1187 }
1188 path.pop();
1189 }
1190 Ok(Event::Text(t)) => {
1191 let (Some(tag), Some(o)) = (path.last(), cur.as_mut()) else {
1192 continue;
1193 };
1194 let raw = t.unescape().unwrap_or_default();
1195 let txt = raw.trim();
1196 if txt.is_empty() {
1197 continue;
1198 }
1199 match tag.as_slice() {
1200 b"rptOwnerName" if o.name.is_empty() => o.name = txt.to_string(),
1201 b"isDirector" => o.is_director |= ownership_flag(txt),
1202 b"isOfficer" => o.is_officer |= ownership_flag(txt),
1203 b"officerTitle" if o.officer_title.is_empty() => {
1204 o.officer_title = txt.to_string()
1205 }
1206 _ => {}
1207 }
1208 }
1209 Ok(Event::Eof) | Err(_) => break,
1210 _ => {}
1211 }
1212 buf.clear();
1213 }
1214
1215 owners
1216 .into_iter()
1217 .filter(|o| !o.name.is_empty())
1218 .map(|o| OwnershipPerson {
1219 name: o.name,
1220 is_director: o.is_director,
1221 is_officer: o.is_officer,
1222 officer_title: (!o.officer_title.is_empty()).then_some(o.officer_title),
1223 })
1224 .collect()
1225}
1226
1227#[cfg(test)]
1228mod tests {
1229 use super::*;
1230
1231 fn d(s: &str) -> NaiveDate {
1232 NaiveDate::parse_from_str(s, "%Y-%m-%d").unwrap()
1233 }
1234
1235 #[test]
1236 fn normalize_ticker_strips_punctuation() {
1237 assert_eq!(normalize_ticker("BRK.B"), "BRKB");
1238 assert_eq!(normalize_ticker("BF-B"), "BFB");
1239 assert_eq!(normalize_ticker("aapl"), "AAPL");
1240 }
1241
1242 #[test]
1243 fn fiscal_year_of_handles_non_calendar_year_end() {
1244 // September fiscal-year end: an Oct–Dec quarter is Q1 of the NEXT FY.
1245 assert_eq!(fiscal_year_of(d("2024-11-30"), 9), 2025);
1246 assert_eq!(fiscal_year_of(d("2024-08-31"), 9), 2024);
1247 // Calendar fiscal year: the period's calendar year is its fiscal year.
1248 assert_eq!(fiscal_year_of(d("2024-12-31"), 12), 2024);
1249 }
1250
1251 /// Build a duration UnitEntry (an income-statement fact).
1252 fn dur(start: &str, end: &str, val: f64, fp: &str, form: Option<&str>) -> UnitEntry {
1253 UnitEntry {
1254 start: Some(start.to_string()),
1255 end: end.to_string(),
1256 val,
1257 fp: Some(fp.to_string()),
1258 form: form.map(str::to_string),
1259 filed: Some("2025-02-01".to_string()),
1260 }
1261 }
1262 /// Build an instantaneous UnitEntry (a balance-sheet snapshot).
1263 fn inst(end: &str, val: f64, fp: &str, form: Option<&str>) -> UnitEntry {
1264 UnitEntry {
1265 start: None,
1266 end: end.to_string(),
1267 val,
1268 fp: Some(fp.to_string()),
1269 form: form.map(str::to_string),
1270 filed: Some("2025-02-01".to_string()),
1271 }
1272 }
1273
1274 #[test]
1275 fn classify_keeps_full_years_and_discrete_quarters() {
1276 // Full fiscal year (≈365-day duration, fp FY).
1277 assert_eq!(
1278 classify(&dur("2024-01-01", "2024-12-31", 100.0, "FY", Some("10-K")), 12),
1279 Some(("FY2024".to_string(), 2024, None))
1280 );
1281 // Discrete quarter (≈90-day duration).
1282 assert_eq!(
1283 classify(&dur("2024-01-01", "2024-03-31", 25.0, "Q1", Some("10-Q")), 12),
1284 Some(("Q1-2024".to_string(), 2024, Some(1)))
1285 );
1286 // Year-end balance from an annual report (instantaneous, fp FY, 10-K).
1287 assert_eq!(
1288 classify(&inst("2024-12-31", 500.0, "FY", Some("10-K")), 12),
1289 Some(("FY2024".to_string(), 2024, None))
1290 );
1291 }
1292
1293 #[test]
1294 fn classify_drops_ytd_rollups_and_quarterly_balances() {
1295 // A 6-month year-to-date roll-up is neither a quarter nor a full year.
1296 assert_eq!(
1297 classify(&dur("2024-01-01", "2024-06-30", 50.0, "Q2", Some("10-Q")), 12),
1298 None
1299 );
1300 // A balance-sheet snapshot from a 10-Q is dropped (it would mislabel a
1301 // prior year-end comparative as the filing's own quarter).
1302 assert_eq!(
1303 classify(&inst("2024-03-31", 480.0, "Q1", Some("10-Q")), 12),
1304 None
1305 );
1306 }
1307
1308 /// Parse a `companyfacts`-shaped JSON body for the select_facts tests.
1309 fn body(json: &str) -> CompanyFacts {
1310 serde_json::from_str(json).expect("valid companyfacts json")
1311 }
1312
1313 #[test]
1314 fn select_facts_pins_one_concept_per_metric() {
1315 // Revenue is reported under TWO non-interchangeable concepts. The newer
1316 // concept (RevenueFromContract…) covers FY2023 AND FY2024; the legacy
1317 // `Revenues` covers FY2023 only, with a DIFFERENT value. The pinned
1318 // concept must be the one reaching the most recent period, and BOTH its
1319 // years must come from it — so FY2023 is 1000 (its value), never 999.
1320 let b = body(
1321 r#"{
1322 "facts": { "us-gaap": {
1323 "RevenueFromContractWithCustomerExcludingAssessedTax": { "units": { "USD": [
1324 {"start":"2023-01-01","end":"2023-12-31","val":1000,"fp":"FY","form":"10-K","filed":"2024-02-01"},
1325 {"start":"2024-01-01","end":"2024-12-31","val":1200,"fp":"FY","form":"10-K","filed":"2025-02-01"}
1326 ] } },
1327 "Revenues": { "units": { "USD": [
1328 {"start":"2023-01-01","end":"2023-12-31","val":999,"fp":"FY","form":"10-K","filed":"2024-02-01"}
1329 ] } }
1330 } }
1331 }"#,
1332 );
1333 let facts = select_facts(&b, 2025);
1334 let rev: std::collections::HashMap<i64, f64> = facts
1335 .iter()
1336 .filter(|f| f.metric == "revenue" && f.fiscal_qtr.is_none())
1337 .map(|f| (f.fiscal_year, f.value))
1338 .collect();
1339 assert_eq!(rev.get(&2024), Some(&1200.0));
1340 assert_eq!(
1341 rev.get(&2023),
1342 Some(&1000.0),
1343 "FY2023 must come from the SAME pinned concept, not the mismatched 999"
1344 );
1345 // Year-over-year growth is then a like-for-like comparison.
1346 let growth: f64 = (1200.0 - 1000.0) / 1000.0 * 100.0;
1347 assert!((growth - 20.0).abs() < 1e-9);
1348 }
1349
1350 #[test]
1351 fn select_facts_takes_latest_restatement_within_a_concept() {
1352 // The same period reported twice; the later `filed` wins.
1353 let b = body(
1354 r#"{
1355 "facts": { "us-gaap": {
1356 "NetIncomeLoss": { "units": { "USD": [
1357 {"start":"2024-01-01","end":"2024-12-31","val":300,"fp":"FY","form":"10-K","filed":"2025-02-01"},
1358 {"start":"2024-01-01","end":"2024-12-31","val":280,"fp":"FY","form":"10-K","filed":"2025-06-01"}
1359 ] } }
1360 } }
1361 }"#,
1362 );
1363 let facts = select_facts(&b, 2025);
1364 let ni = facts.iter().find(|f| f.metric == "net_income" && f.fiscal_year == 2024);
1365 assert_eq!(ni.map(|f| f.value), Some(280.0), "the later-filed restatement (280) wins over 300");
1366 }
1367}