status-django

  1"""HTML parsing: raw body -> structured page dict."""
  2import hashlib
  3import json
  4import logging
  5from urllib.parse import urljoin
  6
  7from bs4 import BeautifulSoup
  8
  9
 10logger = logging.getLogger(__name__)
 11
 12
 13def parse_html(body, url):
 14    """Extract everything the checks need from one HTML page.
 15
 16    Returns a dict with title, meta, headings, links, images, resources,
 17    forms, json_ld, word count, text hash, favicon, lang, viewport, robots.
 18    """
 19    soup = BeautifulSoup(body, "lxml")
 20
 21    title = ""
 22    if soup.title and soup.title.string:
 23        title = soup.title.string.strip()
 24
 25    def meta_name(name):
 26        tag = soup.find("meta", attrs={"name": name})
 27        if tag and tag.get("content"):
 28            return tag["content"].strip()
 29        return ""
 30
 31    def meta_property(prop):
 32        tag = soup.find("meta", attrs={"property": prop})
 33        if tag and tag.get("content"):
 34            return tag["content"].strip()
 35        return ""
 36
 37    description = meta_name("description")
 38    robots_meta = meta_name("robots")
 39    viewport = meta_name("viewport")
 40
 41    canonical_tag = soup.find("link", rel="canonical")
 42    canonical = ""
 43    if canonical_tag and canonical_tag.get("href"):
 44        canonical = urljoin(url, canonical_tag["href"].strip())
 45
 46    og = {
 47        "title": meta_property("og:title"),
 48        "description": meta_property("og:description"),
 49        "image": meta_property("og:image"),
 50        "url": meta_property("og:url"),
 51    }
 52
 53    twitter = {
 54        "card": meta_name("twitter:card"),
 55        "title": meta_name("twitter:title"),
 56        "description": meta_name("twitter:description"),
 57    }
 58
 59    html_tag = soup.find("html")
 60    lang = html_tag.get("lang", "").strip() if html_tag else ""
 61
 62    headings = {f"h{i}": [] for i in range(1, 7)}
 63    for level in range(1, 7):
 64        for h in soup.find_all(f"h{level}"):
 65            headings[f"h{level}"].append(h.get_text(" ", strip=True))
 66
 67    links = []
 68    for a in soup.find_all("a", href=True):
 69        href = a["href"].strip()
 70        if not href or href.startswith(("javascript:", "mailto:", "tel:", "#")):
 71            continue
 72        rel = a.get("rel") or []
 73        if isinstance(rel, str):
 74            rel = rel.split()
 75        links.append(
 76            {
 77                "url": urljoin(url, href),
 78                "text": a.get_text(" ", strip=True),
 79                "rel": list(rel),
 80            }
 81        )
 82
 83    images = []
 84    for img in soup.find_all("img"):
 85        src = img.get("src", "").strip()
 86        alt = img.get("alt")  # None = missing attribute, "" = explicitly empty
 87        images.append(
 88            {
 89                "src": urljoin(url, src) if src else "",
 90                "alt": alt,
 91            }
 92        )
 93
 94    resources = []
 95    for tag in soup.find_all(["script", "link", "img", "iframe", "source"]):
 96        src = tag.get("src") or tag.get("href")
 97        if src and src.strip():
 98            resources.append(urljoin(url, src.strip()))
 99
100    json_ld = []
101    for s in soup.find_all("script", type="application/ld+json"):
102        raw = s.string or s.get_text() or ""
103        if not raw.strip():
104            continue
105        try:
106            json_ld.append(json.loads(raw))
107        except (ValueError, TypeError):
108            json_ld.append(None)  # parse error
109
110    favicon = ""
111    for link in soup.find_all("link", rel=True):
112        rels = link.get("rel", [])
113        if isinstance(rels, str):
114            rels = rels.split()
115        if any("icon" in r.lower() for r in rels):
116            href = link.get("href", "").strip()
117            if href:
118                favicon = urljoin(url, href)
119                break
120
121    forms = []
122    for form in soup.find_all("form"):
123        inputs = []
124        for i in form.find_all(["input", "textarea", "select"]):
125            inputs.append(
126                {
127                    "type": i.get("type", "text"),
128                    "name": i.get("name"),
129                    "id": i.get("id"),
130                    "aria_label": i.get("aria-label"),
131                }
132            )
133        label_fors = {lb.get("for") for lb in form.find_all("label") if lb.get("for")}
134        forms.append(
135            {
136                "action": urljoin(url, form.get("action", "")) if form.get("action") else url,
137                "inputs": inputs,
138                "label_fors": list(label_fors),
139            }
140        )
141
142    # Visible text for word count + duplicate detection.
143    for tag in soup(["script", "style", "noscript"]):
144        tag.decompose()
145    text = soup.get_text(" ", strip=True)
146    word_count = len(text.split())
147    text_hash = hashlib.sha256(text.encode("utf-8", errors="replace")).hexdigest()
148
149    return {
150        "title": title,
151        "description": description,
152        "canonical": canonical,
153        "robots_meta": robots_meta,
154        "viewport": viewport,
155        "lang": lang,
156        "og": og,
157        "twitter": twitter,
158        "headings": headings,
159        "links": links,
160        "images": images,
161        "resources": resources,
162        "json_ld": json_ld,
163        "favicon": favicon,
164        "forms": forms,
165        "word_count": word_count,
166        "text_hash": text_hash,
167    }