analytics-django

  1"""
  2Download or refresh the DB-IP City Lite GeoIP database.
  3
  4DB-IP publishes a fresh build on the 1st of each month at:
  5
  6    https://download.db-ip.com/free/dbip-city-lite-YYYY-MM.mmdb.gz
  7
  8License is CC-BY-4.0 — attribution lives in the dashboard footer. No account,
  9no API key. The mmdb format is MaxMind-compatible so the existing geoip2
 10reader picks it up unchanged.
 11
 12Run on container start (idempotent, skips if the on-disk file is fresh) and
 13again monthly via host cron.
 14"""
 15
 16import gzip
 17import os
 18import shutil
 19import sys
 20import tempfile
 21import urllib.error
 22import urllib.request
 23from datetime import date, timedelta
 24from pathlib import Path
 25
 26from django.conf import settings
 27from django.core.management.base import BaseCommand
 28
 29
 30URL_TEMPLATE = "https://download.db-ip.com/free/dbip-city-lite-{year}-{month:02d}.mmdb.gz"
 31USER_AGENT = "analytics-refresh-geoip/1.0 (+https://github.com/overshard/analytics-django)"
 32MAX_AGE_DAYS = 30
 33
 34
 35def _candidate_months(today=None):
 36    """
 37    Yield (year, month) tuples to try, newest first.
 38
 39    DB-IP publishes on the 1st but may lag a few hours; if the current month
 40    isn't up yet, fall back to the previous month, and one more before that
 41    in case we're catching up after a long outage.
 42    """
 43    today = today or date.today()
 44    for offset in (0, 1, 2):
 45        d = (today.replace(day=1) - timedelta(days=offset * 28)).replace(day=1)
 46        yield d.year, d.month
 47
 48
 49class Command(BaseCommand):
 50    help = "Download (or refresh) the DB-IP City Lite GeoIP database to GEOIP_PATH."
 51
 52    def add_arguments(self, parser):
 53        parser.add_argument(
 54            "--force",
 55            action="store_true",
 56            help="Re-download even if the existing file is younger than 30 days.",
 57        )
 58
 59    def handle(self, *args, **options):
 60        target = Path(getattr(settings, "GEOIP_PATH", "")).resolve()
 61        if not target.parent.exists():
 62            self.stderr.write(f"GEOIP_PATH parent dir does not exist: {target.parent}")
 63            sys.exit(0)
 64
 65        if not options["force"] and target.exists():
 66            age_days = (date.today() - date.fromtimestamp(target.stat().st_mtime)).days
 67            if age_days < MAX_AGE_DAYS:
 68                self.stdout.write(f"GeoIP database is {age_days}d old; skipping refresh.")
 69                return
 70
 71        last_error = None
 72        for year, month in _candidate_months():
 73            url = URL_TEMPLATE.format(year=year, month=month)
 74            try:
 75                self.stdout.write(f"Fetching {url}")
 76                self._download(url, target)
 77                self.stdout.write(self.style.SUCCESS(f"GeoIP database updated at {target}"))
 78                return
 79            except urllib.error.HTTPError as e:
 80                if e.code == 404:
 81                    self.stdout.write(f"  not yet published ({year}-{month:02d})")
 82                    last_error = e
 83                    continue
 84                last_error = e
 85                break
 86            except (urllib.error.URLError, OSError) as e:
 87                last_error = e
 88                break
 89
 90        # Non-fatal: dashboard works without GeoIP, collector silently skips
 91        # enrichment when the file is missing or stale.
 92        self.stderr.write(f"GeoIP refresh failed: {last_error}")
 93        sys.exit(0)
 94
 95    def _download(self, url, target):
 96        request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
 97        with urllib.request.urlopen(request, timeout=120) as response:
 98            tmp_dir = target.parent
 99            with tempfile.NamedTemporaryFile(
100                dir=tmp_dir, prefix=".geoip-", suffix=".mmdb", delete=False
101            ) as tmp:
102                tmp_path = Path(tmp.name)
103                try:
104                    with gzip.GzipFile(fileobj=response) as gz:
105                        shutil.copyfileobj(gz, tmp)
106                    tmp.flush()
107                    os.fsync(tmp.fileno())
108                except Exception:
109                    tmp_path.unlink(missing_ok=True)
110                    raise
111        os.replace(tmp_path, target)