Self-hostable uptime monitor and status page on Django: HTTP checks, Lighthouse audits, SEO crawls, and email and Discord alerts.
djangodockerhandcodedpythonself-hostedsqlitestatus-pageuptime-monitoringvite
1import logging
2import signal
3import threading
4from concurrent.futures import ThreadPoolExecutor
5
6from django import db
7from django.core.management.base import BaseCommand
8from django.db.models import Q
9from django.utils import timezone
10
11from properties.models import Check, Property
12
13logger = logging.getLogger(__name__)
14
15
16class Command(BaseCommand):
17 # Two pools so slow lighthouse/crawler work can't starve quick HTTP pings.
18 SLOW_WORKERS = 2
19 FAST_WORKERS = 2
20 CYCLE_SECONDS = 30
21 CLEANUP_INTERVAL_SECONDS = 86400
22
23 def __init__(self):
24 super().__init__()
25 self._stop = threading.Event()
26 self._last_cleanup = None
27
28 def handle(self, *args, **options):
29 self.stdout.write("[Scheduler] Starting scheduler...")
30
31 signal.signal(signal.SIGTERM, self._on_signal)
32 signal.signal(signal.SIGINT, self._on_signal)
33
34 # Clear any running/queued states left over from a prior crash so
35 # rows don't sit stuck and block new runs.
36 Property.objects.filter(crawl_state__in=["queued", "running"]).update(
37 crawl_state="idle"
38 )
39 Property.objects.filter(lighthouse_state__in=["queued", "running"]).update(
40 lighthouse_state="idle"
41 )
42
43 slow = ThreadPoolExecutor(
44 max_workers=self.SLOW_WORKERS, thread_name_prefix="slow"
45 )
46 fast = ThreadPoolExecutor(
47 max_workers=self.FAST_WORKERS, thread_name_prefix="fast"
48 )
49
50 try:
51 while not self._stop.is_set():
52 try:
53 self._enqueue_status(fast)
54 self._enqueue_lighthouse(slow)
55 self._enqueue_crawler(slow)
56 self.reset_wedged_states()
57 self._maybe_cleanup()
58 except Exception:
59 logger.exception("[Scheduler] cycle error")
60
61 self.stdout.write(
62 f"[Scheduler] Sleeping scheduler for {self.CYCLE_SECONDS} seconds..."
63 )
64 self._stop.wait(self.CYCLE_SECONDS)
65 finally:
66 self.stdout.write("[Scheduler] Stopping scheduler...")
67 slow.shutdown(wait=False, cancel_futures=True)
68 fast.shutdown(wait=False, cancel_futures=True)
69
70 def _on_signal(self, signum, frame):
71 self.stdout.write(f"[Scheduler] Received signal {signum}, shutting down...")
72 self._stop.set()
73
74 def reset_wedged_states(self):
75 """Flip running rows back to idle once they've overrun their deadline.
76
77 Only "running" rows count as wedged. "queued" rows are waiting their
78 turn in the thread pool and will be picked up when a worker frees up;
79 flipping them here would mark healthy backlog as failed whenever the
80 user fans out manual re-triggers.
81
82 The startup path in handle() also wipes any leftover queued/running
83 state unconditionally to cover crashes.
84 """
85 now = timezone.now()
86 crawl_cutoff = now - timezone.timedelta(seconds=900)
87 lh_cutoff = now - timezone.timedelta(seconds=300)
88
89 Property.objects.filter(
90 crawl_state="running",
91 crawl_started_at__lt=crawl_cutoff,
92 ).update(
93 crawl_state="idle",
94 last_crawl_error="Crawl timed out or was interrupted",
95 )
96
97 Property.objects.filter(
98 lighthouse_state="running",
99 lighthouse_started_at__lt=lh_cutoff,
100 ).update(
101 lighthouse_state="idle",
102 last_lighthouse_error="Lighthouse run timed out or was interrupted",
103 )
104
105 def _maybe_cleanup(self):
106 now = timezone.now()
107 if (
108 self._last_cleanup
109 and (now - self._last_cleanup).total_seconds()
110 < self.CLEANUP_INTERVAL_SECONDS
111 ):
112 return
113 self.stdout.write("[Scheduler] Cleaning checks older than 3 days...")
114 Check.objects.filter(
115 created_at__lt=now - timezone.timedelta(days=3)
116 ).delete()
117 self._last_cleanup = now
118
119 def _enqueue_status(self, pool):
120 now = timezone.now()
121 due = list(
122 Property.objects.filter(
123 Q(last_run_at__isnull=True)
124 | Q(next_run_at__isnull=True)
125 | Q(next_run_at__lte=now)
126 )
127 )
128 for p in due:
129 p.next_run_at = p.get_next_run_at()
130 p.last_run_at = timezone.now()
131 p.save(update_fields=["next_run_at", "last_run_at"])
132 pool.submit(self._run_status, p.id)
133 db.connections.close_all()
134
135 def _enqueue_lighthouse(self, pool):
136 now = timezone.now()
137 due = list(
138 Property.objects.filter(
139 Q(last_lighthouse_run_at__isnull=True)
140 | Q(next_lighthouse_run_at__isnull=True)
141 | Q(next_lighthouse_run_at__lte=now)
142 ).exclude(lighthouse_state__in=["queued", "running"])
143 )
144 for p in due:
145 p.next_lighthouse_run_at = p.get_next_run_at_lighthouse()
146 p.last_lighthouse_run_at = timezone.now()
147 p.lighthouse_state = "queued"
148 p.save(
149 update_fields=[
150 "next_lighthouse_run_at",
151 "last_lighthouse_run_at",
152 "lighthouse_state",
153 ]
154 )
155 pool.submit(self._run_lighthouse, p.id)
156 db.connections.close_all()
157
158 def _enqueue_crawler(self, pool):
159 now = timezone.now()
160 due = list(
161 Property.objects.filter(
162 Q(last_run_at_crawler__isnull=True)
163 | Q(next_run_at_crawler__isnull=True)
164 | Q(next_run_at_crawler__lte=now)
165 ).exclude(crawl_state__in=["queued", "running"])
166 )
167 for p in due:
168 p.next_run_at_crawler = p.get_next_run_at_crawl()
169 p.last_run_at_crawler = timezone.now()
170 p.crawl_state = "queued"
171 p.save(
172 update_fields=[
173 "next_run_at_crawler",
174 "last_run_at_crawler",
175 "crawl_state",
176 ]
177 )
178 pool.submit(self._run_crawler, p.id)
179 db.connections.close_all()
180
181 def _run_status(self, property_id):
182 try:
183 prop = Property.objects.get(id=property_id)
184 self.stdout.write(f"[Scheduler] Checking status {prop.url}")
185 prop.process_check()
186 except Exception:
187 logger.exception("[Scheduler] status check failed for %s", property_id)
188 finally:
189 db.close_old_connections()
190
191 def _run_lighthouse(self, property_id):
192 try:
193 prop = Property.objects.get(id=property_id)
194 self.stdout.write(f"[Scheduler] Checking lighthouse {prop.url}")
195 prop.process_check_lighthouse()
196 except Exception:
197 logger.exception("[Scheduler] lighthouse failed for %s", property_id)
198 finally:
199 db.close_old_connections()
200
201 def _run_crawler(self, property_id):
202 try:
203 prop = Property.objects.get(id=property_id)
204 self.stdout.write(f"[Scheduler] Checking crawler {prop.url}")
205 prop.crawl_site()
206 except Exception:
207 logger.exception("[Scheduler] crawler failed for %s", property_id)
208 finally:
209 db.close_old_connections()