status-rust

  1//! Import an existing Django status SQLite into the rust schema.
  2//!
  3//! Reads `properties_property` and `properties_check` from the Django DB
  4//! and writes them into the new schema, preserving Property UUIDs so
  5//! existing public status URLs keep working.
  6use anyhow::{Context, Result};
  7use sqlx::sqlite::SqliteConnectOptions;
  8use sqlx::{Connection, SqliteConnection};
  9use std::path::Path;
 10use std::path::PathBuf;
 11use std::str::FromStr;
 12
 13pub async fn run(source: PathBuf, force: bool) -> Result<()> {
 14    if !source.exists() {
 15        anyhow::bail!("source DB not found: {}", source.display());
 16    }
 17
 18    let data_dir = std::env::var("STATUS_DATA_DIR")
 19        .map(PathBuf::from)
 20        .unwrap_or_else(|_| PathBuf::from("data"));
 21    std::fs::create_dir_all(&data_dir)?;
 22    let dest_path = data_dir.join("db.sqlite3");
 23
 24    if dest_path.exists() && !force {
 25        anyhow::bail!(
 26            "{} already exists. Pass --force to wipe it first.",
 27            dest_path.display()
 28        );
 29    }
 30    if dest_path.exists() {
 31        std::fs::remove_file(&dest_path).context("removing existing dest db")?;
 32    }
 33
 34    let pool = crate::db::init(&data_dir).await?;
 35
 36    let src_url = format!("sqlite://{}?mode=ro", source.display());
 37    let opts = SqliteConnectOptions::from_str(&src_url)?
 38        .create_if_missing(false)
 39        .read_only(true);
 40    let mut src = SqliteConnection::connect_with(&opts).await.context("opening source db")?;
 41
 42    // ---------- properties ----------
 43    // Django columns: id (UUID hex string), url, is_public, last_run_at,
 44    // next_run_at, last_run_at_crawler, next_run_at_crawler, crawler_insights
 45    // (JSON or NULL), crawl_state, crawl_started_at, last_crawl_success_at,
 46    // last_crawl_error, last_crawl_duration_ms, last_crawl_pages_count,
 47    // lighthouse_scores, lighthouse_details, last_lighthouse_run_at,
 48    // last_lighthouse_success_at, last_lighthouse_error,
 49    // last_lighthouse_duration_ms, next_lighthouse_run_at, lighthouse_state,
 50    // lighthouse_started_at, last_alert_sent, alert_state, created_at,
 51    // updated_at, user_id.
 52    let rows: Vec<DjangoProperty> = sqlx::query_as::<_, DjangoProperty>(
 53        "SELECT id, url, is_public, \
 54                last_run_at, next_run_at, \
 55                last_run_at_crawler, next_run_at_crawler, crawler_insights, \
 56                crawl_state, crawl_started_at, last_crawl_success_at, last_crawl_error, \
 57                last_crawl_duration_ms, last_crawl_pages_count, \
 58                lighthouse_scores, lighthouse_details, last_lighthouse_run_at, \
 59                last_lighthouse_success_at, last_lighthouse_error, last_lighthouse_duration_ms, \
 60                next_lighthouse_run_at, lighthouse_state, lighthouse_started_at, \
 61                last_alert_sent, alert_state, created_at, updated_at \
 62         FROM properties_property",
 63    )
 64    .fetch_all(&mut src)
 65    .await
 66    .context("reading properties from source db")?;
 67
 68    let mut prop_count = 0;
 69    for row in &rows {
 70        let uuid = uuid::Uuid::parse_str(&row.id)
 71            .or_else(|_| uuid::Uuid::parse_str(&format!(
 72                "{}-{}-{}-{}-{}",
 73                &row.id[0..8], &row.id[8..12], &row.id[12..16], &row.id[16..20], &row.id[20..]
 74            )))
 75            .with_context(|| format!("parsing uuid: {}", row.id))?;
 76        let blob = uuid.as_bytes().to_vec();
 77        sqlx::query(
 78            r#"INSERT INTO properties (
 79                id, url, is_public, is_protected,
 80                last_run_at, next_run_at,
 81                last_run_at_crawler, next_run_at_crawler, crawler_insights,
 82                crawl_state, crawl_started_at, last_crawl_success_at, last_crawl_error,
 83                last_crawl_duration_ms, last_crawl_pages_count,
 84                lighthouse_scores, lighthouse_details, last_lighthouse_run_at,
 85                last_lighthouse_success_at, last_lighthouse_error, last_lighthouse_duration_ms,
 86                next_lighthouse_run_at, lighthouse_state, lighthouse_started_at,
 87                alert_state, last_alert_sent, created_at, updated_at
 88            ) VALUES (?, ?, ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"#,
 89        )
 90        .bind(&blob)
 91        .bind(&row.url)
 92        .bind(row.is_public)
 93        .bind(parse_django_dt(&row.last_run_at))
 94        .bind(parse_django_dt(&row.next_run_at))
 95        .bind(parse_django_dt(&row.last_run_at_crawler))
 96        .bind(parse_django_dt(&row.next_run_at_crawler))
 97        .bind(&row.crawler_insights)
 98        .bind(&row.crawl_state)
 99        .bind(parse_django_dt(&row.crawl_started_at))
100        .bind(parse_django_dt(&row.last_crawl_success_at))
101        .bind(&row.last_crawl_error)
102        .bind(row.last_crawl_duration_ms)
103        .bind(row.last_crawl_pages_count)
104        .bind(&row.lighthouse_scores)
105        .bind(&row.lighthouse_details)
106        .bind(parse_django_dt(&row.last_lighthouse_run_at))
107        .bind(parse_django_dt(&row.last_lighthouse_success_at))
108        .bind(&row.last_lighthouse_error)
109        .bind(row.last_lighthouse_duration_ms)
110        .bind(parse_django_dt(&row.next_lighthouse_run_at))
111        .bind(&row.lighthouse_state)
112        .bind(parse_django_dt(&row.lighthouse_started_at))
113        .bind(&row.alert_state)
114        .bind(parse_django_dt(&row.last_alert_sent))
115        .bind(parse_django_dt(&row.created_at).unwrap_or(0))
116        .bind(parse_django_dt(&row.updated_at).unwrap_or(0))
117        .execute(&pool)
118        .await
119        .with_context(|| format!("inserting property {}", row.url))?;
120        prop_count += 1;
121    }
122
123    // ---------- checks ----------
124    let checks: Vec<DjangoCheck> = sqlx::query_as::<_, DjangoCheck>(
125        "SELECT property_id, status_code, response_time, headers, created_at FROM properties_check",
126    )
127    .fetch_all(&mut src)
128    .await
129    .context("reading checks from source db")?;
130
131    let mut check_count = 0;
132    for row in &checks {
133        let uuid = match uuid::Uuid::parse_str(&row.property_id) {
134            Ok(u) => u,
135            Err(_) => continue,
136        };
137        let blob = uuid.as_bytes().to_vec();
138        let created = parse_django_dt(&row.created_at).unwrap_or(0);
139        let response_ms = row.response_time;
140        let _ = sqlx::query(
141            "INSERT INTO checks (property_id, status_code, response_ms, headers, created_at) \
142             VALUES (?, ?, ?, ?, ?)",
143        )
144        .bind(&blob)
145        .bind(row.status_code)
146        .bind(response_ms)
147        .bind(&row.headers)
148        .bind(created)
149        .execute(&pool)
150        .await;
151        check_count += 1;
152    }
153
154    println!(
155        "[migrate] imported {prop_count} properties, {check_count} checks into {}",
156        dest_path.display()
157    );
158    let _ = src.close().await;
159    Ok(())
160}
161
162#[derive(sqlx::FromRow)]
163struct DjangoProperty {
164    id: String,
165    url: String,
166    is_public: i64,
167    last_run_at: Option<String>,
168    next_run_at: Option<String>,
169    last_run_at_crawler: Option<String>,
170    next_run_at_crawler: Option<String>,
171    crawler_insights: Option<String>,
172    crawl_state: String,
173    crawl_started_at: Option<String>,
174    last_crawl_success_at: Option<String>,
175    last_crawl_error: Option<String>,
176    last_crawl_duration_ms: Option<i64>,
177    last_crawl_pages_count: Option<i64>,
178    lighthouse_scores: Option<String>,
179    lighthouse_details: Option<String>,
180    last_lighthouse_run_at: Option<String>,
181    last_lighthouse_success_at: Option<String>,
182    last_lighthouse_error: Option<String>,
183    last_lighthouse_duration_ms: Option<i64>,
184    next_lighthouse_run_at: Option<String>,
185    lighthouse_state: String,
186    lighthouse_started_at: Option<String>,
187    last_alert_sent: Option<String>,
188    alert_state: String,
189    created_at: Option<String>,
190    updated_at: Option<String>,
191}
192
193#[derive(sqlx::FromRow)]
194struct DjangoCheck {
195    property_id: String,
196    status_code: i64,
197    // Django schema declared `INTEGER NOT NULL`; the Python ORM stored
198    // milliseconds as ints. (An earlier draft tried `f64` and crashed on
199    // strict-typed sqlx decoding.)
200    response_time: i64,
201    headers: String,
202    created_at: Option<String>,
203}
204
205/// Django stores datetimes as ISO8601 strings (`YYYY-MM-DD HH:MM:SS[.f]`).
206/// Parse to UTC ms since epoch; return None if absent or unparseable.
207fn parse_django_dt(s: &Option<String>) -> Option<i64> {
208    let raw = s.as_deref()?.trim();
209    if raw.is_empty() {
210        return None;
211    }
212    let normalized = raw.replace('T', " ");
213    let parsed = chrono::NaiveDateTime::parse_from_str(&normalized, "%Y-%m-%d %H:%M:%S%.f")
214        .or_else(|_| chrono::NaiveDateTime::parse_from_str(&normalized, "%Y-%m-%d %H:%M:%S"))
215        .ok()?;
216    Some(chrono::Utc.from_utc_datetime(&parsed).timestamp_millis())
217}
218
219use chrono::TimeZone;
220
221#[allow(dead_code)]
222fn _unused(_: &Path) {}