maarcadetweet: initial commit

AT Protocol PDS + AppView + Tauri Desktop Client, 160-char post limit. - PDS (Rust + axum + sqlx) - Auth: createAccount, createSession, refreshSession - Records: createRecord, deleteRecord (race-safe via SELECT FOR UPDATE) - Feed: feed.like.create, feed.repost.create - Sync: getRepo, getBlocks, getLatestCommit, getRecord (with MST proof), listRepos - Identity: resolveHandle - MST: spec-conformant (at-mst crate, 27 tests) - Repo: signed commits, TID counter (monotonic, 4096 wrap safe) - AppView (Rust + axum + sqlx) - Jetstream consumer (WebSocket, exponential backoff, 38k+ events indexed) - REST API: timeline/home (graph-aware), profile, search, post (with thread hydration) - Handle-sync worker (did:plc + did:web) - JSONB embed storage + thread columns (migration 0003) - Like/repost counter cache (migration 0004) - Tauri 2 + Svelte 5 Desktop Client - System tray (Show/Compose/Quit menu) - OS notifications (tauri-plugin-notification) - Auto-update (tauri-plugin-updater, placeholder endpoint) - Window-state (tauri-plugin-window-state) - 160-char compose with live counter - Image/Link embed rendering - LocalStorage-persisted like state - Timeline with poll (prepend new posts) - Custom TitleBar (transparent, no decorations) - Orange/IBM Plex Mono maarcade design Tests: 231 Rust + 9 vitest = 240 passed.
2026-07-05 20:01:31 +02:00
commit c586fd39c9
134 changed files with 35279 additions and 0 deletions
@@ -0,0 +1,225 @@
+//! Connects `at-firehose::JetstreamConsumer` into the AppView indexer.
+//!
+//! The handler is intentionally tiny: it routes events by `kind`, delegates
+//! all DB work to [`crate::indexer`], and feeds a small `mpsc` channel that
+//! a background task drains to flush the cursor.
+//!
+//! Stats (events processed, last cursor seen, last-event timestamp for
+//! connection health) live in an `Arc<Stats>` so both the consumer closure
+//! and the HTTP `/healthz` handler can read them without locking.
+
+use anyhow::Result;
+use at_firehose::JetstreamEvent;
+use sqlx::PgPool;
+use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::mpsc;
+use tracing::{debug, info, trace, warn};
+
+use crate::indexer;
+
+/// Shared counters consumed by `/healthz`.
+pub struct Stats {
+    pub events_processed: AtomicU64,
+    /// Microseconds since epoch of the most recent event seen.
+    pub last_event_time_us: AtomicI64,
+    /// Microseconds since epoch of the last persisted cursor (best-effort).
+    pub last_cursor_persisted_us: AtomicI64,
+    /// Whether the Jetstream WebSocket is currently up. The consumer
+    /// writes this; `/healthz` reads it. Wrapped in `Arc` so the consumer
+    /// can hold its own clone without borrowing from us.
+    pub jetstream_connected: Arc<AtomicBool>,
+}
+
+impl Default for Stats {
+    fn default() -> Self {
+        Self {
+            events_processed: AtomicU64::new(0),
+            last_event_time_us: AtomicI64::new(0),
+            last_cursor_persisted_us: AtomicI64::new(0),
+            jetstream_connected: Arc::new(AtomicBool::new(false)),
+        }
+    }
+}
+
+impl std::fmt::Debug for Stats {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Stats")
+            .field("events_processed", &self.events_processed())
+            .field("last_event_time_us", &self.last_event_time_us.load(Ordering::Relaxed))
+            .field("jetstream_connected", &self.jetstream_connected())
+            .finish()
+    }
+}
+
+impl Stats {
+    pub fn new() -> Arc<Self> {
+        Arc::new(Self::default())
+    }
+
+    /// Cheap clone of the `connected` flag (what the consumer stores into).
+    pub fn jetstream_connected_arc(&self) -> Arc<AtomicBool> {
+        self.jetstream_connected.clone()
+    }
+
+    /// Lag in milliseconds (last event time − local wall clock). When the
+    /// local clock is ahead (very common — Jetstream events usually look
+    /// "in the past" by a few seconds because the spec is `time_us` from
+    /// the producer), this returns 0.
+    pub fn lag_ms(&self) -> i64 {
+        let last = self.last_event_time_us.load(Ordering::Relaxed);
+        if last == 0 {
+            return 0;
+        }
+        let now_us = chrono::Utc::now().timestamp_micros();
+        let lag_us = (now_us - last).max(0);
+        lag_us / 1000
+    }
+
+    pub fn events_processed(&self) -> u64 {
+        self.events_processed.load(Ordering::Relaxed)
+    }
+
+    pub fn jetstream_connected(&self) -> bool {
+        self.jetstream_connected.load(Ordering::Relaxed)
+    }
+}
+
+/// The thing the Jetstream consumer calls once per event.
+#[derive(Clone)]
+pub struct IndexHandler {
+    pub db: PgPool,
+    pub stats: Arc<Stats>,
+    /// Sender side of the cursor-flush channel. The consumer closure pushes
+    /// `event.time_us` every 100 events; a background task drains it and
+    /// writes the running maximum to `jetstream_cursor` at most once every
+    /// 500ms. The bound of 32 is plenty — the background task keeps up.
+    cursor_tx: mpsc::Sender<i64>,
+}
+
+impl IndexHandler {
+    pub fn new(db: PgPool, stats: Arc<Stats>, cursor_tx: mpsc::Sender<i64>) -> Self {
+        Self { db, stats, cursor_tx }
+    }
+
+    pub async fn handle(&self, ev: JetstreamEvent) -> Result<()> {
+        // Update per-event stats first so /healthz reflects liveness even
+        // when DB writes are slow.
+        self.stats.events_processed.fetch_add(1, Ordering::Relaxed);
+        let prev = self
+            .stats
+            .last_event_time_us
+            .fetch_max(ev.time_us, Ordering::Relaxed);
+        if ev.time_us < prev {
+            // Out-of-order event: keep the larger value but still try to
+            // process. (Jetstream delivers near-monotonically but it's
+            // not guaranteed.)
+            self.stats
+                .last_event_time_us
+                .store(prev, Ordering::Relaxed);
+        }
+
+        let ok: bool = match ev.kind.as_str() {
+            "commit" => match indexer::apply_commit(&self.db, &ev).await {
+                Ok(true) => true,
+                Ok(false) => {
+                    debug!(kind = "commit", "apply_commit skipped event (unrecognized sub-collection)");
+                    true // cursor can still advance — we "saw" the event
+                }
+                Err(e) => {
+                    warn!(error = %e, kind = "commit", "apply_commit failed; NOT advancing cursor — expect reconnect replay");
+                    false
+                }
+            },
+            "identity" => {
+                trace!(did = %ev.did, "identity event (logged only)");
+                let _ = handle_identity(&ev);
+                true
+            }
+            "account" => {
+                trace!(did = %ev.did, "account event (logged only)");
+                let _ = handle_account(&ev);
+                true
+            }
+            other => {
+                debug!(kind = %other, "ignoring event of unknown kind");
+                true
+            }
+        };
+
+        // ONLY advance the cursor when the event was processed or skipped
+        // legitimately. Failures leave the cursor pinned so reconnect replays
+        // the event.
+        if !ok {
+            return Ok(());
+        }
+
+        // Forward the cursor only every 100 events to avoid hammering the DB.
+        if self.stats.events_processed() % 100 == 0 {
+            let _ = self.cursor_tx.try_send(ev.time_us);
+        }
+
+        Ok(())
+    }
+}
+
+fn handle_identity(_ev: &JetstreamEvent) -> Result<()> {
+    info!("identity change (DID doc rotation)");
+    Ok(())
+}
+
+fn handle_account(_ev: &JetstreamEvent) -> Result<()> {
+    info!("account change (active/-status)");
+    Ok(())
+}
+
+/// Spawn the background task that drains the cursor-flush channel and
+/// writes the running maximum to the DB. Returns when the receiver is
+/// dropped (i.e. the main process is shutting down).
+pub fn spawn_cursor_flush(
+    db: PgPool,
+    mut rx: mpsc::Receiver<i64>,
+    stats: Arc<Stats>,
+) -> tokio::task::JoinHandle<()> {
+    tokio::spawn(async move {
+        let mut buf: Vec<i64> = Vec::with_capacity(128);
+        let mut interval = tokio::time::interval(Duration::from_millis(500));
+        interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+
+        loop {
+            tokio::select! {
+                biased;
+                maybe = rx.recv() => {
+                    match maybe {
+                        Some(ts) => buf.push(ts),
+                        None => {
+                            // Channel closed — flush whatever's left and
+                            // exit cleanly.
+                            if !buf.is_empty() {
+                                let max = buf.iter().copied().max().unwrap_or(0);
+                                if let Err(e) = indexer::cursor_advance(&db, max).await {
+                                    warn!(error = %e, "final cursor flush failed");
+                                } else {
+                                    stats.last_cursor_persisted_us.store(max, Ordering::Relaxed);
+                                }
+                                buf.clear();
+                            }
+                            return;
+                        }
+                    }
+                }
+                _ = interval.tick() => {
+                    if buf.is_empty() { continue; }
+                    let max = buf.iter().copied().max().unwrap_or(0);
+                    if let Err(e) = indexer::cursor_advance(&db, max).await {
+                        warn!(error = %e, "cursor flush failed");
+                    } else {
+                        stats.last_cursor_persisted_us.store(max, Ordering::Relaxed);
+                    }
+                    buf.clear();
+                }
+            }
+        }
+    })
+}