Compare commits
17
Commits
ec8fe187fe
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
beab66648c | ||
|
|
73959f9dde | ||
|
|
f04d63dd7b | ||
|
|
b58cb75cfe | ||
|
|
b7ce114677 | ||
|
|
6fbea4fe6f | ||
|
|
2b695d6892 | ||
|
|
124a90dc07 | ||
|
|
d6947c2576 | ||
|
|
0646fbeebe | ||
|
|
6fd046417a | ||
|
|
f7b78fd5db | ||
|
|
9ee717bbc7 | ||
|
|
ac18ff7a16 | ||
|
|
73da8f0140 | ||
|
|
a2a371b7d9 | ||
|
|
786a892658 |
@@ -61,9 +61,45 @@ S3_BUCKET_APPVIEW=maarcadetweet-appview
|
||||
PLC_DIRECTORY_URL=https://plc.directory
|
||||
# PLC_DIRECTORY_URL=http://127.0.0.1:2582
|
||||
|
||||
# --- Registrierung ---
|
||||
# Verlangt createAccount einen Einladungscode? Default false, damit Dev-
|
||||
# Instanzen und die Integrationstests frei Konten anlegen können — eine
|
||||
# öffentlich erreichbare PDS gehört auf true gestellt, sonst kann jeder
|
||||
# beliebig viele Repos anlegen. Der Server warnt beim Start, solange es
|
||||
# aus ist. Codes erzeugen: `pds-server invite create --count 5 --uses 1`
|
||||
PDS_INVITE_REQUIRED=false
|
||||
|
||||
# --- AppView ingest auth (optional, dev ok if unset) ---
|
||||
# Wenn gesetzt, muss die PDS denselben Wert als Header
|
||||
# `X-Ingest-Secret` mitschicken; ist er nicht gesetzt, nimmt
|
||||
# /internal/ingest-commit unauthentifizierte Writes entgegen (jeder,
|
||||
# der den Port erreicht, kann Posts/Follows/Notifications fälschen).
|
||||
# Die AppView warnt beim Start, solange er fehlt.
|
||||
# APPVIEW_INGEST_SECRET=change-me-to-a-shared-secret-between-pds-and-appview
|
||||
|
||||
# --- AppView auth (Bearer-Token der PDS) ---
|
||||
# Erzwingt ein gültiges Access-JWT auf den privaten Endpoints
|
||||
# (/api/notifications, /api/notifications/count,
|
||||
# /api/notifications/seen, /api/timeline/home); `sub` im Token muss
|
||||
# dem `did`-Parameter entsprechen, sonst 403. Default: true.
|
||||
# Den öffentlichen P-256-Schlüssel holt sich die AppView von
|
||||
# `PDS_INTERNAL_URL` (sonst `PDS_PUBLIC_URL`) unter
|
||||
# /.well-known/did.json — `PDS_JWT_SECRET` verlässt die PDS nie.
|
||||
# Auf `false` verhält sich die AppView wie vor der Auth-Einführung
|
||||
# (alles öffentlich): nötig für die fail-open-Integrationstests mit
|
||||
# synthetischen DIDs und für eine Instanz, die schon per VPN
|
||||
# abgeschottet ist. Die AppView warnt beim Start laut, wenn er aus ist.
|
||||
# APPVIEW_AUTH_REQUIRED=true
|
||||
# Kommaseparierte Allowlist der Browser-Origins, die `/api/*` aufrufen
|
||||
# dürfen. Nicht gesetzt = bisheriges Verhalten
|
||||
# (`Access-Control-Allow-Origin: *`) plus Startup-Warnung.
|
||||
# Der Tauri-Webview hat je nach Plattform eine eigene Origin:
|
||||
# macOS/iOS `tauri://localhost`, Windows `http://tauri.localhost`,
|
||||
# im Dev-Modus der Vite-Server `http://127.0.0.1:1430`.
|
||||
# `Authorization` steht immer in den erlaubten Headern (sonst
|
||||
# scheitert der Preflight), `allow_credentials` bleibt aus.
|
||||
# APPVIEW_CORS_ORIGINS=tauri://localhost,http://127.0.0.1:1430
|
||||
|
||||
# --- Tauri-Client (Build-/Laufzeit-Overrides des Desktop-Clients) ---
|
||||
# MAARCADETWEET_PDS_URL=http://127.0.0.1:2583
|
||||
# MAARCADETWEET_APPVIEW_URL=http://127.0.0.1:2584
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
name: Release Desktop Client
|
||||
|
||||
# Baut den Tauri-Client aus crates/tauri-app fuer Windows und Linux, laedt die
|
||||
# Bundles als Job-Artefakte hoch und haengt sie an ein Gitea-Release zum Tag.
|
||||
#
|
||||
# ACHTUNG — Verzeichnis-Falle (hat im Nachbarprojekt lserver einen Tag CI
|
||||
# gekostet): Sobald `.gitea/workflows/` existiert, ignoriert Gitea
|
||||
# `.github/workflows/` KOMPLETT. In diesem Repo gibt es kein `.github/`, also
|
||||
# ist heute nichts betroffen — wer aber spaeter einen Workflow unter
|
||||
# `.github/workflows/` anlegt, bekommt keinen Lauf und auch keinen roten
|
||||
# Fehler, sondern schlicht Stille. Alle Workflows gehoeren hierher.
|
||||
#
|
||||
# VOR DEM TAG VERSION BUMPEN (analog zu lserver, wo package.json gebumpt wird):
|
||||
# Die Version im Release kommt aus dem Tag, die Version IM Artefaktnamen aus
|
||||
# der Config. Beide muessen zusammenpassen, sonst heisst die Datei zu einem
|
||||
# Tag v0.2.0 weiterhin `maarcadetweet_0.1.0_x64-setup.exe`:
|
||||
# * crates/tauri-app/src-tauri/tauri.conf.json -> "version"
|
||||
# * crates/tauri-app/src-tauri/Cargo.toml -> [package] version
|
||||
# * crates/tauri-app/package.json -> "version" (Konsistenz)
|
||||
# Details: docs/tauri-release.md, Abschnitt "Release-Checkliste".
|
||||
#
|
||||
# macOS: DAFUER GIBT ES KEINEN RUNNER. Weder Gitea-Instanz noch Infrastruktur
|
||||
# haben einen macOS-Host; .dmg/.app werden lokal gebaut und von Hand an das
|
||||
# hier erzeugte Release gehaengt:
|
||||
# cd crates/tauri-app && npm ci && npm run tauri -- build --ci
|
||||
# # Artefakte: src-tauri/target/release/bundle/dmg/*.dmg und macos/*.app
|
||||
# # Universal-Build: npm run tauri -- build --ci --target universal-apple-darwin
|
||||
# Danach im Gitea-Release "Edit release" -> Dateien anhaengen. Ohne
|
||||
# Notarisierung meldet Gatekeeper die App als nicht verifiziert (siehe
|
||||
# docs/tauri-release.md, Abschnitt 9 "Offene Punkte").
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*.*.*"
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
windows:
|
||||
name: Windows (MSI + NSIS)
|
||||
# Label `windows` = act_runner auf der Build-VM winbuild (192.168.1.69),
|
||||
# Win11, cargo 1.98.1, Node 24, Tauri-CLI 2.11.4. Laut Infrastruktur-Doku
|
||||
# on-demand — laeuft die VM nicht, wird der Job nie geplant (Gitea zeigt
|
||||
# dann gar keinen Lauf an, keinen fehlgeschlagenen).
|
||||
runs-on: windows
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: powershell
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# package-lock.json liegt unter crates/tauri-app/ -> npm ci (reproduzierbar).
|
||||
# Kein `npm install`: das wuerde den Lock im Build veraendern.
|
||||
- name: Install frontend dependencies
|
||||
working-directory: crates/tauri-app
|
||||
run: npm ci
|
||||
|
||||
# package.json definiert `"tauri": "tauri"` — der npm-Umweg nutzt die
|
||||
# @tauri-apps/cli-devDependency aus dem Lock (2.x) statt einer global
|
||||
# installierten `cargo tauri`-Version, ist also an das Repo gebunden.
|
||||
# `cargo tauri build --ci` waere gleichwertig, haengt aber an dem, was
|
||||
# gerade auf dem Runner installiert ist.
|
||||
# bundle.targets in tauri.conf.json steht auf "all" -> unter Windows
|
||||
# heisst das msi + nsis.
|
||||
- name: Build Tauri bundles
|
||||
working-directory: crates/tauri-app
|
||||
run: npm run tauri -- build --ci
|
||||
|
||||
- name: Bundles auflisten
|
||||
working-directory: crates/tauri-app
|
||||
run: |
|
||||
$bundle = "src-tauri\target\release\bundle"
|
||||
Get-ChildItem -Path $bundle -Recurse -Include *.msi, *.exe |
|
||||
ForEach-Object { Write-Host "$($_.FullName) ($([math]::Round($_.Length / 1MB)) MB)" }
|
||||
|
||||
- name: Upload bundles
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: maarcadetweet-windows
|
||||
path: |
|
||||
crates/tauri-app/src-tauri/target/release/bundle/msi/*.msi
|
||||
crates/tauri-app/src-tauri/target/release/bundle/nsis/*.exe
|
||||
retention-days: 30
|
||||
|
||||
- name: Gitea-Release anlegen und Bundles anhaengen
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$api = "$env:GITHUB_SERVER_URL/api/v1/repos/$env:GITHUB_REPOSITORY"
|
||||
$tag = $env:GITHUB_REF_NAME
|
||||
$auth = "Authorization: token $env:GITEA_TOKEN"
|
||||
|
||||
# Release-Notes aus dem passenden CHANGELOG-Abschnitt ziehen.
|
||||
# CHANGELOG.md existiert in diesem Repo noch nicht — sobald es
|
||||
# angelegt wird (Format `## [0.2.0] - ...` wie bei lserver), landet
|
||||
# der Abschnitt automatisch im Release.
|
||||
$notes = "Automatisch gebaut aus $env:GITHUB_SHA."
|
||||
$version = $tag.TrimStart("v")
|
||||
if (Test-Path CHANGELOG.md) {
|
||||
# Ausdruecklich als UTF-8 lesen: Windows PowerShell 5.1 nimmt sonst
|
||||
# die ANSI-Codepage und macht aus "Aenderungen" Buchstabensalat.
|
||||
$lines = [IO.File]::ReadAllText((Resolve-Path CHANGELOG.md), [Text.UTF8Encoding]::new($false)) -split "`r?`n"
|
||||
$start = ($lines | Select-String -Pattern "^## \[$([regex]::Escape($version))\]" | Select-Object -First 1)
|
||||
if ($start) {
|
||||
$from = $start.LineNumber
|
||||
$rest = $lines[$from..($lines.Count - 1)]
|
||||
$next = ($rest | Select-String -Pattern "^## \[" | Select-Object -First 1)
|
||||
$take = if ($next) { $next.LineNumber - 2 } else { $rest.Count - 1 }
|
||||
if ($take -ge 0) { $notes = ($rest[0..$take] -join "`n").Trim() }
|
||||
}
|
||||
}
|
||||
|
||||
$body = @{ tag_name = $tag; name = "maarcadetweet $tag"; body = $notes } | ConvertTo-Json -Depth 3
|
||||
$bodyFile = Join-Path $env:RUNNER_TEMP "release.json"
|
||||
[IO.File]::WriteAllText($bodyFile, $body, [Text.UTF8Encoding]::new($false))
|
||||
|
||||
# Beide Jobs (windows + linux) haengen an DASSELBE Release und laufen
|
||||
# parallel. Deshalb: anlegen versuchen, und wenn das scheitert (der
|
||||
# andere Job war schneller, oder es ist ein Re-Run), das vorhandene
|
||||
# Release per Tag holen.
|
||||
$created = curl.exe -s -X POST -H $auth -H "Content-Type: application/json" --data-binary "@$bodyFile" "$api/releases" | ConvertFrom-Json
|
||||
if (-not $created.id) {
|
||||
$created = curl.exe -s -H $auth "$api/releases/tags/$tag" | ConvertFrom-Json
|
||||
}
|
||||
if (-not $created.id) { throw "Konnte kein Release fuer $tag anlegen oder finden." }
|
||||
|
||||
$bundle = "crates\tauri-app\src-tauri\target\release\bundle"
|
||||
$files = Get-ChildItem -Path $bundle -Recurse -Include *.msi, *.exe
|
||||
if (-not $files) { throw "Keine Windows-Bundles unter $bundle gefunden." }
|
||||
foreach ($f in $files) {
|
||||
$name = [Uri]::EscapeDataString($f.Name)
|
||||
curl.exe -s -o NUL -w "Asset-Upload $($f.Name): HTTP %{http_code}`n" -X POST -H $auth -F "attachment=@$($f.FullName)" "$api/releases/$($created.id)/assets?name=$name"
|
||||
}
|
||||
Write-Host "Release: $env:GITHUB_SERVER_URL/$env:GITHUB_REPOSITORY/releases/tag/$tag"
|
||||
|
||||
linux:
|
||||
name: Linux (deb + rpm + AppImage)
|
||||
# NICHT `ubuntu-latest` — auf diesem Runner (VM ci-runner, 192.168.1.72)
|
||||
# ist dieses Label auf `docker://node:22-bookworm` gemappt, also einen
|
||||
# Container mit Node, aber ohne Rust und ohne GTK. Der Job braeche dort
|
||||
# bei `cargo` ab. Nachgesehen in /var/lib/gitea-runner/.runner:
|
||||
# labels: ['ubuntu-latest:docker://node:22-bookworm', 'linux-amd64:host']
|
||||
# `linux-amd64` ist das Host-Label, und auf dem Host liegen Rust
|
||||
# (/root/.cargo/bin, auch ohne Login-Shell im PATH), Node 22 und die
|
||||
# Tauri-GTK-Deps. Der Runner-Dienst laeuft als root.
|
||||
runs-on: linux-amd64
|
||||
timeout-minutes: 60
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
env:
|
||||
# linuxdeploy/appimagetool werden als AppImage aus ~/.cache/tauri
|
||||
# gestartet und brauchen sonst FUSE, was in der VM nicht zuverlaessig
|
||||
# funktioniert (siehe MAARCADE-INFRASTRUKTUR.md: "AppImage scheitert an
|
||||
# linuxdeploy/FUSE"). Mit dieser Variable entpacken sie sich selbst.
|
||||
APPIMAGE_EXTRACT_AND_RUN: "1"
|
||||
# Das Release-Profil in src-tauri/Cargo.toml strippt bereits selbst;
|
||||
# linuxdeploys eigener strip-Lauf ist dann nur eine weitere Fehlerquelle.
|
||||
NO_STRIP: "true"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Toolchain melden
|
||||
run: |
|
||||
node --version
|
||||
npm --version
|
||||
cargo --version || echo "cargo fehlt im PATH — ggf. ~/.cargo/env sourcen"
|
||||
|
||||
- name: Install frontend dependencies
|
||||
working-directory: crates/tauri-app
|
||||
run: npm ci
|
||||
|
||||
# bundle.targets in tauri.conf.json ist "all" -> unter Linux sind das
|
||||
# deb, rpm und appimage. Bewusst in zwei Aufrufe getrennt: der
|
||||
# AppImage-Schritt ist der fragile (Downloads von linuxdeploy +
|
||||
# appimagetool beim ersten Lauf, FUSE), und wenn er faellt, sollen deb
|
||||
# und rpm trotzdem im Release landen. Der zweite Aufruf ist billig — der
|
||||
# Cargo-Release-Build ist dann schon im target/-Cache.
|
||||
- name: Build Tauri bundles (deb + rpm)
|
||||
working-directory: crates/tauri-app
|
||||
run: npm run tauri -- build --ci --bundles deb,rpm
|
||||
|
||||
- name: Build Tauri bundle (AppImage)
|
||||
working-directory: crates/tauri-app
|
||||
continue-on-error: true
|
||||
run: npm run tauri -- build --ci --bundles appimage
|
||||
|
||||
- name: Bundles auflisten
|
||||
run: |
|
||||
find crates/tauri-app/src-tauri/target/release/bundle \
|
||||
\( -name '*.deb' -o -name '*.rpm' -o -name '*.AppImage' \) \
|
||||
-printf '%p (%kK)\n' || true
|
||||
|
||||
- name: Upload bundles
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: maarcadetweet-linux
|
||||
path: |
|
||||
crates/tauri-app/src-tauri/target/release/bundle/deb/*.deb
|
||||
crates/tauri-app/src-tauri/target/release/bundle/rpm/*.rpm
|
||||
crates/tauri-app/src-tauri/target/release/bundle/appimage/*.AppImage
|
||||
# AppImage darf fehlen (siehe continue-on-error oben), deb/rpm nicht —
|
||||
# ein komplett leerer Upload soll auffallen.
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Gitea-Release anlegen und Bundles anhaengen
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
api="${GITHUB_SERVER_URL}/api/v1/repos/${GITHUB_REPOSITORY}"
|
||||
tag="${GITHUB_REF_NAME}"
|
||||
version="${tag#v}"
|
||||
|
||||
# Gleiche CHANGELOG-Logik wie im Windows-Job. Die Datei existiert
|
||||
# heute nicht; ohne sie bleibt es beim Fallback-Text.
|
||||
notes="Automatisch gebaut aus ${GITHUB_SHA}."
|
||||
if [ -f CHANGELOG.md ]; then
|
||||
section="$(awk -v v="$version" '
|
||||
/^## \[/ { if (found) exit; if ($0 ~ "^## \\[" v "\\]") { found = 1; next } }
|
||||
found { print }
|
||||
' CHANGELOG.md)"
|
||||
if [ -n "$(printf '%s' "$section" | tr -d '[:space:]')" ]; then
|
||||
notes="$section"
|
||||
fi
|
||||
fi
|
||||
|
||||
# jq ist auf dem Runner nicht garantiert, Node 22 schon.
|
||||
payload="${RUNNER_TEMP}/release.json"
|
||||
NOTES="$notes" TAG="$tag" node -e '
|
||||
const fs = require("fs");
|
||||
fs.writeFileSync(process.argv[1], JSON.stringify({
|
||||
tag_name: process.env.TAG,
|
||||
name: `maarcadetweet ${process.env.TAG}`,
|
||||
body: process.env.NOTES,
|
||||
}));
|
||||
' "$payload"
|
||||
|
||||
# Anlegen oder — falls der Windows-Job schneller war bzw. das Release
|
||||
# vom Re-Run schon existiert — das vorhandene holen.
|
||||
id="$(curl -s -X POST -H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-H "Content-Type: application/json" --data-binary "@${payload}" \
|
||||
"${api}/releases" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{process.stdout.write(String(JSON.parse(s).id||""))}catch{}})')"
|
||||
if [ -z "$id" ]; then
|
||||
id="$(curl -s -H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${api}/releases/tags/${tag}" | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{process.stdout.write(String(JSON.parse(s).id||""))}catch{}})')"
|
||||
fi
|
||||
if [ -z "$id" ]; then
|
||||
echo "Konnte kein Release fuer ${tag} anlegen oder finden." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
bundle="crates/tauri-app/src-tauri/target/release/bundle"
|
||||
found=0
|
||||
while IFS= read -r f; do
|
||||
found=1
|
||||
name="$(basename "$f")"
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
-F "attachment=@${f}" \
|
||||
"${api}/releases/${id}/assets?name=${name}")"
|
||||
echo "Asset-Upload ${name}: HTTP ${code}"
|
||||
done < <(find "$bundle" \( -name '*.deb' -o -name '*.rpm' -o -name '*.AppImage' \) | sort)
|
||||
[ "$found" -eq 1 ] || { echo "Keine Linux-Bundles unter ${bundle} gefunden." >&2; exit 1; }
|
||||
|
||||
echo "Release: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/releases/tag/${tag}"
|
||||
Generated
+9
@@ -51,14 +51,18 @@ dependencies = [
|
||||
"axum",
|
||||
"base64",
|
||||
"chrono",
|
||||
"ciborium",
|
||||
"cid",
|
||||
"dotenvy",
|
||||
"futures",
|
||||
"p256",
|
||||
"reqwest",
|
||||
"rustls",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlx",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
@@ -286,6 +290,7 @@ checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"base64",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
@@ -304,8 +309,10 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sha1",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
@@ -1884,6 +1891,7 @@ dependencies = [
|
||||
"ciborium",
|
||||
"cid",
|
||||
"dotenvy",
|
||||
"futures",
|
||||
"hex",
|
||||
"k256",
|
||||
"p256",
|
||||
@@ -1894,6 +1902,7 @@ dependencies = [
|
||||
"sha2",
|
||||
"sqlx",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
|
||||
@@ -25,6 +25,7 @@ crates/tauri-app/ Tauri 2 + Svelte 5 + Vite + TS Desktop-Client
|
||||
└── src-tauri/ Rust-IPC-Layer
|
||||
|
||||
lexicons/app/twi/post.json Custom Lexicon mit maxLength: 160
|
||||
lexicons/app/bsky/ like, repost, follow, actor.profile
|
||||
migrations/pds/ PDS-DB-Schema (users, repos, blobs, sessions, plc_ops)
|
||||
migrations/appview/ AppView-DB-Schema (posts, likes, follows, notifications, profiles, jetstream_cursor)
|
||||
docs/ Deployment, Architektur, Tauri-Release (siehe unten)
|
||||
@@ -68,6 +69,8 @@ cargo run -p appview
|
||||
| 6 Tauri-UI-Logik an Backend koppeln | ✅ done — LoginScreen, NavRail, PostCard, ComposeBox, Profile/Compose/Search/Settings-Views |
|
||||
| 7 Polish (Tray, Notifications, Auto-Update) | ✅ done — Tray-Icon custom (`tauri::include_image!`), Notification-Click navigiert via `app://notification`-Event + `openThread`-Helper zu Thread-Detail, Auto-Update in Dev inert (Production-Weg: [`docs/tauri-release.md`](docs/tauri-release.md)) |
|
||||
| 8 Social-Graph + Benachrichtigungen | ✅ done — `notifications`-Tabelle, Schreibpfad im Jetstream-Indexer (idempotent, keine Selbst-Notifications), `/api/notifications[/count|/seen]`, `/api/followers`, `/api/following`, eigene `/api/thread`-Route; im Client Notifications-View mit Unread-Badge und klickbare Follower-/Following-Listen im Profil |
|
||||
| 9 Auth + Performance | ✅ done — AppView prüft Bearer-Tokens (ES256, Schlüssel aus dem neuen `/.well-known/did.json` der PDS, fail closed); Timeline und Notifications nur noch für die eigene DID; CORS-Allowlist statt `Any`; Indizes für Handle-Lookup und Cold-Start-Feed |
|
||||
| 10 Lokaler Firehose | ✅ done — `com.atproto.sync.subscribeRepos` auf der PDS (Event in derselben Transaktion wie der Commit, `seq`-Cursor mit lückenfreiem Replay, WebSocket-Frames in atproto-Form); die AppView konsumiert ihn mit persistiertem Cursor. Ein verlorener Push ist damit nicht mehr endgültig. |
|
||||
|
||||
## Tests
|
||||
|
||||
@@ -81,11 +84,13 @@ Stand zuletzt gegen den lokalen Dev-Stack (docker compose + laufender PDS + AppV
|
||||
Rust-Workspace grün (u.a. 27 MST, 24 PDS-Integration, 49 AppView-Lib, 14 AppView-Integration),
|
||||
Frontend grün. Zwei Vorbehalte:
|
||||
|
||||
* Die DB-gestützten Integrationstests sind *fail-open* — ohne erreichbare Postgres/PDS
|
||||
überspringen sie sich selbst und melden das nur auf stderr. Ein grüner Lauf ohne
|
||||
laufenden Stack sagt also weniger, als er aussieht.
|
||||
* Einige Tests hängen am Zustand der Dev-Datenbank; auf einer frischen DB können
|
||||
`handle_sync`-Tests abweichen. Wer sie ernst nimmt, prüft sie gegen eine definierte DB.
|
||||
* Die DB-gestützten Tests sind *fail-open*: **ohne `DATABASE_URL_APPVIEW` in der
|
||||
Umgebung überspringen sie sich selbst** und melden das nur auf stderr. `cargo test
|
||||
--workspace` in einer nackten Shell meldet dann grün, ohne sie ausgeführt zu haben —
|
||||
für einen aussagekräftigen Lauf `set -a; . ./.env; set +a` voranstellen und PDS +
|
||||
AppView laufen lassen.
|
||||
* Läuft die Auth (Default), holen sich die Integrationstests echte Tokens von der PDS
|
||||
bzw. signieren sie aus `PDS_JWT_SECRET`; ohne erreichbare PDS überspringen sie.
|
||||
|
||||
`crates/tauri-app/src-tauri` hat ein eigenes `[workspace]` und ist **nicht** Teil des
|
||||
Root-Workspace; `cargo test --workspace` von oben erfasst den IPC-Layer nicht.
|
||||
@@ -100,11 +105,14 @@ Root-Workspace; `cargo test --workspace` von oben erfasst den IPC-Layer nicht.
|
||||
|
||||
## Bekannte Lücken
|
||||
|
||||
* Die eigene PDS speist **keinen** Firehose (`com.atproto.sync.subscribeRepos` fehlt) —
|
||||
eigene Records erreichen die AppView nur über den Best-Effort-Push
|
||||
`POST /internal/ingest-commit`.
|
||||
* Die AppView-Leseschnittstelle hat **keine Auth** und CORS `Any`; bei
|
||||
`/api/notifications` sind das erstmals halbwegs private Daten.
|
||||
* Der Firehose ist **lokal**: er verbindet die eigenen zwei Dienste. Ein fremder
|
||||
Relay erfährt von dieser PDS weiterhin nichts.
|
||||
* Die Frame-Hülle ist spec-konformes DAG-CBOR, die Blöcke darin nicht: CIDs
|
||||
innerhalb von Commit-Blöcken sind Strings statt Tag-42-Links. Ein fremder
|
||||
atproto-Consumer liest die Frames, scheitert aber an den Blockinhalten. Das
|
||||
zu ändern hieße, jede CID im System zu ändern — inklusive der
|
||||
`did:plc:`-Ableitung.
|
||||
* `firehose_events` wird nie beschnitten.
|
||||
* Notifications werden nie gelöscht: Unlike/Unfollow lässt die Zeile stehen, und der
|
||||
Dedupe-Key macht sie „einmal pro (Empfänger, Autor, Art, Subject) für immer".
|
||||
* Auto-Update ist nur dokumentiert, nicht verdrahtet: niemand ruft `check()` auf, das
|
||||
@@ -112,8 +120,6 @@ Root-Workspace; `cargo test --workspace` von oben erfasst den IPC-Layer nicht.
|
||||
* Reply-Notifications gehen verloren, wenn die Antwort vor ihrem Parent indiziert wird
|
||||
(kein Nachlauf) — bei Jetstream möglich.
|
||||
* `at-blob` spricht MinIO ohne Signature V4 — echtes AWS S3 funktioniert damit nicht.
|
||||
* Die PDS liefert kein `.well-known/did.json`; `describeServer` gibt die DID hart
|
||||
als `did:web:pds.maarcadetweet.local` zurück.
|
||||
|
||||
## Design
|
||||
|
||||
|
||||
@@ -35,13 +35,27 @@ at-shared = { workspace = true }
|
||||
at-firehose = { workspace = true }
|
||||
at-crypto = { workspace = true }
|
||||
at-identity = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
rustls = { version = "0.23", default-features = false, features = ["ring", "std", "logging", "tls12"] }
|
||||
base64 = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
# The local PDS firehose (`src/pds_firehose.rs`): a WebSocket carrying
|
||||
# DAG-CBOR frames whose `blocks` field is a CAR of record blocks.
|
||||
# `tokio-tungstenite` for the socket, `cid` for the block addresses,
|
||||
# `ciborium` for the record blocks themselves (they are written with
|
||||
# `ciborium::into_writer` on the PDS side, so it is their exact inverse).
|
||||
# The frame envelope is decoded by `src/cbor.rs`, which needs no crate.
|
||||
tokio-tungstenite = { workspace = true }
|
||||
cid = { workspace = true }
|
||||
ciborium = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
# Unit tests in `src/auth.rs` build a P-256 verification key in the
|
||||
# same `0x8012 + uncompressed point` encoding the PDS publishes, which
|
||||
# needs the curve's `ToEncodedPoint`.
|
||||
p256 = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,735 @@
|
||||
//! Bearer-token authentication for the AppView's private endpoints.
|
||||
//!
|
||||
//! ## What is being proven
|
||||
//!
|
||||
//! The PDS already issues an ES256 access JWT on
|
||||
//! `com.atproto.server.createSession` / `createAccount`. That token
|
||||
//! carries `sub = <did>`, `scope = "com.atproto.access"` and
|
||||
//! `aud = "did:web:appview.maarcadetweet.local"` — it was always meant
|
||||
//! to be presented *here*. All the AppView has to do is verify the
|
||||
//! signature and read `sub`.
|
||||
//!
|
||||
//! ## Where the key comes from
|
||||
//!
|
||||
//! Verifying an ES256 signature needs only the public half of the PDS's
|
||||
//! P-256 key. The PDS publishes it in its DID document at
|
||||
//! `GET /.well-known/did.json` (see `pds-server/src/main.rs`), so
|
||||
//! `PDS_JWT_SECRET` never leaves the PDS process. We fetch that
|
||||
//! document once, cache the `publicKeyMultibase` behind an `RwLock`,
|
||||
//! and re-fetch on a verification failure so an operator can rotate
|
||||
//! `PDS_JWT_SECRET` without restarting the AppView.
|
||||
//!
|
||||
//! Two failure modes are deliberately handled differently:
|
||||
//!
|
||||
//! - **PDS unreachable at startup.** The AppView has always booted
|
||||
//! independently of the PDS (it indexes the firehose, which has
|
||||
//! nothing to do with the local PDS), and a crash-loop on a
|
||||
//! colocated service that happens to boot second would be a
|
||||
//! self-inflicted outage. So startup only *warns*; the first
|
||||
//! authenticated request retries the fetch.
|
||||
//! - **PDS unreachable when a token must be checked.** There is no
|
||||
//! safe way to guess, so the request gets a `503`. Fail closed —
|
||||
//! never fail open.
|
||||
//!
|
||||
//! ## Error contract
|
||||
//!
|
||||
//! The body shape matches every other AppView error
|
||||
//! (`{"error": …, "message": …}`). The `error` codes are load-bearing:
|
||||
//!
|
||||
//! | case | status | `error` |
|
||||
//! |----------------------------------------|--------|-------------------|
|
||||
//! | no / malformed `Authorization` header | 401 | `AuthMissing` |
|
||||
//! | bad signature, expired, wrong `scope` | 401 | `TokenInvalid` |
|
||||
//! | valid token, but `sub` ≠ requested did | 403 | `Forbidden` |
|
||||
//! | PDS key not obtainable | 503 | `AuthUnavailable` |
|
||||
//!
|
||||
//! **`TokenInvalid` is a contract with the desktop client**: seeing it,
|
||||
//! the client refreshes its access JWT (`com.atproto.server.refreshSession`)
|
||||
//! and retries the request once. Renaming it silently logs every user
|
||||
//! out an hour after login.
|
||||
|
||||
use at_crypto::jwt::JwtClaims;
|
||||
use axum::async_trait;
|
||||
use axum::extract::FromRequestParts;
|
||||
use axum::http::request::Parts;
|
||||
use axum::http::{HeaderMap, StatusCode};
|
||||
use axum::Json;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
/// The scope an *access* token carries. Refresh tokens are minted with
|
||||
/// `com.atproto.refresh` by the same key, so without this check a
|
||||
/// refresh token — which lives for 90 days instead of an hour — would
|
||||
/// be accepted as a session credential everywhere.
|
||||
const ACCESS_SCOPE: &str = "com.atproto.access";
|
||||
|
||||
/// Minimum spacing between two key re-fetches triggered by a failed
|
||||
/// verification.
|
||||
///
|
||||
/// Re-fetching on failure is what makes key rotation work without a
|
||||
/// restart. Doing it on *every* failure would also hand anyone who can
|
||||
/// reach the AppView a free amplifier: a stream of garbage tokens
|
||||
/// becomes a stream of requests to the PDS. One re-fetch per minute is
|
||||
/// far quicker than any plausible rotation cadence needs and costs the
|
||||
/// PDS nothing.
|
||||
const KEY_REFETCH_MIN_INTERVAL: Duration = Duration::from_secs(60);
|
||||
|
||||
/// HTTP timeout for the DID-document fetch. The PDS is colocated; if it
|
||||
/// doesn't answer in two seconds it isn't answering, and a request
|
||||
/// blocked on auth is a request the user is staring at.
|
||||
const DID_DOC_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
// -- error type --------------------------------------------------------------
|
||||
|
||||
/// Everything that can go wrong between "a request arrived" and "we know
|
||||
/// which DID it belongs to". Converted into the AppView's standard error
|
||||
/// body by [`AuthError::into_response_parts`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AuthError {
|
||||
/// No `Authorization` header, or one that isn't `Bearer <token>`.
|
||||
Missing,
|
||||
/// Signature, expiry or scope check failed. The client's cue to
|
||||
/// refresh its access token and retry.
|
||||
Invalid(String),
|
||||
/// Authenticated fine, but the token belongs to somebody else.
|
||||
Forbidden,
|
||||
/// We could not obtain the PDS's public key, so we cannot decide.
|
||||
Unavailable(String),
|
||||
}
|
||||
|
||||
impl AuthError {
|
||||
pub fn into_response_parts(self) -> (StatusCode, Json<Value>) {
|
||||
let (status, code, message) = match self {
|
||||
AuthError::Missing => (
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"AuthMissing",
|
||||
"missing Authorization: Bearer header".to_string(),
|
||||
),
|
||||
AuthError::Invalid(detail) => {
|
||||
(StatusCode::UNAUTHORIZED, "TokenInvalid", detail)
|
||||
}
|
||||
AuthError::Forbidden => (
|
||||
StatusCode::FORBIDDEN,
|
||||
"Forbidden",
|
||||
"token sub does not match the requested did".to_string(),
|
||||
),
|
||||
AuthError::Unavailable(detail) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"AuthUnavailable",
|
||||
format!("cannot verify tokens: {detail}"),
|
||||
),
|
||||
};
|
||||
(
|
||||
status,
|
||||
Json(json!({
|
||||
"error": code,
|
||||
"message": message,
|
||||
})),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AuthError> for (StatusCode, Json<Value>) {
|
||||
fn from(e: AuthError) -> Self {
|
||||
e.into_response_parts()
|
||||
}
|
||||
}
|
||||
|
||||
// -- key cache ---------------------------------------------------------------
|
||||
|
||||
#[derive(Default)]
|
||||
struct CachedKey {
|
||||
multibase: Option<String>,
|
||||
/// When we last *attempted* a fetch — successful or not. Throttles
|
||||
/// the on-failure refresh path; see [`KEY_REFETCH_MIN_INTERVAL`].
|
||||
last_attempt: Option<Instant>,
|
||||
}
|
||||
|
||||
/// The PDS signing key, lazily fetched and cached.
|
||||
///
|
||||
/// Lives in [`AppState`] behind an `Arc`, so all handlers share one
|
||||
/// cache and one HTTP client.
|
||||
pub struct PdsKeys {
|
||||
http: reqwest::Client,
|
||||
/// Fully-qualified URL of the PDS's DID document.
|
||||
did_doc_url: String,
|
||||
/// The `aud` every access token must carry: this AppView's own
|
||||
/// service DID. See [`verify_with_key`] for why it's checked.
|
||||
expected_aud: String,
|
||||
inner: RwLock<CachedKey>,
|
||||
}
|
||||
|
||||
impl PdsKeys {
|
||||
/// Build a cache pointed at `base_url` (no trailing slash required),
|
||||
/// accepting only tokens addressed to `expected_aud`.
|
||||
pub fn new(base_url: &str, expected_aud: impl Into<String>) -> Self {
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(DID_DOC_TIMEOUT)
|
||||
.build()
|
||||
.expect("reqwest client build should never fail");
|
||||
Self {
|
||||
http,
|
||||
did_doc_url: format!(
|
||||
"{}/.well-known/did.json",
|
||||
base_url.trim_end_matches('/')
|
||||
),
|
||||
expected_aud: expected_aud.into(),
|
||||
inner: RwLock::new(CachedKey::default()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Same PDS the handle-sync worker talks to: `PDS_INTERNAL_URL`
|
||||
/// when set, else `PDS_PUBLIC_URL`.
|
||||
pub fn from_config(cfg: &at_shared::config::AppConfig) -> Self {
|
||||
Self::new(&cfg.pds_base_url(), cfg.appview_did())
|
||||
}
|
||||
|
||||
pub fn did_doc_url(&self) -> &str {
|
||||
&self.did_doc_url
|
||||
}
|
||||
|
||||
/// The cached key, if we have ever fetched one.
|
||||
pub async fn cached(&self) -> Option<String> {
|
||||
self.inner.read().await.multibase.clone()
|
||||
}
|
||||
|
||||
/// Fetch the DID document and replace the cached key.
|
||||
///
|
||||
/// Called once at startup (best effort), on the first authenticated
|
||||
/// request if startup failed, and — throttled — after a failed
|
||||
/// verification.
|
||||
pub async fn refresh(&self) -> anyhow::Result<String> {
|
||||
// Record the attempt before the await so two concurrent
|
||||
// failures can't both decide they're the first one.
|
||||
self.inner.write().await.last_attempt = Some(Instant::now());
|
||||
|
||||
let resp = self.http.get(&self.did_doc_url).send().await?;
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!(
|
||||
"{} returned {}",
|
||||
self.did_doc_url,
|
||||
resp.status().as_u16()
|
||||
);
|
||||
}
|
||||
let doc: Value = resp.json().await?;
|
||||
let key = extract_public_key_multibase(&doc)?;
|
||||
{
|
||||
let mut guard = self.inner.write().await;
|
||||
guard.multibase = Some(key.clone());
|
||||
}
|
||||
debug!(url = %self.did_doc_url, "loaded PDS signing key");
|
||||
Ok(key)
|
||||
}
|
||||
|
||||
/// The key to verify with, fetching it if we don't have one yet.
|
||||
async fn key_or_fetch(&self) -> Result<String, AuthError> {
|
||||
if let Some(k) = self.cached().await {
|
||||
return Ok(k);
|
||||
}
|
||||
self.refresh().await.map_err(|e| {
|
||||
warn!(url = %self.did_doc_url, error = %e, "PDS signing key unavailable");
|
||||
AuthError::Unavailable(format!(
|
||||
"PDS did document at {} not reachable: {e}",
|
||||
self.did_doc_url
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Re-fetch after a verification failure, at most once per
|
||||
/// [`KEY_REFETCH_MIN_INTERVAL`]. Returns the new key only when it
|
||||
/// actually differs from `stale` — re-running the same failing
|
||||
/// verification against an unchanged key proves nothing.
|
||||
async fn refetch_if_stale(&self, stale: &str) -> Option<String> {
|
||||
{
|
||||
let guard = self.inner.read().await;
|
||||
if let Some(last) = guard.last_attempt {
|
||||
if last.elapsed() < KEY_REFETCH_MIN_INTERVAL {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
match self.refresh().await {
|
||||
Ok(fresh) if fresh != stale => {
|
||||
warn!("PDS signing key changed; re-verifying with the rotated key");
|
||||
Some(fresh)
|
||||
}
|
||||
Ok(_) => None,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "PDS key re-fetch after verification failure failed");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify an access token and return its claims.
|
||||
///
|
||||
/// Retries exactly once against a freshly fetched key, so a rotated
|
||||
/// `PDS_JWT_SECRET` costs one extra HTTP round trip rather than a
|
||||
/// restart.
|
||||
pub async fn verify_access_token(&self, token: &str) -> Result<JwtClaims, AuthError> {
|
||||
let key = self.key_or_fetch().await?;
|
||||
match verify_with_key(token, &key, &self.expected_aud) {
|
||||
Ok(claims) => Ok(claims),
|
||||
Err(first) => {
|
||||
let Some(fresh) = self.refetch_if_stale(&key).await else {
|
||||
return Err(first);
|
||||
};
|
||||
verify_with_key(token, &fresh, &self.expected_aud).map_err(|_| first)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pull `verificationMethod[].publicKeyMultibase` out of a DID document.
|
||||
///
|
||||
/// We take the first entry that carries a `publicKeyMultibase` rather
|
||||
/// than insisting on a fragment name: the PDS writes `#atproto`, but a
|
||||
/// document served by a proxy or a future PDS version may order or name
|
||||
/// its methods differently, and any key in the document is a key the
|
||||
/// controller published for itself. A document with none is an error,
|
||||
/// not an empty key — silently caching `""` would turn every later
|
||||
/// verification into a confusing signature failure.
|
||||
fn extract_public_key_multibase(doc: &Value) -> anyhow::Result<String> {
|
||||
let methods = doc
|
||||
.get("verificationMethod")
|
||||
.and_then(|v| v.as_array())
|
||||
.ok_or_else(|| anyhow::anyhow!("did document has no verificationMethod array"))?;
|
||||
for m in methods {
|
||||
if let Some(k) = m.get("publicKeyMultibase").and_then(|v| v.as_str()) {
|
||||
if !k.is_empty() {
|
||||
return Ok(k.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
anyhow::bail!("did document has no verificationMethod with publicKeyMultibase")
|
||||
}
|
||||
|
||||
/// Signature + expiry + scope check against one specific key.
|
||||
///
|
||||
/// `verify_jwt` handles the ES256 signature and `exp` (with a 30 s
|
||||
/// leeway for clock skew); the scope check is ours, and it is the line
|
||||
/// that keeps a 90-day refresh token from working as a session
|
||||
/// credential.
|
||||
fn verify_with_key(
|
||||
token: &str,
|
||||
pubkey_multibase: &str,
|
||||
expected_aud: &str,
|
||||
) -> Result<JwtClaims, AuthError> {
|
||||
let claims = at_crypto::jwt::verify_jwt(token, pubkey_multibase)
|
||||
.map_err(|e| AuthError::Invalid(format!("invalid token: {e}")))?;
|
||||
match claims.scope.as_deref() {
|
||||
Some(ACCESS_SCOPE) => {}
|
||||
other => {
|
||||
return Err(AuthError::Invalid(format!(
|
||||
"token scope {:?} is not {ACCESS_SCOPE}",
|
||||
other.unwrap_or("<none>")
|
||||
)))
|
||||
}
|
||||
}
|
||||
// Audience. `at_crypto::jwt::verify_jwt` sets `validate_aud = false`
|
||||
// because it has no way of knowing who the caller is, so the check
|
||||
// belongs here.
|
||||
//
|
||||
// What it buys: the PDS signs tokens for *its* AppView. Without an
|
||||
// audience check, a token handed to any other service that trusts
|
||||
// the same PDS key would be replayable here — and, the other way
|
||||
// round, a token this AppView issued trust in could be replayed
|
||||
// there. It is the difference between "the PDS vouches for this
|
||||
// user" and "the PDS vouches for this user *talking to us*".
|
||||
//
|
||||
// A mismatch is `TokenInvalid` rather than `Forbidden` on purpose:
|
||||
// that is the code the desktop client refreshes on, so a
|
||||
// deployment that changes `APPVIEW_PUBLIC_URL` heals itself on the
|
||||
// next refresh instead of stranding every signed-in user.
|
||||
if claims.aud != expected_aud {
|
||||
return Err(AuthError::Invalid(format!(
|
||||
"token audience {:?} is not {expected_aud:?}",
|
||||
claims.aud
|
||||
)));
|
||||
}
|
||||
Ok(claims)
|
||||
}
|
||||
|
||||
/// Extract the bearer token from an `Authorization` header.
|
||||
///
|
||||
/// The scheme match is case-insensitive (RFC 7235 says it is) — some
|
||||
/// HTTP clients send `bearer`.
|
||||
fn bearer_token(headers: &HeaderMap) -> Result<String, AuthError> {
|
||||
let raw = headers
|
||||
.get("authorization")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.ok_or(AuthError::Missing)?;
|
||||
let (scheme, token) = raw.split_once(' ').ok_or(AuthError::Missing)?;
|
||||
if !scheme.eq_ignore_ascii_case("bearer") {
|
||||
return Err(AuthError::Missing);
|
||||
}
|
||||
let token = token.trim();
|
||||
if token.is_empty() {
|
||||
return Err(AuthError::Missing);
|
||||
}
|
||||
Ok(token.to_string())
|
||||
}
|
||||
|
||||
// -- extractor ---------------------------------------------------------------
|
||||
|
||||
/// The authenticated DID of the caller.
|
||||
///
|
||||
/// `None` means auth is switched off for this instance
|
||||
/// (`APPVIEW_AUTH_REQUIRED=false`) — *not* "anonymous but allowed
|
||||
/// through". When auth is on, this is always `Some` by construction:
|
||||
/// the extractor rejects the request otherwise.
|
||||
///
|
||||
/// Handlers must still call [`AuthedDid::ensure_matches`] with the DID
|
||||
/// the request asks about. Proving *who you are* is not the same as
|
||||
/// proving *whose notifications you may read*.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct AuthedDid(pub Option<String>);
|
||||
|
||||
impl AuthedDid {
|
||||
/// 403 unless the token's `sub` is the DID the request targets.
|
||||
///
|
||||
/// With auth disabled this is a no-op, which is exactly what
|
||||
/// `APPVIEW_AUTH_REQUIRED=false` means.
|
||||
pub fn ensure_matches(&self, did: &str) -> Result<(), (StatusCode, Json<Value>)> {
|
||||
match self.0.as_deref() {
|
||||
None => Ok(()),
|
||||
Some(sub) if sub == did => Ok(()),
|
||||
Some(_) => Err(AuthError::Forbidden.into_response_parts()),
|
||||
}
|
||||
}
|
||||
|
||||
/// The caller's DID, if authenticated.
|
||||
///
|
||||
/// No handler needs it yet — every private endpoint takes the DID
|
||||
/// as a parameter and compares it — but an endpoint that acts
|
||||
/// purely on "whoever is calling" would read it here instead of
|
||||
/// trusting a query parameter.
|
||||
#[allow(dead_code)]
|
||||
pub fn did(&self) -> Option<&str> {
|
||||
self.0.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FromRequestParts<AppState> for AuthedDid {
|
||||
type Rejection = (StatusCode, Json<Value>);
|
||||
|
||||
async fn from_request_parts(
|
||||
parts: &mut Parts,
|
||||
state: &AppState,
|
||||
) -> Result<Self, Self::Rejection> {
|
||||
if !state.cfg.appview_auth_required {
|
||||
// Opt-out instance. The startup log says so in capitals.
|
||||
return Ok(AuthedDid(None));
|
||||
}
|
||||
let token = bearer_token(&parts.headers).map_err(AuthError::into_response_parts)?;
|
||||
let claims = state
|
||||
.pds_keys
|
||||
.verify_access_token(&token)
|
||||
.await
|
||||
.map_err(AuthError::into_response_parts)?;
|
||||
Ok(AuthedDid(Some(claims.sub)))
|
||||
}
|
||||
}
|
||||
|
||||
// -- startup posture ---------------------------------------------------------
|
||||
|
||||
/// Log, once at boot, every way this instance is configured to be less
|
||||
/// strict than the defaults.
|
||||
///
|
||||
/// All three of these are legitimate configurations — a VPN-only
|
||||
/// deployment, a dev box, a single-machine setup — and all three are
|
||||
/// also exactly what an accidentally-public instance looks like. The
|
||||
/// only defence that survives a hurried deployment is a log line the
|
||||
/// operator cannot miss, so each one is a `warn!` naming the variable
|
||||
/// that turns it back on.
|
||||
pub fn log_startup_posture(cfg: &at_shared::config::AppConfig) {
|
||||
if !cfg.appview_auth_required {
|
||||
warn!(
|
||||
"APPVIEW_AUTH_REQUIRED=false — /api/notifications* and \
|
||||
/api/timeline/home are served to ANY caller for ANY did. \
|
||||
Only safe when this instance is unreachable from untrusted \
|
||||
networks."
|
||||
);
|
||||
}
|
||||
if cfg.appview_cors_origins.is_empty() {
|
||||
warn!(
|
||||
"APPVIEW_CORS_ORIGINS unset — sending Access-Control-Allow-Origin: * \
|
||||
so any web page can call this AppView from a browser. Set it to the \
|
||||
origins your client actually uses, e.g. \
|
||||
'tauri://localhost,http://127.0.0.1:1430'."
|
||||
);
|
||||
}
|
||||
if cfg.appview_ingest_secret.is_none() {
|
||||
warn!(
|
||||
"APPVIEW_INGEST_SECRET unset — POST /internal/ingest-commit accepts \
|
||||
unauthenticated writes into the index (anyone who can reach this port \
|
||||
can forge posts, follows and notifications). Set the same value here \
|
||||
and on the PDS."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// -- tests -------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::ecdsa::P256Keypair;
|
||||
use at_crypto::jwt::{issue_jwt, JwtClaims};
|
||||
use axum::http::HeaderValue;
|
||||
|
||||
/// A throwaway server key plus its `publicKeyMultibase`, in the
|
||||
/// exact encoding the PDS publishes: `0x8012` (P-256) followed by
|
||||
/// the *uncompressed* affine coordinates. `P256Keypair::generate`
|
||||
/// stores a compressed point, which `verify_jwt` cannot decode — so
|
||||
/// this mirrors `pds-server`'s `server_p256_keypair`.
|
||||
fn test_key() -> (P256Keypair, String) {
|
||||
let kp = P256Keypair::generate().unwrap();
|
||||
let vk = kp.verifying_key().unwrap();
|
||||
let pt = vk.to_encoded_point(false);
|
||||
let mut raw = vec![0x80u8, 0x12u8];
|
||||
raw.extend_from_slice(pt.x().unwrap());
|
||||
raw.extend_from_slice(pt.y().unwrap());
|
||||
let multibase = at_crypto::multibase_util::encode_b58btc(&raw);
|
||||
(kp, multibase)
|
||||
}
|
||||
|
||||
/// The audience the tests' AppView identifies as — what
|
||||
/// `AppConfig::appview_did()` would return for
|
||||
/// `APPVIEW_PUBLIC_URL=http://127.0.0.1:2584`.
|
||||
const TEST_AUD: &str = "did:web:127.0.0.1%3A2584";
|
||||
|
||||
fn mint(kp: &P256Keypair, did: &str, scope: &str, ttl_secs: i64) -> String {
|
||||
mint_for(kp, did, scope, ttl_secs, TEST_AUD)
|
||||
}
|
||||
|
||||
fn mint_for(
|
||||
kp: &P256Keypair,
|
||||
did: &str,
|
||||
scope: &str,
|
||||
ttl_secs: i64,
|
||||
aud: &str,
|
||||
) -> String {
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
issue_jwt(
|
||||
kp,
|
||||
&JwtClaims {
|
||||
iss: "did:web:127.0.0.1%3A2583".into(),
|
||||
sub: did.into(),
|
||||
aud: aud.into(),
|
||||
iat: now - 1,
|
||||
exp: now + ttl_secs,
|
||||
jti: None,
|
||||
scope: Some(scope.into()),
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn header_map(value: &str) -> HeaderMap {
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert("authorization", HeaderValue::from_str(value).unwrap());
|
||||
h
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_header_is_auth_missing() {
|
||||
assert_eq!(bearer_token(&HeaderMap::new()), Err(AuthError::Missing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_header_is_auth_missing() {
|
||||
// No scheme, wrong scheme, and an empty token all read as
|
||||
// "the client never presented a credential" — 401 AuthMissing,
|
||||
// not TokenInvalid, so the client re-authenticates instead of
|
||||
// burning a refresh round trip.
|
||||
assert_eq!(bearer_token(&header_map("abc.def.ghi")), Err(AuthError::Missing));
|
||||
assert_eq!(bearer_token(&header_map("Basic dXNlcjpwdw==")), Err(AuthError::Missing));
|
||||
assert_eq!(bearer_token(&header_map("Bearer ")), Err(AuthError::Missing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bearer_scheme_is_case_insensitive() {
|
||||
assert_eq!(bearer_token(&header_map("bearer tok")).unwrap(), "tok");
|
||||
assert_eq!(bearer_token(&header_map("Bearer tok")).unwrap(), "tok");
|
||||
}
|
||||
|
||||
/// A token minted for a different AppView must not work here, and
|
||||
/// must fail as `TokenInvalid` so the client refreshes rather than
|
||||
/// treating it as a permanent rejection.
|
||||
#[test]
|
||||
fn token_for_another_audience_is_rejected() {
|
||||
let (kp, mb) = test_key();
|
||||
let token = mint_for(
|
||||
&kp,
|
||||
"did:plc:alice",
|
||||
ACCESS_SCOPE,
|
||||
3600,
|
||||
"did:web:someone-elses-appview.example",
|
||||
);
|
||||
let err = verify_with_key(&token, &mb, TEST_AUD).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, AuthError::Invalid(ref m) if m.contains("audience")),
|
||||
"expected an audience rejection, got {err:?}"
|
||||
);
|
||||
let (status, body) = err.into_response_parts();
|
||||
assert_eq!(status, StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(body.0["error"], "TokenInvalid");
|
||||
|
||||
// The same token *is* fine for the AppView it was minted for.
|
||||
assert!(
|
||||
verify_with_key(&token, &mb, "did:web:someone-elses-appview.example").is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_bodies_carry_the_documented_codes() {
|
||||
// These strings are a contract: the desktop client keys its
|
||||
// "refresh and retry" behaviour off `TokenInvalid`.
|
||||
let (s, b) = AuthError::Missing.into_response_parts();
|
||||
assert_eq!(s, StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(b.0["error"], "AuthMissing");
|
||||
let (s, b) = AuthError::Invalid("x".into()).into_response_parts();
|
||||
assert_eq!(s, StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(b.0["error"], "TokenInvalid");
|
||||
let (s, b) = AuthError::Forbidden.into_response_parts();
|
||||
assert_eq!(s, StatusCode::FORBIDDEN);
|
||||
assert_eq!(b.0["error"], "Forbidden");
|
||||
let (s, b) = AuthError::Unavailable("pds down".into()).into_response_parts();
|
||||
assert_eq!(s, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(b.0["error"], "AuthUnavailable");
|
||||
// Every body carries both fields the client parses.
|
||||
assert!(b.0["message"].as_str().unwrap().contains("pds down"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_access_token_verifies() {
|
||||
let (kp, mb) = test_key();
|
||||
let token = mint(&kp, "did:plc:alice", ACCESS_SCOPE, 3600);
|
||||
let claims = verify_with_key(&token, &mb, TEST_AUD).unwrap();
|
||||
assert_eq!(claims.sub, "did:plc:alice");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_token_is_invalid() {
|
||||
let (_, mb) = test_key();
|
||||
let err = verify_with_key("not-a-jwt", &mb, TEST_AUD).unwrap_err();
|
||||
assert!(matches!(err, AuthError::Invalid(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn token_signed_by_another_key_is_invalid() {
|
||||
let (kp, _) = test_key();
|
||||
let (_, other_mb) = test_key();
|
||||
let token = mint(&kp, "did:plc:alice", ACCESS_SCOPE, 3600);
|
||||
assert!(matches!(
|
||||
verify_with_key(&token, &other_mb, TEST_AUD).unwrap_err(),
|
||||
AuthError::Invalid(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refresh_scope_is_rejected() {
|
||||
// The refresh token is signed by the same key and lives 90
|
||||
// days. Without the scope check it would be a session token.
|
||||
let (kp, mb) = test_key();
|
||||
let token = mint(&kp, "did:plc:alice", "com.atproto.refresh", 3600);
|
||||
let err = verify_with_key(&token, &mb, TEST_AUD).unwrap_err();
|
||||
match err {
|
||||
AuthError::Invalid(msg) => assert!(msg.contains("com.atproto.refresh")),
|
||||
other => panic!("expected Invalid, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expired_token_is_rejected() {
|
||||
// Beyond `verify_jwt`'s 30 s leeway.
|
||||
let (kp, mb) = test_key();
|
||||
let token = mint(&kp, "did:plc:alice", ACCESS_SCOPE, -120);
|
||||
assert!(matches!(
|
||||
verify_with_key(&token, &mb, TEST_AUD).unwrap_err(),
|
||||
AuthError::Invalid(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_matches_enforces_sub_equals_did() {
|
||||
let me = AuthedDid(Some("did:plc:alice".into()));
|
||||
assert!(me.ensure_matches("did:plc:alice").is_ok());
|
||||
let (status, body) = me.ensure_matches("did:plc:bob").unwrap_err();
|
||||
assert_eq!(status, StatusCode::FORBIDDEN);
|
||||
assert_eq!(body.0["error"], "Forbidden");
|
||||
// A prefix of the real DID must not pass.
|
||||
assert!(me.ensure_matches("did:plc:ali").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ensure_matches_is_a_noop_when_auth_disabled() {
|
||||
let off = AuthedDid(None);
|
||||
assert!(off.ensure_matches("did:plc:anyone").is_ok());
|
||||
assert_eq!(off.did(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn did_doc_key_extraction() {
|
||||
let doc = json!({
|
||||
"id": "did:web:127.0.0.1%3A2583",
|
||||
"verificationMethod": [{
|
||||
"id": "did:web:127.0.0.1%3A2583#atproto",
|
||||
"type": "Multikey",
|
||||
"controller": "did:web:127.0.0.1%3A2583",
|
||||
"publicKeyMultibase": "zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme"
|
||||
}]
|
||||
});
|
||||
assert_eq!(
|
||||
extract_public_key_multibase(&doc).unwrap(),
|
||||
"zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme"
|
||||
);
|
||||
|
||||
// A document without a usable key must error rather than cache
|
||||
// an empty string.
|
||||
assert!(extract_public_key_multibase(&json!({})).is_err());
|
||||
assert!(extract_public_key_multibase(&json!({
|
||||
"verificationMethod": [{"id": "#x", "type": "Multikey"}]
|
||||
}))
|
||||
.is_err());
|
||||
assert!(extract_public_key_multibase(&json!({
|
||||
"verificationMethod": [{"publicKeyMultibase": ""}]
|
||||
}))
|
||||
.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn did_doc_url_is_built_from_the_base_url() {
|
||||
assert_eq!(
|
||||
PdsKeys::new("http://127.0.0.1:2583", TEST_AUD).did_doc_url(),
|
||||
"http://127.0.0.1:2583/.well-known/did.json"
|
||||
);
|
||||
// A trailing slash must not produce a double slash — some
|
||||
// servers 404 on it.
|
||||
assert_eq!(
|
||||
PdsKeys::new("http://pds:3000/", TEST_AUD).did_doc_url(),
|
||||
"http://pds:3000/.well-known/did.json"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn verification_fails_closed_when_the_pds_is_unreachable() {
|
||||
// Port 1 on loopback: nothing listens there, so the fetch fails
|
||||
// fast. The result must be a 503, never a pass-through.
|
||||
let keys = PdsKeys::new("http://127.0.0.1:1", TEST_AUD);
|
||||
let err = keys.verify_access_token("whatever").await.unwrap_err();
|
||||
assert!(
|
||||
matches!(err, AuthError::Unavailable(_)),
|
||||
"expected Unavailable, got {err:?}"
|
||||
);
|
||||
let (status, body) = err.into_response_parts();
|
||||
assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(body.0["error"], "AuthUnavailable");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
//! CAR v1 *reader*.
|
||||
//!
|
||||
//! The repo has a writer (`pds-server/src/car.rs`) but no reader the
|
||||
//! AppView could use: `pds-server` is a binary crate with no library
|
||||
//! target, so its `parse` helper is unreachable from here. The PDS
|
||||
//! firehose hands us a CAR in every `#commit` frame's `blocks` field, so
|
||||
//! the AppView needs its own.
|
||||
//!
|
||||
//! Format (<https://ipld.io/specs/transport/car/carv1/>):
|
||||
//!
|
||||
//! ```text
|
||||
//! [ varint: header_len | DAG-CBOR header ] { version: 1, roots: [CID] }
|
||||
//! [ varint: section_len | CID | block bytes ] block 1
|
||||
//! [ varint: section_len | CID | block bytes ] block 2
|
||||
//! ...
|
||||
//! ```
|
||||
//!
|
||||
//! The header is decoded with [`crate::cbor`], which accepts both the
|
||||
//! spec's `tag(42) + bytes(0x00 || cid)` link and the bare
|
||||
//! `tag(42) + bytes(cid)` this codebase's writer emits.
|
||||
//!
|
||||
//! Only structure is validated. Block CIDs are *not* re-hashed here:
|
||||
//! the firehose connection is to our own PDS over the cluster-internal
|
||||
//! URL, and a mismatch would mean a bug rather than an attack. See
|
||||
//! [`verify_block_cids`] for the opt-in check the tests use.
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use cid::Cid;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::cbor;
|
||||
|
||||
/// The parsed CAR header.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CarHeader {
|
||||
pub version: u64,
|
||||
pub roots: Vec<Cid>,
|
||||
}
|
||||
|
||||
/// One `(CID, bytes)` pair out of a CAR file.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CarBlock {
|
||||
pub cid: Cid,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A CAR file's header plus its blocks, in file order.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Car {
|
||||
pub header: CarHeader,
|
||||
pub blocks: Vec<CarBlock>,
|
||||
}
|
||||
|
||||
impl Car {
|
||||
/// Index the blocks by CID for lookup by the commit's `ops`.
|
||||
///
|
||||
/// Duplicate CIDs keep the first occurrence, matching the writer's
|
||||
/// own de-duplication.
|
||||
pub fn block_map(&self) -> HashMap<Cid, &[u8]> {
|
||||
let mut map = HashMap::with_capacity(self.blocks.len());
|
||||
for b in &self.blocks {
|
||||
map.entry(b.cid).or_insert(b.data.as_slice());
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// The first root, if the header declares one.
|
||||
///
|
||||
/// `allow(dead_code)`: the ingest path doesn't need the commit
|
||||
/// block itself (the ops carry the record CIDs), but a CAR reader
|
||||
/// that cannot name its root is a reader with a hole in it, and the
|
||||
/// tests read it.
|
||||
#[allow(dead_code)]
|
||||
pub fn root(&self) -> Option<Cid> {
|
||||
self.roots().first().copied()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn roots(&self) -> &[Cid] {
|
||||
&self.header.roots
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a CAR v1 byte stream.
|
||||
pub fn parse(bytes: &[u8]) -> Result<Car> {
|
||||
let mut p = 0usize;
|
||||
|
||||
let (header_len, n) = read_varint(bytes, p)?;
|
||||
p += n;
|
||||
let header_end = checked_end(bytes, p, header_len, "CAR header")?;
|
||||
let header = decode_header(&bytes[p..header_end])?;
|
||||
p = header_end;
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
while p < bytes.len() {
|
||||
let (section_len, n) = read_varint(bytes, p)?;
|
||||
let section_start = p + n;
|
||||
let section_end = checked_end(bytes, section_start, section_len, "CAR section")?;
|
||||
let section = &bytes[section_start..section_end];
|
||||
let cid = Cid::read_bytes(section)
|
||||
.map_err(|e| anyhow!("invalid CID in CAR section at offset {section_start}: {e}"))?;
|
||||
let cid_len = cid.encoded_len();
|
||||
if cid_len > section.len() {
|
||||
bail!("CAR section at {section_start} is shorter than its CID");
|
||||
}
|
||||
blocks.push(CarBlock {
|
||||
cid,
|
||||
data: section[cid_len..].to_vec(),
|
||||
});
|
||||
p = section_end;
|
||||
}
|
||||
|
||||
Ok(Car { header, blocks })
|
||||
}
|
||||
|
||||
/// Re-hash every block and compare against its declared CID.
|
||||
///
|
||||
/// Not called on the ingest path (see the module docs); the CAR reader
|
||||
/// tests use it to prove the reader hands back the bytes the writer put
|
||||
/// in, unshifted by an off-by-one in the section framing.
|
||||
#[allow(dead_code)]
|
||||
pub fn verify_block_cids(car: &Car) -> Result<()> {
|
||||
for b in &car.blocks {
|
||||
let recomputed = at_crypto::cid::cid_for_cbor(&b.data)?;
|
||||
if recomputed != b.cid {
|
||||
bail!("CAR block CID mismatch: declared {}, computed {recomputed}", b.cid);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_header(bytes: &[u8]) -> Result<CarHeader> {
|
||||
let value = cbor::decode(bytes)?;
|
||||
let version = value
|
||||
.get("version")
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("CAR header missing `version`"))?;
|
||||
if version != 1 {
|
||||
bail!("unsupported CAR version {version} (only v1 is defined for atproto)");
|
||||
}
|
||||
// `roots` is required by the spec but may legitimately be empty.
|
||||
let roots = match value.get("roots") {
|
||||
Some(v) => v
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("CAR header `roots` is not an array"))?
|
||||
.iter()
|
||||
.map(|item| {
|
||||
item.as_cid()
|
||||
.ok_or_else(|| anyhow!("CAR header root is not a CID link"))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
Ok(CarHeader {
|
||||
version: version as u64,
|
||||
roots,
|
||||
})
|
||||
}
|
||||
|
||||
/// LEB128 unsigned varint, the length prefix CAR uses.
|
||||
fn read_varint(bytes: &[u8], offset: usize) -> Result<(u64, usize)> {
|
||||
let mut value: u64 = 0;
|
||||
let mut shift = 0u32;
|
||||
let mut i = offset;
|
||||
loop {
|
||||
let b = *bytes
|
||||
.get(i)
|
||||
.ok_or_else(|| anyhow!("varint extends past end of CAR input at {offset}"))?;
|
||||
i += 1;
|
||||
value |= u64::from(b & 0x7f) << shift;
|
||||
if b & 0x80 == 0 {
|
||||
return Ok((value, i - offset));
|
||||
}
|
||||
shift += 7;
|
||||
if shift >= 64 {
|
||||
bail!("varint longer than 64 bits at offset {offset}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_end(bytes: &[u8], pos: usize, len: u64, what: &str) -> Result<usize> {
|
||||
let len = usize::try_from(len).map_err(|_| anyhow!("{what} length overflows usize"))?;
|
||||
let end = pos
|
||||
.checked_add(len)
|
||||
.ok_or_else(|| anyhow!("{what} length overflows"))?;
|
||||
if end > bytes.len() {
|
||||
bail!("{what} length {len} exceeds input (offset {pos}, total {})", bytes.len());
|
||||
}
|
||||
Ok(end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_writer {
|
||||
//! A byte-for-byte copy of `pds-server/src/car.rs`'s encoder, so the
|
||||
//! reader's tests exercise *the writer's* output rather than a
|
||||
//! convenient fiction.
|
||||
//!
|
||||
//! Copied rather than imported because `pds-server` has no library
|
||||
//! target. If the writer ever changes shape, the integration test
|
||||
//! `pds_firehose_integration::car_reader_parses_a_real_repo_export`
|
||||
//! is the tripwire: it parses a CAR produced by the running PDS.
|
||||
|
||||
use cid::Cid;
|
||||
|
||||
fn cbor_head(out: &mut Vec<u8>, major: u8, n: u64) {
|
||||
let m = (major & 0x07) << 5;
|
||||
if n < 24 {
|
||||
out.push(m | n as u8);
|
||||
} else if n < 0x100 {
|
||||
out.push(m | 24);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x10000 {
|
||||
out.push(m | 25);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x100_0000 {
|
||||
out.push(m | 26);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else {
|
||||
out.push(m | 27);
|
||||
out.push((n >> 24) as u8);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
}
|
||||
}
|
||||
|
||||
fn cbor_text(out: &mut Vec<u8>, s: &str) {
|
||||
cbor_head(out, 3, s.len() as u64);
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
fn cbor_bytes(out: &mut Vec<u8>, b: &[u8]) {
|
||||
cbor_head(out, 2, b.len() as u64);
|
||||
out.extend_from_slice(b);
|
||||
}
|
||||
|
||||
pub fn encode_header(roots: &[Cid]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
cbor_head(&mut out, 5, 2);
|
||||
cbor_text(&mut out, "version");
|
||||
cbor_head(&mut out, 0, 1);
|
||||
cbor_text(&mut out, "roots");
|
||||
cbor_head(&mut out, 4, roots.len() as u64);
|
||||
for cid in roots {
|
||||
cbor_head(&mut out, 6, 42);
|
||||
cbor_bytes(&mut out, &cid.to_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn write_varint(out: &mut Vec<u8>, mut n: u64) {
|
||||
loop {
|
||||
let mut byte = (n & 0x7f) as u8;
|
||||
n >>= 7;
|
||||
if n != 0 {
|
||||
byte |= 0x80;
|
||||
}
|
||||
out.push(byte);
|
||||
if n == 0 {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CarWriter {
|
||||
blocks: Vec<(Cid, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl CarWriter {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn append(&mut self, cid: Cid, data: &[u8]) {
|
||||
if self.blocks.iter().any(|(c, _)| *c == cid) {
|
||||
return;
|
||||
}
|
||||
self.blocks.push((cid, data.to_vec()));
|
||||
}
|
||||
|
||||
pub fn finish(&self, roots: &[Cid]) -> Vec<u8> {
|
||||
let header = encode_header(roots);
|
||||
let mut out = Vec::new();
|
||||
write_varint(&mut out, header.len() as u64);
|
||||
out.extend_from_slice(&header);
|
||||
for (cid, data) in &self.blocks {
|
||||
let cid_bytes = cid.to_bytes();
|
||||
write_varint(&mut out, (cid_bytes.len() + data.len()) as u64);
|
||||
out.extend_from_slice(&cid_bytes);
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::test_writer::CarWriter;
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
#[test]
|
||||
fn round_trips_a_single_block() {
|
||||
let data = b"hello world".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[cid])).unwrap();
|
||||
assert_eq!(car.header.version, 1);
|
||||
assert_eq!(car.roots(), &[cid]);
|
||||
assert_eq!(car.blocks.len(), 1);
|
||||
assert_eq!(car.blocks[0].data, data);
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_many_blocks_in_order() {
|
||||
let payloads: Vec<Vec<u8>> = (0..6)
|
||||
.map(|i| format!("block-{i}").into_bytes())
|
||||
.collect();
|
||||
let cids: Vec<Cid> = payloads.iter().map(|p| cid_for_cbor(p).unwrap()).collect();
|
||||
let mut w = CarWriter::new();
|
||||
for (cid, data) in cids.iter().zip(&payloads) {
|
||||
w.append(*cid, data);
|
||||
}
|
||||
let car = parse(&w.finish(&[cids[3]])).unwrap();
|
||||
assert_eq!(car.root(), Some(cids[3]));
|
||||
assert_eq!(car.blocks.len(), 6);
|
||||
for (i, b) in car.blocks.iter().enumerate() {
|
||||
assert_eq!(b.cid, cids[i]);
|
||||
assert_eq!(b.data, payloads[i]);
|
||||
}
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_map_finds_blocks_by_cid() {
|
||||
let a = b"record a".to_vec();
|
||||
let b = b"record b".to_vec();
|
||||
let (ca, cb) = (cid_for_cbor(&a).unwrap(), cid_for_cbor(&b).unwrap());
|
||||
let mut w = CarWriter::new();
|
||||
w.append(ca, &a);
|
||||
w.append(cb, &b);
|
||||
let car = parse(&w.finish(&[ca])).unwrap();
|
||||
let map = car.block_map();
|
||||
assert_eq!(map.get(&ca).copied(), Some(a.as_slice()));
|
||||
assert_eq!(map.get(&cb).copied(), Some(b.as_slice()));
|
||||
assert!(!map.contains_key(&cid_for_cbor(b"absent").unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_a_multi_byte_varint_section_length() {
|
||||
// A >127 byte block forces a two-byte varint, which is where an
|
||||
// off-by-one in the length prefix would show up.
|
||||
let data = vec![0x42u8; 500];
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[cid])).unwrap();
|
||||
assert_eq!(car.blocks[0].data.len(), 500);
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_an_empty_root_list() {
|
||||
let data = b"orphan".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[])).unwrap();
|
||||
assert!(car.roots().is_empty());
|
||||
assert_eq!(car.blocks.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_input() {
|
||||
let data = b"hello".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let bytes = w.finish(&[cid]);
|
||||
// Cut into the last block's payload.
|
||||
assert!(parse(&bytes[..bytes.len() - 3]).is_err());
|
||||
// Cut inside the header.
|
||||
assert!(parse(&bytes[..3]).is_err());
|
||||
// Nothing at all.
|
||||
assert!(parse(&[]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
//! A minimal, allocation-honest CBOR reader — just enough to decode the
|
||||
//! frames of `com.atproto.sync.subscribeRepos` and the header of a CAR
|
||||
//! file.
|
||||
//!
|
||||
//! ## Why not `ciborium`?
|
||||
//!
|
||||
//! Two reasons, and both come from what the wire format actually is.
|
||||
//!
|
||||
//! 1. **Two values per message.** A subscribeRepos frame is *two*
|
||||
//! DAG-CBOR values written back to back (header, then body) inside
|
||||
//! one WebSocket binary message. A `serde`-shaped reader gives us
|
||||
//! "decode one value from this slice" and no cursor we can resume
|
||||
//! from, so we would have to guess where the header ended.
|
||||
//! 2. **Tag 42.** DAG-CBOR encodes a CID link as `tag(42) +
|
||||
//! bytes(<cid>)`. `ciborium`'s `serde` mapping turns tags into a
|
||||
//! private newtype dance that does not survive a round trip through
|
||||
//! `serde_json::Value`, which is the shape the rest of the AppView
|
||||
//! speaks.
|
||||
//!
|
||||
//! So this module decodes CBOR into its own small [`Cbor`] tree with an
|
||||
//! explicit byte offset, which makes "read the header, then read the
|
||||
//! body from where the header stopped" a two-line function.
|
||||
//!
|
||||
//! ## What it deliberately does not do
|
||||
//!
|
||||
//! No indefinite-length items (DAG-CBOR forbids them; we reject them
|
||||
//! rather than guess), no half floats beyond a plain `f64` widening, no
|
||||
//! canonicalisation checks. It is a *reader* for input we already
|
||||
//! decided to trust at the transport layer, with hard length checks so
|
||||
//! a malformed frame returns `Err` instead of panicking.
|
||||
//!
|
||||
//! Record blocks inside the CAR payload are NOT decoded with this
|
||||
//! module: they are written by `ciborium::into_writer(&serde_json::Value)`
|
||||
//! on the PDS side (see `pds-server/src/routes/repo.rs`), which means
|
||||
//! CIDs inside a record are plain strings, not tag-42 links. Their
|
||||
//! inverse is `ciborium::from_reader::<serde_json::Value, _>`, and
|
||||
//! that is what [`crate::pds_firehose`] uses for them.
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use cid::Cid;
|
||||
|
||||
/// A decoded CBOR value.
|
||||
///
|
||||
/// `Nint` carries the already-negated value (CBOR stores `-1 - n`), so
|
||||
/// callers never have to remember the bias.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Cbor {
|
||||
Uint(u64),
|
||||
Nint(i64),
|
||||
Bytes(Vec<u8>),
|
||||
Text(String),
|
||||
Array(Vec<Cbor>),
|
||||
/// Kept as an ordered key/value list rather than a map: DAG-CBOR
|
||||
/// keys are text and already canonically ordered, and a `Vec` keeps
|
||||
/// the decoder free of hashing while the maps we read have a
|
||||
/// handful of entries at most.
|
||||
Map(Vec<(Cbor, Cbor)>),
|
||||
Tag(u64, Box<Cbor>),
|
||||
Bool(bool),
|
||||
Null,
|
||||
Undefined,
|
||||
Float(f64),
|
||||
}
|
||||
|
||||
impl Cbor {
|
||||
/// Look up a text key in a map. `None` for non-maps and misses.
|
||||
pub fn get(&self, key: &str) -> Option<&Cbor> {
|
||||
match self {
|
||||
Cbor::Map(entries) => entries.iter().find_map(|(k, v)| match k {
|
||||
Cbor::Text(s) if s == key => Some(v),
|
||||
_ => None,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The value as a signed integer, accepting both CBOR integer
|
||||
/// majors. `None` when the value is not an integer, or when an
|
||||
/// unsigned value exceeds `i64::MAX` (which cannot happen for a
|
||||
/// `seq`, but silently wrapping would be worse than a miss).
|
||||
pub fn as_i64(&self) -> Option<i64> {
|
||||
match self {
|
||||
Cbor::Uint(n) => i64::try_from(*n).ok(),
|
||||
Cbor::Nint(n) => Some(*n),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> Option<&str> {
|
||||
match self {
|
||||
Cbor::Text(s) => Some(s.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Cbor::Bytes(b) => Some(b.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bool(&self) -> Option<bool> {
|
||||
match self {
|
||||
Cbor::Bool(b) => Some(*b),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_array(&self) -> Option<&[Cbor]> {
|
||||
match self {
|
||||
Cbor::Array(items) => Some(items.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_null(&self) -> bool {
|
||||
matches!(self, Cbor::Null | Cbor::Undefined)
|
||||
}
|
||||
|
||||
/// Decode a DAG-CBOR CID link: `tag(42) + bytes(...)`.
|
||||
///
|
||||
/// The spec prefixes the CID bytes with a single `0x00` (the
|
||||
/// multibase "identity" marker), because a CID inside a byte string
|
||||
/// has no textual multibase prefix to carry. **This repository's own
|
||||
/// writer omits that byte** — `pds-server/src/car.rs` writes
|
||||
/// `cbor_bytes(&cid.to_bytes())` — so we accept both spellings: a
|
||||
/// leading `0x00` is skipped, anything else is parsed as-is. Being
|
||||
/// lenient here costs nothing (a real CIDv1 never starts with
|
||||
/// `0x00`, and CIDv0 starts with `0x12`) and it means the AppView
|
||||
/// keeps working whether the PDS follows the spec or the house
|
||||
/// convention.
|
||||
pub fn as_cid(&self) -> Option<Cid> {
|
||||
let inner = match self {
|
||||
Cbor::Tag(42, inner) => inner.as_ref(),
|
||||
// A bare byte string where a link is expected: CARs written by
|
||||
// older builds of this PDS tagged the root CID without the
|
||||
// `0x00` identity prefix, and some encoders drop the tag
|
||||
// entirely. Both still have to parse.
|
||||
Cbor::Bytes(_) => self,
|
||||
// A record block written from `serde_json::Value` spells a
|
||||
// CID as a plain string — accept that too, so callers do
|
||||
// not need a second code path for the block contents.
|
||||
Cbor::Text(s) => return s.parse::<Cid>().ok(),
|
||||
_ => return None,
|
||||
};
|
||||
let raw = inner.as_bytes()?;
|
||||
let raw = match raw.first() {
|
||||
Some(0x00) => &raw[1..],
|
||||
_ => raw,
|
||||
};
|
||||
Cid::read_bytes(raw).ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode exactly one CBOR value starting at `pos`.
|
||||
///
|
||||
/// Returns the value and the offset just past it, so a caller can read
|
||||
/// the next value from the same buffer — which is precisely what a
|
||||
/// two-value subscribeRepos frame needs.
|
||||
pub fn decode_at(bytes: &[u8], pos: usize) -> Result<(Cbor, usize)> {
|
||||
let (major, arg, mut p) = read_head(bytes, pos)?;
|
||||
match major {
|
||||
0 => Ok((Cbor::Uint(arg), p)),
|
||||
1 => {
|
||||
// CBOR negative integers store `-1 - n`. Values below
|
||||
// `i64::MIN` cannot occur in anything we consume, and
|
||||
// wrapping them would produce a positive number, so bail.
|
||||
let n = i64::try_from(arg)
|
||||
.map_err(|_| anyhow!("negative integer out of i64 range"))?;
|
||||
Ok((Cbor::Nint(-1 - n), p))
|
||||
}
|
||||
2 => {
|
||||
let end = checked_end(bytes, p, arg, "byte string")?;
|
||||
let v = bytes[p..end].to_vec();
|
||||
Ok((Cbor::Bytes(v), end))
|
||||
}
|
||||
3 => {
|
||||
let end = checked_end(bytes, p, arg, "text string")?;
|
||||
let s = std::str::from_utf8(&bytes[p..end])
|
||||
.map_err(|e| anyhow!("invalid UTF-8 in CBOR text: {e}"))?
|
||||
.to_string();
|
||||
Ok((Cbor::Text(s), end))
|
||||
}
|
||||
4 => {
|
||||
let mut items = Vec::with_capacity(sane_capacity(arg));
|
||||
for _ in 0..arg {
|
||||
let (v, next) = decode_at(bytes, p)?;
|
||||
items.push(v);
|
||||
p = next;
|
||||
}
|
||||
Ok((Cbor::Array(items), p))
|
||||
}
|
||||
5 => {
|
||||
let mut entries = Vec::with_capacity(sane_capacity(arg));
|
||||
for _ in 0..arg {
|
||||
let (k, next) = decode_at(bytes, p)?;
|
||||
let (v, next) = decode_at(bytes, next)?;
|
||||
entries.push((k, v));
|
||||
p = next;
|
||||
}
|
||||
Ok((Cbor::Map(entries), p))
|
||||
}
|
||||
6 => {
|
||||
let (inner, next) = decode_at(bytes, p)?;
|
||||
Ok((Cbor::Tag(arg, Box::new(inner)), next))
|
||||
}
|
||||
7 => match arg {
|
||||
20 => Ok((Cbor::Bool(false), p)),
|
||||
21 => Ok((Cbor::Bool(true), p)),
|
||||
22 => Ok((Cbor::Null, p)),
|
||||
23 => Ok((Cbor::Undefined, p)),
|
||||
// Floats arrive as the raw bit pattern in `arg`; the width
|
||||
// is implied by the additional-information byte, which
|
||||
// `read_head` has already consumed. We only ever see f64 in
|
||||
// practice (DAG-CBOR requires it), so the narrower widths
|
||||
// are decoded for completeness rather than need.
|
||||
_ => Ok((Cbor::Float(f64::from_bits(arg)), p)),
|
||||
},
|
||||
other => bail!("unsupported CBOR major type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a single CBOR value that must span the whole buffer.
|
||||
pub fn decode(bytes: &[u8]) -> Result<Cbor> {
|
||||
let (v, end) = decode_at(bytes, 0)?;
|
||||
if end != bytes.len() {
|
||||
bail!("trailing bytes after CBOR value ({} left)", bytes.len() - end);
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Read a CBOR head: major type plus its argument.
|
||||
///
|
||||
/// Indefinite-length encodings (`additional information == 31`) are
|
||||
/// rejected: DAG-CBOR forbids them, and accepting them would mean
|
||||
/// implementing break-stop scanning for input that should never carry
|
||||
/// it.
|
||||
fn read_head(bytes: &[u8], pos: usize) -> Result<(u8, u64, usize)> {
|
||||
let first = *bytes
|
||||
.get(pos)
|
||||
.ok_or_else(|| anyhow!("CBOR read past end of input at {pos}"))?;
|
||||
let major = first >> 5;
|
||||
let low = first & 0x1f;
|
||||
let (arg, extra) = match low {
|
||||
0..=23 => (low as u64, 0usize),
|
||||
24 => (read_uint(bytes, pos + 1, 1)?, 1),
|
||||
25 => (read_uint(bytes, pos + 1, 2)?, 2),
|
||||
26 => (read_uint(bytes, pos + 1, 4)?, 4),
|
||||
27 => (read_uint(bytes, pos + 1, 8)?, 8),
|
||||
31 => bail!("indefinite-length CBOR item is not valid DAG-CBOR"),
|
||||
other => bail!("reserved CBOR additional information {other}"),
|
||||
};
|
||||
// For major 7 the "argument" of a float is the raw bit pattern, and
|
||||
// f32/f16 need widening before `f64::from_bits` makes sense.
|
||||
let arg = match (major, low) {
|
||||
(7, 25) => f64::from(half_to_f32(arg as u16)).to_bits(),
|
||||
(7, 26) => f64::from(f32::from_bits(arg as u32)).to_bits(),
|
||||
_ => arg,
|
||||
};
|
||||
Ok((major, arg, pos + 1 + extra))
|
||||
}
|
||||
|
||||
fn read_uint(bytes: &[u8], pos: usize, len: usize) -> Result<u64> {
|
||||
if pos + len > bytes.len() {
|
||||
bail!("truncated CBOR integer of {len} byte(s) at {pos}");
|
||||
}
|
||||
let mut n: u64 = 0;
|
||||
for b in &bytes[pos..pos + len] {
|
||||
n = (n << 8) | u64::from(*b);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// IEEE-754 half → single. Only reached for `f16` inputs, which nothing
|
||||
/// in this protocol emits; kept so a stray value decodes instead of
|
||||
/// erroring out mid-frame.
|
||||
fn half_to_f32(bits: u16) -> f32 {
|
||||
let sign = ((bits >> 15) & 1) as u32;
|
||||
let exp = ((bits >> 10) & 0x1f) as u32;
|
||||
let frac = (bits & 0x3ff) as u32;
|
||||
let out = match exp {
|
||||
0 if frac == 0 => sign << 31,
|
||||
0 => {
|
||||
// Subnormal: renormalise.
|
||||
let mut e = -1i32;
|
||||
let mut f = frac;
|
||||
while f & 0x400 == 0 {
|
||||
f <<= 1;
|
||||
e -= 1;
|
||||
}
|
||||
let exp32 = (127 - 15 + e) as u32;
|
||||
(sign << 31) | (exp32 << 23) | ((f & 0x3ff) << 13)
|
||||
}
|
||||
0x1f => (sign << 31) | (0xff << 23) | (frac << 13),
|
||||
_ => (sign << 31) | ((exp + 127 - 15) << 23) | (frac << 13),
|
||||
};
|
||||
f32::from_bits(out)
|
||||
}
|
||||
|
||||
/// Bounds-check a string/bytes payload before slicing it.
|
||||
fn checked_end(bytes: &[u8], pos: usize, len: u64, what: &str) -> Result<usize> {
|
||||
let len = usize::try_from(len).map_err(|_| anyhow!("{what} length overflows usize"))?;
|
||||
let end = pos
|
||||
.checked_add(len)
|
||||
.ok_or_else(|| anyhow!("{what} length overflows"))?;
|
||||
if end > bytes.len() {
|
||||
bail!("{what} of {len} byte(s) exceeds input at offset {pos}");
|
||||
}
|
||||
Ok(end)
|
||||
}
|
||||
|
||||
/// Cap the pre-allocation a declared array/map length can trigger. A
|
||||
/// corrupt frame claiming `map(2^40)` must not make us reserve 40 GiB
|
||||
/// before the first missing byte errors out; the collection still grows
|
||||
/// naturally for genuinely large inputs.
|
||||
fn sane_capacity(declared: u64) -> usize {
|
||||
usize::try_from(declared.min(1024)).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
/// Build `{"op": 1, "t": "#commit"}` by hand — the exact bytes the
|
||||
/// PDS writes for a regular frame header.
|
||||
fn commit_header_bytes() -> Vec<u8> {
|
||||
let mut v = vec![0xA2]; // map(2)
|
||||
v.push(0x62); // text(2)
|
||||
v.extend_from_slice(b"op");
|
||||
v.push(0x01); // uint 1
|
||||
v.push(0x61); // text(1)
|
||||
v.extend_from_slice(b"t");
|
||||
v.push(0x67); // text(7)
|
||||
v.extend_from_slice(b"#commit");
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_a_frame_header() {
|
||||
let v = decode(&commit_header_bytes()).unwrap();
|
||||
assert_eq!(v.get("op").unwrap().as_i64(), Some(1));
|
||||
assert_eq!(v.get("t").unwrap().as_str(), Some("#commit"));
|
||||
assert!(v.get("missing").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_negative_op_of_an_error_header() {
|
||||
// {"op": -1} → map(1), text(2)"op", nint(0) = -1
|
||||
let bytes = vec![0xA1, 0x62, b'o', b'p', 0x20];
|
||||
let v = decode(&bytes).unwrap();
|
||||
assert_eq!(v.get("op").unwrap().as_i64(), Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_multi_byte_integers() {
|
||||
// uint16 300, uint32 70000, uint64 2^33, nint -300
|
||||
assert_eq!(decode(&[0x19, 0x01, 0x2C]).unwrap().as_i64(), Some(300));
|
||||
assert_eq!(
|
||||
decode(&[0x1A, 0x00, 0x01, 0x11, 0x70]).unwrap().as_i64(),
|
||||
Some(70000)
|
||||
);
|
||||
assert_eq!(
|
||||
decode(&[0x1B, 0, 0, 0, 2, 0, 0, 0, 0]).unwrap().as_i64(),
|
||||
Some(8_589_934_592)
|
||||
);
|
||||
assert_eq!(decode(&[0x39, 0x01, 0x2B]).unwrap().as_i64(), Some(-300));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_at_reads_two_values_back_to_back() {
|
||||
// This is the whole reason the module exists: a frame is header
|
||||
// + body concatenated with no separator.
|
||||
let mut buf = commit_header_bytes();
|
||||
let body_start = buf.len();
|
||||
buf.extend_from_slice(&[0xA1, 0x63, b's', b'e', b'q', 0x18, 0x2A]); // {"seq": 42}
|
||||
let (header, next) = decode_at(&buf, 0).unwrap();
|
||||
assert_eq!(next, body_start);
|
||||
assert_eq!(header.get("t").unwrap().as_str(), Some("#commit"));
|
||||
let (body, end) = decode_at(&buf, next).unwrap();
|
||||
assert_eq!(end, buf.len());
|
||||
assert_eq!(body.get("seq").unwrap().as_i64(), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_tag_42_cid_link_both_spellings() {
|
||||
let cid = cid_for_cbor(b"a block").unwrap();
|
||||
let raw = cid.to_bytes();
|
||||
|
||||
// House spelling: tag(42) + bytes(<cid>) with no 0x00 prefix.
|
||||
let mut bare = vec![0xD8, 42]; // tag(42) via 1-byte extension
|
||||
bare.push(0x58); // bytes, 1-byte length
|
||||
bare.push(raw.len() as u8);
|
||||
bare.extend_from_slice(&raw);
|
||||
assert_eq!(decode(&bare).unwrap().as_cid(), Some(cid));
|
||||
|
||||
// Spec spelling: the same, with the identity multibase prefix.
|
||||
let mut prefixed = vec![0xD8, 42, 0x58, (raw.len() + 1) as u8, 0x00];
|
||||
prefixed.extend_from_slice(&raw);
|
||||
assert_eq!(decode(&prefixed).unwrap().as_cid(), Some(cid));
|
||||
|
||||
// And the string spelling records use.
|
||||
let text = {
|
||||
let s = cid.to_string();
|
||||
let mut v = vec![0x78, s.len() as u8];
|
||||
v.extend_from_slice(s.as_bytes());
|
||||
v
|
||||
};
|
||||
assert_eq!(decode(&text).unwrap().as_cid(), Some(cid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_simple_values_and_containers() {
|
||||
// [true, false, null] → array(3)
|
||||
let v = decode(&[0x83, 0xF5, 0xF4, 0xF6]).unwrap();
|
||||
let items = v.as_array().unwrap();
|
||||
assert_eq!(items[0].as_bool(), Some(true));
|
||||
assert_eq!(items[1].as_bool(), Some(false));
|
||||
assert!(items[2].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_and_indefinite_input() {
|
||||
// text(7) claiming 7 bytes but carrying 2.
|
||||
assert!(decode(&[0x67, b'a', b'b']).is_err());
|
||||
// Indefinite-length array.
|
||||
assert!(decode(&[0x9F, 0x01, 0xFF]).is_err());
|
||||
// Trailing garbage after a complete value.
|
||||
assert!(decode(&[0x01, 0x02]).is_err());
|
||||
// Empty input.
|
||||
assert!(decode(&[]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_declared_length_errors_instead_of_allocating() {
|
||||
// map(2^32) with nothing behind it. Must return Err quickly
|
||||
// rather than trying to reserve the declared capacity.
|
||||
let bytes = vec![0xBA, 0xFF, 0xFF, 0xFF, 0xFF];
|
||||
assert!(decode(&bytes).is_err());
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::indexer;
|
||||
|
||||
@@ -31,6 +31,18 @@ pub struct Stats {
|
||||
/// writes this; `/healthz` reads it. Wrapped in `Arc` so the consumer
|
||||
/// can hold its own clone without borrowing from us.
|
||||
pub jetstream_connected: Arc<AtomicBool>,
|
||||
/// Whether the **local PDS** firehose WebSocket is currently up
|
||||
/// ([`crate::pds_firehose`]). Separate from `jetstream_connected`
|
||||
/// because the two streams fail independently and for different
|
||||
/// reasons: a dead Jetstream means no view of the wider network, a
|
||||
/// dead PDS firehose means the AppView has lost the guaranteed
|
||||
/// delivery path for its *own* users' records and is running on the
|
||||
/// best-effort push alone. `/healthz` has to be able to say which.
|
||||
pub pds_connected: AtomicBool,
|
||||
/// Number of `#commit` frames applied from the PDS firehose.
|
||||
pub pds_frames_processed: AtomicU64,
|
||||
/// Highest `seq` applied from the PDS firehose in this process.
|
||||
pub pds_last_seq: AtomicI64,
|
||||
}
|
||||
|
||||
impl Default for Stats {
|
||||
@@ -40,6 +52,9 @@ impl Default for Stats {
|
||||
last_event_time_us: AtomicI64::new(0),
|
||||
last_cursor_persisted_us: AtomicI64::new(0),
|
||||
jetstream_connected: Arc::new(AtomicBool::new(false)),
|
||||
pds_connected: AtomicBool::new(false),
|
||||
pds_frames_processed: AtomicU64::new(0),
|
||||
pds_last_seq: AtomicI64::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,6 +65,9 @@ impl std::fmt::Debug for Stats {
|
||||
.field("events_processed", &self.events_processed())
|
||||
.field("last_event_time_us", &self.last_event_time_us.load(Ordering::Relaxed))
|
||||
.field("jetstream_connected", &self.jetstream_connected())
|
||||
.field("pds_connected", &self.pds_connected())
|
||||
.field("pds_frames_processed", &self.pds_frames_processed())
|
||||
.field("pds_last_seq", &self.pds_last_seq())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -85,6 +103,23 @@ impl Stats {
|
||||
pub fn jetstream_connected(&self) -> bool {
|
||||
self.jetstream_connected.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Is the local PDS firehose connected right now?
|
||||
pub fn pds_connected(&self) -> bool {
|
||||
self.pds_connected.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn pds_frames_processed(&self) -> u64 {
|
||||
self.pds_frames_processed.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Highest PDS-firehose `seq` this process has applied. 0 before the
|
||||
/// first frame — note this is the *in-process* high-water mark, not
|
||||
/// the persisted cursor, which lives in `pds_firehose_cursor` and
|
||||
/// survives restarts.
|
||||
pub fn pds_last_seq(&self) -> i64 {
|
||||
self.pds_last_seq.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The thing the Jetstream consumer calls once per event.
|
||||
|
||||
@@ -153,7 +153,10 @@ impl HandleSyncWorker {
|
||||
|
||||
/// The SELECT half of [`Self::run_once`]: up to [`BATCH_SIZE`]
|
||||
/// distinct DIDs still waiting for a handle.
|
||||
async fn select_candidates(&self) -> Result<Vec<String>> {
|
||||
///
|
||||
/// Public so integration tests can assert on the batch cap without
|
||||
/// depending on what else the live indexer left pending.
|
||||
pub async fn select_candidates(&self) -> Result<Vec<String>> {
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
r#"SELECT DISTINCT did
|
||||
FROM posts
|
||||
@@ -177,7 +180,7 @@ impl HandleSyncWorker {
|
||||
/// freshly seeded DID may simply not make the batch — which made
|
||||
/// the dispatch tests fail for reasons that had nothing to do with
|
||||
/// dispatch. Passing the DIDs in removes that coupling.
|
||||
async fn resolve_batch(&self, dids: Vec<String>) -> Result<SyncReport> {
|
||||
pub async fn resolve_batch(&self, dids: Vec<String>) -> Result<SyncReport> {
|
||||
let mut report = SyncReport::default();
|
||||
if dids.is_empty() {
|
||||
return Ok(report);
|
||||
@@ -576,6 +579,66 @@ mod tests {
|
||||
assert_eq!(h.as_deref(), Some("from-ingest"));
|
||||
}
|
||||
|
||||
/// The documented PDS-first rule: the local PDS is asked before the
|
||||
/// method dispatch, so a `did:key:` user hosted here resolves
|
||||
/// without ever dialing plc.directory. This is the flip side of
|
||||
/// `unknown_methods_are_skipped` — same DID method, opposite
|
||||
/// outcome, and the difference is solely whether the PDS hosts it.
|
||||
#[tokio::test]
|
||||
async fn pds_resolves_did_key_before_method_dispatch() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let did = format!("did:key:z{}", uuid::Uuid::new_v4().simple());
|
||||
let _ = sqlx::query("DELETE FROM posts WHERE did = $1")
|
||||
.bind(&did)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
seed_post(&db, &did, "rk", "").await.unwrap();
|
||||
|
||||
// The PDS hosts this user; the outbound resolvers know nothing
|
||||
// and must never be consulted.
|
||||
let pds = StubResolver::new(HashMap::from([(
|
||||
did.clone(),
|
||||
Some("local-user.maarcadetweet.local".into()),
|
||||
)]))
|
||||
.into_arc();
|
||||
let plc_log: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let plc = TrackingResolver::new(
|
||||
HashMap::from([(did.clone(), Some("must-not-be-used".into()))]),
|
||||
Arc::clone(&plc_log),
|
||||
);
|
||||
let plc_arc: Arc<dyn DidHandleResolver> = Arc::new(plc);
|
||||
|
||||
let worker = HandleSyncWorker {
|
||||
db: db.clone(),
|
||||
pds_resolver: pds,
|
||||
plc_resolver: Arc::clone(&plc_arc),
|
||||
web_resolver: plc_arc,
|
||||
interval_secs: 999,
|
||||
};
|
||||
let report = worker.resolve_batch(vec![did.clone()]).await.unwrap();
|
||||
assert_eq!(
|
||||
report.resolved, 1,
|
||||
"a did:key hosted by the local PDS must resolve, got {report:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
get_handle(&worker.db, &did).await.as_deref(),
|
||||
Some("local-user.maarcadetweet.local")
|
||||
);
|
||||
assert!(
|
||||
plc_log.lock().unwrap().is_empty(),
|
||||
"the PDS answered, so no outbound resolver may be consulted"
|
||||
);
|
||||
|
||||
let _ = sqlx::query("DELETE FROM posts WHERE did = $1")
|
||||
.bind(&did)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Dispatch test: a `did:web:` DID must be routed to the
|
||||
/// `web_resolver` (not the PLC one). Without this routing, every
|
||||
/// `did:web:` post would stay `@<did-prefix>…` forever.
|
||||
@@ -606,9 +669,18 @@ mod tests {
|
||||
)]))
|
||||
.into_arc();
|
||||
|
||||
// The local PDS is consulted before the method dispatch (see the
|
||||
// module docs), and it does NOT host this DID — a foreign
|
||||
// `did:web:` is exactly the case where it answers "don't know".
|
||||
// Wiring one of the other stubs in here instead would make the
|
||||
// PDS claim a DID it doesn't have, and the test would be
|
||||
// asserting against the documented PDS-first rule rather than
|
||||
// against the method dispatch it's named for.
|
||||
let pds = StubResolver::new(HashMap::new()).into_arc();
|
||||
|
||||
let worker = HandleSyncWorker {
|
||||
db: db.clone(),
|
||||
pds_resolver: Arc::clone(&plc),
|
||||
pds_resolver: pds,
|
||||
plc_resolver: plc,
|
||||
web_resolver: web,
|
||||
interval_secs: 999,
|
||||
@@ -665,9 +737,16 @@ mod tests {
|
||||
let plc_arc: Arc<dyn DidHandleResolver> = Arc::new(plc);
|
||||
let web_arc: Arc<dyn DidHandleResolver> = Arc::new(web);
|
||||
|
||||
// A DID the local PDS does not host — otherwise the PDS-first
|
||||
// rule would (correctly) resolve it and this test would be
|
||||
// measuring the wrong thing. The "local PDS *does* host it"
|
||||
// case is covered by `pds_resolves_did_key_before_method_dispatch`.
|
||||
let pds_arc: Arc<dyn DidHandleResolver> =
|
||||
Arc::new(StubResolver::new(HashMap::new()));
|
||||
|
||||
let worker = HandleSyncWorker {
|
||||
db: db.clone(),
|
||||
pds_resolver: Arc::clone(&plc_arc),
|
||||
pds_resolver: pds_arc,
|
||||
plc_resolver: plc_arc,
|
||||
web_resolver: web_arc,
|
||||
interval_secs: 999,
|
||||
|
||||
+414
-27
@@ -693,24 +693,46 @@ where
|
||||
|
||||
// -- follows ---------------------------------------------------------------
|
||||
|
||||
/// Insert or update the follow edge `follower_did -> subject_did`.
|
||||
///
|
||||
/// `rkey` is the record key of the `app.bsky.graph.follow` record this
|
||||
/// edge came from. It is stored as a *second access path* to the row —
|
||||
/// the primary key stays `(follower_did, subject_did)`, which is what
|
||||
/// keeps this upsert idempotent across the push path, the firehose and
|
||||
/// any replay of either. See migration 0011 for the full reasoning.
|
||||
///
|
||||
/// Pass `None` only when the caller genuinely has no rkey. On conflict
|
||||
/// the column is `COALESCE(EXCLUDED.rkey, follows.rkey)`: a newer record
|
||||
/// overwrites it (youngest record wins, so a re-follow's rkey replaces
|
||||
/// the old one and a stale delete for the old rkey can no longer match),
|
||||
/// but a caller that omits the rkey must not blank out one another
|
||||
/// transport already recorded — that would re-open the very gap this
|
||||
/// column closes.
|
||||
pub async fn upsert_follow(
|
||||
db: &PgPool,
|
||||
follower_did: &str,
|
||||
subject_did: &str,
|
||||
rkey: Option<&str>,
|
||||
record: Option<&Value>,
|
||||
) -> Result<()> {
|
||||
let created_at = parse_created_at(
|
||||
record.and_then(|r| r.get("createdAt")).and_then(|v| v.as_str()),
|
||||
);
|
||||
// An empty rkey is not an rkey — treat it like the absent case so a
|
||||
// caller forwarding a blank field can't write a row that a
|
||||
// `WHERE rkey = ''` delete would later match by accident.
|
||||
let rkey = rkey.filter(|r| !r.is_empty());
|
||||
sqlx::query(
|
||||
r#"INSERT INTO follows (follower_did, subject_did, created_at)
|
||||
VALUES ($1, $2, $3)
|
||||
r#"INSERT INTO follows (follower_did, subject_did, rkey, created_at)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (follower_did, subject_did) DO UPDATE SET
|
||||
rkey = COALESCE(EXCLUDED.rkey, follows.rkey),
|
||||
created_at = EXCLUDED.created_at,
|
||||
indexed_at = now()"#,
|
||||
)
|
||||
.bind(follower_did)
|
||||
.bind(subject_did)
|
||||
.bind(rkey)
|
||||
.bind(created_at)
|
||||
.execute(db)
|
||||
.await?;
|
||||
@@ -733,6 +755,12 @@ pub async fn upsert_follow(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete the follow edge by its relationship identity.
|
||||
///
|
||||
/// This is the PDS-push path: `/internal/ingest-commit` carries the
|
||||
/// `subject_did` from the PDS's own snapshot, so the row can be
|
||||
/// addressed directly. Idempotent — deleting an edge that is already
|
||||
/// gone is a no-op, not an error.
|
||||
pub async fn delete_follow(
|
||||
db: &PgPool,
|
||||
follower_did: &str,
|
||||
@@ -748,6 +776,78 @@ pub async fn delete_follow(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete the follow edge that came from record `rkey` in
|
||||
/// `follower_did`'s repo.
|
||||
///
|
||||
/// This is the firehose / Jetstream path. A delete op carries only
|
||||
/// `did` + `rkey` and no record body, so the subject DID has to be
|
||||
/// recovered from the row itself — which is exactly what the `rkey`
|
||||
/// column added in migration 0011 is for. The `DELETE ... RETURNING`
|
||||
/// resolves and removes in one statement (the same shape
|
||||
/// [`delete_like`] uses to recover its `post_uri`), so there is no
|
||||
/// window in which another writer could move the row between the
|
||||
/// lookup and the delete.
|
||||
///
|
||||
/// Returns the `subject_did` that was unfollowed, or `None` when
|
||||
/// nothing matched. `None` is a normal outcome, never an error:
|
||||
///
|
||||
/// * the row predates migration 0011 and has no rkey (the push path
|
||||
/// with its `subject_did` still handles those), or
|
||||
/// * the delete already landed over the other transport, or
|
||||
/// * the follow was re-created under a newer rkey, in which case this
|
||||
/// delete is a stale replay and the live edge must be left alone.
|
||||
///
|
||||
/// The caller logs and moves on — an unfollow we cannot place must not
|
||||
/// stall the frames queued behind it.
|
||||
pub async fn delete_follow_by_rkey(
|
||||
db: &PgPool,
|
||||
follower_did: &str,
|
||||
rkey: &str,
|
||||
) -> Result<Option<String>> {
|
||||
if rkey.is_empty() {
|
||||
// Guard the degenerate case explicitly: `rkey = ''` can never
|
||||
// identify a record, and letting it through would mean an empty
|
||||
// value written by some future caller could be matched here.
|
||||
tracing::warn!(
|
||||
follower_did,
|
||||
"follow delete with an empty rkey; nothing to do"
|
||||
);
|
||||
return Ok(None);
|
||||
}
|
||||
// Deleted with `fetch_all` rather than `fetch_optional` because the
|
||||
// index on `(follower_did, rkey)` is deliberately not unique (see
|
||||
// migration 0011): in the pathological case of a duplicated rkey,
|
||||
// every matching row is a follow whose record is gone, so all of
|
||||
// them should go.
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
"DELETE FROM follows WHERE follower_did = $1 AND rkey = $2 \
|
||||
RETURNING subject_did",
|
||||
)
|
||||
.bind(follower_did)
|
||||
.bind(rkey)
|
||||
.fetch_all(db)
|
||||
.await?;
|
||||
|
||||
match rows.into_iter().next() {
|
||||
Some((subject_did,)) => {
|
||||
tracing::debug!(
|
||||
follower_did, rkey, subject_did,
|
||||
"applied an unfollow by rkey"
|
||||
);
|
||||
Ok(Some(subject_did))
|
||||
}
|
||||
None => {
|
||||
tracing::debug!(
|
||||
follower_did, rkey,
|
||||
"follow delete by rkey matched no row (already gone, \
|
||||
re-created under a newer rkey, or indexed before the \
|
||||
rkey column existed); skipping"
|
||||
);
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the subject DID from a follow record (`{ "subject": "did:..."}`).
|
||||
pub fn follow_subject_did(record: Option<&Value>) -> Option<String> {
|
||||
record?
|
||||
@@ -906,8 +1006,21 @@ pub async fn apply_commit(
|
||||
applied = true;
|
||||
}
|
||||
"app.bsky.graph.follow" => {
|
||||
let subject_did = match op.action.as_str() {
|
||||
"create" => match follow_subject_did(op.record.as_ref()) {
|
||||
// Both actions need the rkey. On a create it is stored
|
||||
// alongside the edge; on a delete it is the *only*
|
||||
// thing identifying the edge, because a delete op
|
||||
// carries no record body and therefore no subject DID.
|
||||
let rkey = op
|
||||
.rkey
|
||||
.clone()
|
||||
.or_else(|| {
|
||||
op.path
|
||||
.as_deref()
|
||||
.and_then(|p| p.rsplit('/').next().map(str::to_string))
|
||||
})
|
||||
.filter(|r| !r.is_empty());
|
||||
if op.action == "create" {
|
||||
let subject_did = match follow_subject_did(op.record.as_ref()) {
|
||||
Some(s) => s,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
@@ -915,33 +1028,35 @@ pub async fn apply_commit(
|
||||
);
|
||||
continue;
|
||||
}
|
||||
},
|
||||
"delete" => {
|
||||
// Jetstream delete on follows carries no record
|
||||
// value, so we can't know which subject was
|
||||
// unfollowed. The PDS-driven internal ingest path
|
||||
// handles this — it knows the subject from its
|
||||
// own snapshot.
|
||||
tracing::warn!(
|
||||
"follow delete via Jetstream lacks subject; \
|
||||
route through /internal/ingest-commit instead"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
if op.action == "create" {
|
||||
};
|
||||
upsert_follow(
|
||||
db,
|
||||
&ev.did,
|
||||
&subject_did,
|
||||
rkey.as_deref(),
|
||||
op.record.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
applied = true;
|
||||
} else if op.action == "delete" {
|
||||
delete_follow(db, &ev.did, &subject_did).await?;
|
||||
// The rkey → subject_did lookup added in migration
|
||||
// 0011. Before it, this arm could only log and skip,
|
||||
// which left unfollows depending entirely on the
|
||||
// PDS's best-effort push: one lost request and the
|
||||
// follow stayed indexed forever.
|
||||
let Some(rkey) = rkey else {
|
||||
tracing::warn!(
|
||||
did = %ev.did,
|
||||
"follow delete op has no rkey; skipping"
|
||||
);
|
||||
continue;
|
||||
};
|
||||
delete_follow_by_rkey(db, &ev.did, &rkey).await?;
|
||||
// Applied even when no row matched: the event was
|
||||
// understood and acted on, which is what this flag
|
||||
// reports (same as the post / like / repost deletes).
|
||||
applied = true;
|
||||
}
|
||||
applied = true;
|
||||
}
|
||||
"app.bsky.actor.profile" => {
|
||||
// Jetstream carries profile records as plain
|
||||
@@ -1356,8 +1471,10 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(count, 1);
|
||||
|
||||
// Delete via the internal API (not via Jetstream — Jetstream
|
||||
// delete on follows doesn't carry the subject).
|
||||
// Delete through the PDS-push path, which addresses the edge by
|
||||
// `(follower, subject)` because the PDS knows the subject from
|
||||
// its own snapshot. (The firehose path deletes by rkey instead
|
||||
// — see `firehose_unfollow_deletes_by_rkey` below.)
|
||||
delete_follow(&db, "did:plc:test", "did:plc:b").await.unwrap();
|
||||
let (count,): (i64,) = sqlx::query_as(
|
||||
"SELECT COUNT(*) FROM follows WHERE follower_did = $1 AND subject_did = $2",
|
||||
@@ -1369,6 +1486,274 @@ mod tests {
|
||||
.unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
// -- unfollow over the firehose ---------------------------------------
|
||||
//
|
||||
// The tests below cover the gap migration 0011 closes: a delete op
|
||||
// carries only `did` + `rkey`, so the edge has to be recoverable
|
||||
// from the rkey alone. They use per-run unique DIDs because the
|
||||
// suite shares one database with every other test module.
|
||||
|
||||
fn follow_did(tag: &str) -> String {
|
||||
format!("did:plc:follow_{}_{}", tag, uuid::Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
/// A commit event shaped like the ones `pds_firehose::events_from_frame`
|
||||
/// hands to `apply_commit`: single-op, `record` present on create and
|
||||
/// absent on delete.
|
||||
fn follow_event(
|
||||
did: &str,
|
||||
rkey: &str,
|
||||
action: &str,
|
||||
subject: Option<&str>,
|
||||
) -> JetstreamEvent {
|
||||
let mut commit = json!({
|
||||
"operation": action,
|
||||
"collection": "app.bsky.graph.follow",
|
||||
"rkey": rkey,
|
||||
"path": format!("app.bsky.graph.follow/{rkey}"),
|
||||
});
|
||||
if let Some(subject) = subject {
|
||||
commit["cid"] = json!("bafyfollow");
|
||||
commit["record"] = json!({
|
||||
"subject": subject,
|
||||
"createdAt": "2026-01-01T00:00:00Z",
|
||||
});
|
||||
}
|
||||
JetstreamEvent {
|
||||
did: did.to_string(),
|
||||
time_us: 1_700_000_000_000_000,
|
||||
kind: "commit".into(),
|
||||
commit: Some(commit),
|
||||
identity: None,
|
||||
account: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn follow_rows(db: &PgPool, follower: &str) -> Vec<(String, Option<String>)> {
|
||||
sqlx::query_as(
|
||||
"SELECT subject_did, rkey FROM follows WHERE follower_did = $1 \
|
||||
ORDER BY subject_did",
|
||||
)
|
||||
.bind(follower)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// The core case: a follow that arrived over the firehose is removed
|
||||
/// by a delete op that names nothing but the rkey.
|
||||
#[tokio::test]
|
||||
async fn firehose_unfollow_deletes_by_rkey() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let follower = follow_did("er");
|
||||
let subject = follow_did("ee");
|
||||
|
||||
apply_commit(&db, &follow_event(&follower, "frk1", "create", Some(&subject)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), Some("frk1".to_string()))],
|
||||
"the create must store the rkey next to the edge"
|
||||
);
|
||||
|
||||
// The delete op carries no record and no subject — only the rkey.
|
||||
let applied = apply_commit(&db, &follow_event(&follower, "frk1", "delete", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(applied, "a follow delete is now actionable, not skipped");
|
||||
assert!(
|
||||
follow_rows(&db, &follower).await.is_empty(),
|
||||
"the unfollow must remove the edge"
|
||||
);
|
||||
|
||||
// Replaying the same delete (reconnect, or the push path racing
|
||||
// the firehose) must stay a silent no-op.
|
||||
apply_commit(&db, &follow_event(&follower, "frk1", "delete", None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(follow_rows(&db, &follower).await.is_empty());
|
||||
|
||||
let _ = sqlx::query("DELETE FROM notifications WHERE recipient_did = $1")
|
||||
.bind(&subject)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// A delete for an rkey we never indexed resolves to nothing. That
|
||||
/// is a normal outcome (the follow was never seen, or is already
|
||||
/// gone), so it must not error and must not touch other rows.
|
||||
#[tokio::test]
|
||||
async fn delete_follow_by_unknown_rkey_is_a_noop() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let follower = follow_did("er");
|
||||
let subject = follow_did("ee");
|
||||
|
||||
upsert_follow(&db, &follower, &subject, Some("frk1"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let removed = delete_follow_by_rkey(&db, &follower, "no-such-rkey")
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(removed.is_none(), "an unknown rkey resolves to no subject");
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), Some("frk1".to_string()))],
|
||||
"an unmatched delete must leave every other edge alone"
|
||||
);
|
||||
|
||||
// An empty rkey is guarded separately — it can never identify a
|
||||
// record, and must not be allowed to match a blank column.
|
||||
assert!(delete_follow_by_rkey(&db, &follower, "")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
assert_eq!(follow_rows(&db, &follower).await.len(), 1);
|
||||
|
||||
delete_follow(&db, &follower, &subject).await.unwrap();
|
||||
let _ = sqlx::query("DELETE FROM notifications WHERE recipient_did = $1")
|
||||
.bind(&subject)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Rows written before migration 0011 have `rkey IS NULL`: there was
|
||||
/// nothing to backfill them from. A delete-by-rkey must not find
|
||||
/// them (and certainly must not match NULL against anything), while
|
||||
/// the push path that names the subject keeps working.
|
||||
#[tokio::test]
|
||||
async fn legacy_row_without_rkey_still_deletes_via_subject() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let follower = follow_did("legacy");
|
||||
let subject = follow_did("ee");
|
||||
|
||||
// Insert the way migration 0001 through 0010 did — no rkey.
|
||||
sqlx::query(
|
||||
"INSERT INTO follows (follower_did, subject_did, created_at) \
|
||||
VALUES ($1, $2, now())",
|
||||
)
|
||||
.bind(&follower)
|
||||
.bind(&subject)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let removed = delete_follow_by_rkey(&db, &follower, "frk1").await.unwrap();
|
||||
assert!(
|
||||
removed.is_none(),
|
||||
"a row with no rkey is unreachable by rkey — by design"
|
||||
);
|
||||
assert_eq!(follow_rows(&db, &follower).await.len(), 1);
|
||||
|
||||
// The PDS push, which carries the subject, still removes it.
|
||||
delete_follow(&db, &follower, &subject).await.unwrap();
|
||||
assert!(follow_rows(&db, &follower).await.is_empty());
|
||||
}
|
||||
|
||||
/// Follow → unfollow → follow again produces a fresh rkey. The edge
|
||||
/// must stay a single row (the primary key is the relationship, not
|
||||
/// the record), the newest rkey must win, and a stale delete for the
|
||||
/// old rkey must not tear down the live follow.
|
||||
#[tokio::test]
|
||||
async fn refollow_keeps_one_row_and_the_newest_rkey_wins() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let follower = follow_did("er");
|
||||
let subject = follow_did("ee");
|
||||
|
||||
apply_commit(&db, &follow_event(&follower, "frk1", "create", Some(&subject)))
|
||||
.await
|
||||
.unwrap();
|
||||
apply_commit(&db, &follow_event(&follower, "frk2", "create", Some(&subject)))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), Some("frk2".to_string()))],
|
||||
"one edge, carrying the youngest record's rkey"
|
||||
);
|
||||
|
||||
// The old rkey is stale: its delete must find nothing.
|
||||
assert!(delete_follow_by_rkey(&db, &follower, "frk1")
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none());
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await.len(),
|
||||
1,
|
||||
"a replayed delete for a superseded record must not unfollow"
|
||||
);
|
||||
|
||||
// The current rkey does delete it.
|
||||
assert_eq!(
|
||||
delete_follow_by_rkey(&db, &follower, "frk2").await.unwrap(),
|
||||
Some(subject.clone()),
|
||||
"the delete resolves the subject it removed"
|
||||
);
|
||||
assert!(follow_rows(&db, &follower).await.is_empty());
|
||||
|
||||
let _ = sqlx::query("DELETE FROM notifications WHERE recipient_did = $1")
|
||||
.bind(&subject)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
|
||||
/// A follow that first arrives over the PDS push (no rkey stored by
|
||||
/// an older AppView, or a caller that has none) and is then seen
|
||||
/// again over the firehose must end up with the rkey — otherwise
|
||||
/// the firehose could never delete it. And a later push that omits
|
||||
/// the rkey must not blank it out again.
|
||||
#[tokio::test]
|
||||
async fn rkey_is_filled_in_but_never_blanked_out() {
|
||||
let Some(db) = try_test_db().await else {
|
||||
eprintln!("appview DB unavailable; skipping");
|
||||
return;
|
||||
};
|
||||
let follower = follow_did("er");
|
||||
let subject = follow_did("ee");
|
||||
|
||||
upsert_follow(&db, &follower, &subject, None, None).await.unwrap();
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), None)]
|
||||
);
|
||||
|
||||
// The firehose replay of the same follow supplies the rkey.
|
||||
upsert_follow(&db, &follower, &subject, Some("frk1"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), Some("frk1".to_string()))]
|
||||
);
|
||||
|
||||
// A subsequent write without one must leave it in place.
|
||||
upsert_follow(&db, &follower, &subject, None, None).await.unwrap();
|
||||
assert_eq!(
|
||||
follow_rows(&db, &follower).await,
|
||||
vec![(subject.clone(), Some("frk1".to_string()))],
|
||||
"COALESCE keeps the rkey the other transport already gave us"
|
||||
);
|
||||
|
||||
delete_follow(&db, &follower, &subject).await.unwrap();
|
||||
let _ = sqlx::query("DELETE FROM notifications WHERE recipient_did = $1")
|
||||
.bind(&subject)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1555,7 +1940,9 @@ mod notification_tests {
|
||||
.await
|
||||
.unwrap();
|
||||
// Self-follow is legal in the protocol; it must stay silent too.
|
||||
upsert_follow(&db, &author, &author, None).await.unwrap();
|
||||
upsert_follow(&db, &author, &author, Some("frk1"), None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let total: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*)::BIGINT FROM notifications WHERE recipient_did = $1",
|
||||
@@ -1657,10 +2044,10 @@ mod notification_tests {
|
||||
seed_post(&db, &subject, "p1").await;
|
||||
|
||||
let record = json!({ "subject": subject, "createdAt": "2026-01-01T00:00:00Z" });
|
||||
upsert_follow(&db, &follower, &subject, Some(&record))
|
||||
upsert_follow(&db, &follower, &subject, Some("frk1"), Some(&record))
|
||||
.await
|
||||
.unwrap();
|
||||
upsert_follow(&db, &follower, &subject, Some(&record))
|
||||
upsert_follow(&db, &follower, &subject, Some("frk1"), Some(&record))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
|
||||
@@ -15,8 +15,31 @@
|
||||
//! }
|
||||
//! ```
|
||||
//!
|
||||
//! In production this endpoint would be protected with mTLS and a token
|
||||
//! minted by the PDS; for now it's open inside the cluster.
|
||||
//! ## Who may call this
|
||||
//!
|
||||
//! This is the AppView's only write path, and it is not a browser
|
||||
//! endpoint: it is excluded from the CORS layer in
|
||||
//! [`crate::routes::router`], because an `Access-Control-Allow-Origin`
|
||||
//! header here would only ever help a web page forge index entries.
|
||||
//!
|
||||
//! Authentication is the shared secret `APPVIEW_INGEST_SECRET`,
|
||||
//! compared in constant time against the caller's `X-Ingest-Secret`
|
||||
//! header:
|
||||
//!
|
||||
//! - **set** → enforced. A missing or wrong header is `401
|
||||
//! AuthenticationRequired`.
|
||||
//! - **unset** → anonymous writes are accepted, and the AppView shouts
|
||||
//! about it once at startup (see
|
||||
//! [`crate::auth::log_startup_posture`]). Refusing to start would
|
||||
//! break every existing single-machine dev setup for a service that,
|
||||
//! in that configuration, is bound to loopback anyway; accepting
|
||||
//! silently is how an internet-facing deployment ends up letting
|
||||
//! anyone forge posts, follows and notifications. So: keep working,
|
||||
//! but never quietly.
|
||||
//!
|
||||
//! A future hardening step is mTLS or a PDS-minted token, at which
|
||||
//! point the shared secret becomes the fallback rather than the only
|
||||
//! line.
|
||||
|
||||
use crate::indexer;
|
||||
use crate::state::AppState;
|
||||
@@ -51,8 +74,13 @@ pub struct IngestCommitReq {
|
||||
}
|
||||
|
||||
/// Authenticate internal ingest requests.
|
||||
/// - If `APPVIEW_INGEST_SECRET` env var is unset: dev mode, accept anything.
|
||||
/// - If set: require `X-Ingest-Secret: <value>` header to match.
|
||||
///
|
||||
/// - `APPVIEW_INGEST_SECRET` unset: accept anonymous writes (dev mode —
|
||||
/// the startup log warns, see the module docs for why this isn't a
|
||||
/// hard failure).
|
||||
/// - Set: require a matching `X-Ingest-Secret` header. The comparison
|
||||
/// is constant-time so a caller can't recover the secret byte by byte
|
||||
/// from response timings.
|
||||
pub fn check_ingest_secret(
|
||||
headers: &HeaderMap,
|
||||
configured: Option<&str>,
|
||||
@@ -192,10 +220,15 @@ async fn apply(
|
||||
.map(str::to_string)
|
||||
})
|
||||
.ok_or_else(|| bad_request("follow create requires subject_did or record.subject"))?;
|
||||
// Forward the rkey too. The push path doesn't need it to
|
||||
// apply *this* write — it has the subject — but storing it
|
||||
// is what lets a later firehose delete (which carries only
|
||||
// did + rkey) find this row. See migration 0011.
|
||||
indexer::upsert_follow(
|
||||
&state.db,
|
||||
&req.did,
|
||||
&subject,
|
||||
Some(req.rkey.as_str()),
|
||||
req.record.as_ref(),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -3,9 +3,13 @@
|
||||
//! tests under `tests/` import from here so they can build a worker
|
||||
//! against a stub resolver without booting the binary.
|
||||
|
||||
pub mod auth;
|
||||
pub mod car;
|
||||
pub mod cbor;
|
||||
pub mod firehose;
|
||||
pub mod handle_sync;
|
||||
pub mod indexer;
|
||||
pub mod ingest;
|
||||
pub mod pds_firehose;
|
||||
pub mod routes;
|
||||
pub mod state;
|
||||
|
||||
@@ -7,10 +7,14 @@ use tokio::sync::mpsc;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
mod auth;
|
||||
mod car;
|
||||
mod cbor;
|
||||
mod firehose;
|
||||
mod handle_sync;
|
||||
mod indexer;
|
||||
mod ingest;
|
||||
mod pds_firehose;
|
||||
mod routes;
|
||||
mod state;
|
||||
|
||||
@@ -81,8 +85,59 @@ async fn main() -> Result<()> {
|
||||
});
|
||||
}
|
||||
|
||||
// Local PDS firehose. The push path (`/internal/ingest-commit`) is
|
||||
// the fast way a local commit reaches the index; this is the
|
||||
// guaranteed one — it carries a durable cursor, so anything the
|
||||
// push dropped while the AppView was down is replayed on connect.
|
||||
// See the module docs in `pds_firehose.rs` for why both exist.
|
||||
if cfg.pds_firehose_enabled {
|
||||
let start_seq = pds_firehose::cursor_get(&db).await.unwrap_or(0);
|
||||
let base = cfg.pds_base_url();
|
||||
info!(
|
||||
url = %pds_firehose::subscribe_url(&base, (start_seq > 0).then_some(start_seq)),
|
||||
cursor = start_seq,
|
||||
"starting the local PDS firehose consumer"
|
||||
);
|
||||
let consumer = pds_firehose::PdsFirehose::new(db.clone(), base, stats.clone())
|
||||
.with_max_backoff_secs(30);
|
||||
tokio::spawn(consumer.run_forever());
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"PDS_FIREHOSE_ENABLED=false — local commits reach the index only through \
|
||||
the best-effort `/internal/ingest-commit` push; a push lost to a restart \
|
||||
or a network error will NOT be recovered"
|
||||
);
|
||||
}
|
||||
|
||||
let state = AppState::new(cfg.clone(), db.clone(), stats.clone());
|
||||
|
||||
// Announce every relaxed security switch before we serve anything.
|
||||
auth::log_startup_posture(&cfg);
|
||||
|
||||
// Pre-load the PDS's published signing key so the first
|
||||
// authenticated request doesn't pay for the round trip.
|
||||
//
|
||||
// Best effort on purpose: the AppView has always started
|
||||
// independently of the PDS, and in a compose file the two race. A
|
||||
// hard failure here would turn "the PDS booted two seconds later"
|
||||
// into "the AppView is in a crash loop". If the fetch fails, the
|
||||
// first authenticated request retries it — and answers `503
|
||||
// AuthUnavailable` if the PDS is still unreachable. Never open.
|
||||
if cfg.appview_auth_required {
|
||||
match state.pds_keys.refresh().await {
|
||||
Ok(_) => info!(
|
||||
url = %state.pds_keys.did_doc_url(),
|
||||
"loaded PDS signing key for token verification"
|
||||
),
|
||||
Err(e) => tracing::warn!(
|
||||
url = %state.pds_keys.did_doc_url(),
|
||||
error = %e,
|
||||
"could not load the PDS signing key at startup; will retry on the \
|
||||
first authenticated request (which fails with 503 until it works)"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Back-fill the `handle` column on posts that the Jetstream
|
||||
// indexer inserted with an empty placeholder. The worker dispatches
|
||||
// by DID method: `did:plc:` → PLC directory, `did:web:` → a
|
||||
@@ -102,12 +157,12 @@ async fn main() -> Result<()> {
|
||||
// inside docker compose) — `pds_public_url` may not be reachable
|
||||
// from inside the cluster when TLS / DNS is set up for outside
|
||||
// clients only.
|
||||
let pds_base_url = cfg
|
||||
.pds_internal_url
|
||||
.clone()
|
||||
.unwrap_or_else(|| cfg.pds_public_url.clone());
|
||||
//
|
||||
// `AppConfig::pds_base_url()` owns that fallback so the handle
|
||||
// resolver and the signing-key fetch in `auth.rs` can never end up
|
||||
// pointed at different PDS instances.
|
||||
let pds_resolver: Arc<dyn at_identity::DidHandleResolver> = Arc::new(
|
||||
at_identity::pds_handle::PdsHandleResolver::new(pds_base_url),
|
||||
at_identity::pds_handle::PdsHandleResolver::new(cfg.pds_base_url()),
|
||||
);
|
||||
|
||||
let handle_sync = handle_sync::HandleSyncWorker {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+201
-42
@@ -8,6 +8,30 @@
|
||||
//! - `ingest_commit`: the internal-only writer used by the PDS, owned
|
||||
//! in `crate::ingest`.
|
||||
//!
|
||||
//! ## Public vs. private
|
||||
//!
|
||||
//! Most of what the AppView serves is public by construction: in AT
|
||||
//! Proto a post, a profile, a follow edge and a like are records in a
|
||||
//! public repo, replicated over the firehose. Those endpoints
|
||||
//! (`/api/profile*`, `/api/search`, `/api/post/*`, `/api/thread*`,
|
||||
//! `/api/followers`, `/api/following`) need no credential.
|
||||
//!
|
||||
//! Two things are *not* public, and they are the reason this service
|
||||
//! has authentication at all:
|
||||
//!
|
||||
//! - `/api/notifications*` — who interacted with you, and the read
|
||||
//! state of that list. Nothing in the protocol makes it readable by
|
||||
//! anyone but the recipient.
|
||||
//! - `/api/timeline/home` — the timeline is assembled from the
|
||||
//! viewer's follow graph, so serving it to an arbitrary `did`
|
||||
//! parameter answers "what does this person's feed look like" for
|
||||
//! any DID a caller cares to type.
|
||||
//!
|
||||
//! Both require a valid PDS-issued access token whose `sub` equals the
|
||||
//! `did` in the request ([`crate::auth`]). The check is two steps on
|
||||
//! purpose: the extractor proves *who* the caller is, and
|
||||
//! `ensure_matches` proves they are asking about themselves.
|
||||
//!
|
||||
//! The cursor format used by `timeline_home` is opaque: it's a
|
||||
//! `base64url(micros):uri` pair, which is what [`cursor::encode`] and
|
||||
//! [`cursor::decode`] produce/consume.
|
||||
@@ -19,11 +43,14 @@ use axum::{
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use axum::http::{header, HeaderValue, Method};
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::auth::AuthedDid;
|
||||
use crate::state::AppState;
|
||||
|
||||
pub mod cursor;
|
||||
@@ -35,25 +62,73 @@ use types::{
|
||||
ProfileResponse, SearchResponse, ThreadFullResponse, TimelineResponse,
|
||||
};
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
// CORS: the Tauri webview's origin is the Vite dev server
|
||||
// (`http://127.0.0.1:1430`) in dev or the bundled `tauri://` /
|
||||
// `asset://` origin in production. Either way it's a cross-origin
|
||||
// fetch against this service's `http://127.0.0.1:2584` listen
|
||||
// address, so the browser blocks the response without an explicit
|
||||
// allow-origin header. We allow any origin — the AppView's
|
||||
// public read endpoints (`/api/...`) carry no auth cookie and
|
||||
// the AppView runs alongside the user's own PDS, not on the
|
||||
// open internet; production deployments behind a reverse proxy
|
||||
// can tighten this via the proxy itself.
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods(Any)
|
||||
.allow_headers(Any);
|
||||
/// Build the CORS layer for the browser-facing routes.
|
||||
///
|
||||
/// The Tauri webview is a cross-origin caller: in dev its origin is the
|
||||
/// Vite server (`http://127.0.0.1:1430`), in a packaged build it is a
|
||||
/// platform-specific scheme — `tauri://localhost` on macOS/iOS,
|
||||
/// `http://tauri.localhost` on Windows. Either way the browser drops
|
||||
/// the response unless we send `Access-Control-Allow-Origin`.
|
||||
///
|
||||
/// `APPVIEW_CORS_ORIGINS` is a comma-separated allowlist, e.g.
|
||||
/// `tauri://localhost,http://127.0.0.1:1430`. When it is unset we keep
|
||||
/// the historic wildcard so no existing deployment breaks on upgrade —
|
||||
/// [`crate::auth::log_startup_posture`] warns about that at startup.
|
||||
///
|
||||
/// `Authorization` has to be in `allow_headers`: it is not a
|
||||
/// CORS-safelisted header, so without it the browser's preflight fails
|
||||
/// and the authenticated endpoints become unreachable from the webview
|
||||
/// — with an error that looks nothing like an auth problem.
|
||||
///
|
||||
/// `allow_credentials` stays off. We authenticate with a bearer token
|
||||
/// the client attaches deliberately, never with an ambient cookie, so
|
||||
/// there is nothing for a hostile page to replay — and turning it on
|
||||
/// would additionally make the wildcard origin illegal.
|
||||
fn cors_layer(cfg: &at_shared::config::AppConfig) -> CorsLayer {
|
||||
let base = CorsLayer::new()
|
||||
.allow_methods([Method::GET, Method::POST, Method::OPTIONS])
|
||||
.allow_headers([
|
||||
header::AUTHORIZATION,
|
||||
header::CONTENT_TYPE,
|
||||
header::ACCEPT,
|
||||
]);
|
||||
|
||||
Router::new()
|
||||
if cfg.appview_cors_origins.is_empty() {
|
||||
return base.allow_origin(Any);
|
||||
}
|
||||
|
||||
// Anything unparseable as a header value is dropped with a warning
|
||||
// rather than panicking the process — a stray quote in the env var
|
||||
// must not take the service down. If *every* entry is bad we fall
|
||||
// back to the wildcard and say so, because an empty allowlist would
|
||||
// silently break every browser client.
|
||||
let mut origins: Vec<HeaderValue> = Vec::new();
|
||||
for raw in &cfg.appview_cors_origins {
|
||||
match HeaderValue::from_str(raw) {
|
||||
Ok(v) => origins.push(v),
|
||||
Err(_) => warn!(origin = %raw, "APPVIEW_CORS_ORIGINS: ignoring unparseable origin"),
|
||||
}
|
||||
}
|
||||
if origins.is_empty() {
|
||||
warn!("APPVIEW_CORS_ORIGINS contained no usable origin; falling back to allow-any");
|
||||
return base.allow_origin(Any);
|
||||
}
|
||||
base.allow_origin(AllowOrigin::list(origins))
|
||||
}
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
let cors = cors_layer(&state.cfg);
|
||||
|
||||
// Browser-facing surface. Everything here answers a `fetch()` from
|
||||
// the desktop client, so it carries the CORS layer.
|
||||
let api = Router::new()
|
||||
.route("/", get(root))
|
||||
// --- private: token required, `sub` must equal `did` ---
|
||||
.route("/api/timeline/home", get(timeline_home))
|
||||
.route("/api/notifications", get(notifications))
|
||||
.route("/api/notifications/count", get(notifications_count))
|
||||
.route("/api/notifications/seen", post(notifications_seen))
|
||||
// --- public: AT Proto public records ---
|
||||
.route("/api/profile", get(profile_query))
|
||||
.route("/api/profile/:handle", get(profile_path))
|
||||
.route("/api/search", get(search))
|
||||
@@ -66,15 +141,20 @@ pub fn router(state: AppState) -> Router {
|
||||
// implementation, so they can't drift.
|
||||
.route("/api/thread", get(thread_query))
|
||||
.route("/api/thread/*uri", get(thread_path))
|
||||
.route("/api/notifications", get(notifications))
|
||||
.route("/api/notifications/count", get(notifications_count))
|
||||
.route("/api/notifications/seen", post(notifications_seen))
|
||||
.route("/api/followers", get(followers))
|
||||
.route("/api/following", get(following))
|
||||
.route("/healthz", get(healthz))
|
||||
.route("/internal/ingest-commit", post(crate::ingest::ingest_commit))
|
||||
.layer(cors)
|
||||
.with_state(state)
|
||||
.layer(cors);
|
||||
|
||||
// Server-to-server surface. `/internal/ingest-commit` is called by
|
||||
// the PDS with a shared secret, never by a browser, so it stays
|
||||
// outside the CORS layer: handing it an
|
||||
// `Access-Control-Allow-Origin` header would only ever help a web
|
||||
// page try to write to the index.
|
||||
let internal = Router::new()
|
||||
.route("/internal/ingest-commit", post(crate::ingest::ingest_commit));
|
||||
|
||||
api.merge(internal).with_state(state)
|
||||
}
|
||||
|
||||
async fn root() -> Json<Value> {
|
||||
@@ -115,13 +195,21 @@ const MAX_LIMIT: i64 = 100;
|
||||
/// surface.
|
||||
const MAX_FOLLOWED_DIDS: usize = 1000;
|
||||
|
||||
/// `GET /api/timeline/home?did=…`
|
||||
///
|
||||
/// **Authenticated.** The timeline is derived from the viewer's follow
|
||||
/// graph, so `did` must be the caller's own DID — otherwise this
|
||||
/// endpoint would answer "what does this account's feed look like" for
|
||||
/// any DID at all.
|
||||
async fn timeline_home(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthedDid,
|
||||
Query(q): Query<TimelineQuery>,
|
||||
) -> Result<Json<TimelineResponse>, (StatusCode, Json<Value>)> {
|
||||
if q.did.is_empty() {
|
||||
return Err(bad_request("did is required"));
|
||||
}
|
||||
auth.ensure_matches(&q.did)?;
|
||||
let limit = clamp_limit(q.limit);
|
||||
|
||||
// Look up the set of DIDs this user follows, then build the
|
||||
@@ -230,20 +318,51 @@ async fn timeline_home(
|
||||
.map_err(db_err)?,
|
||||
}
|
||||
} else {
|
||||
// Graph-aware branch: filter `posts.did` to the followee set
|
||||
// plus the requesting user's own DID. `target_dids` has been
|
||||
// deduped and capped at MAX_FOLLOWED_DIDS, and the user's own
|
||||
// DID is guaranteed to be in the set.
|
||||
// Graph-aware branch: the followee set plus the requesting
|
||||
// user's own DID. `target_dids` has been deduped and capped at
|
||||
// MAX_FOLLOWED_DIDS, and the user's own DID is guaranteed to
|
||||
// be in the set.
|
||||
//
|
||||
// ## Why this is a LATERAL and not `did = ANY($2)`
|
||||
//
|
||||
// The straightforward `WHERE did = ANY($2) ORDER BY indexed_at
|
||||
// DESC LIMIT n` is a plan-stability trap once
|
||||
// `posts_feed_indexed_at_uri_idx` exists (migration 0009, added
|
||||
// for the cold-start feed). The planner sees an index that
|
||||
// already yields rows in `indexed_at DESC` order and assumes it
|
||||
// will hit `n` matching rows early — so it walks the global
|
||||
// feed and filters. When the followees are sparse (a fresh
|
||||
// account following accounts that haven't posted), "early"
|
||||
// means millions of rows: measured on the dev instance, 2.87 M
|
||||
// rows discarded and 28 s per request, while the same query
|
||||
// took 62 ms with the per-DID index. It also flipped between
|
||||
// the two plans depending on how often the prepared statement
|
||||
// had run, so it looked intermittent.
|
||||
//
|
||||
// Expressing the intent — "for each followee, their newest
|
||||
// posts, merged" — takes that plan off the table: `unnest` is a
|
||||
// relation the planner can size, and each iteration is a bounded
|
||||
// range scan on `posts_did_indexed_at_uri_idx`. Fetching `$1`
|
||||
// per followee is what makes it correct: the global top-N is
|
||||
// always a subset of the union of the per-followee top-Ns.
|
||||
match cursor_ts {
|
||||
Some(ts) => sqlx::query_as::<_, PostRowWithIndexed>(
|
||||
r#"SELECT uri, did, handle, rkey, collection, text, cid,
|
||||
parent_uri, root_uri, embed, langs, created_at,
|
||||
indexed_at
|
||||
FROM posts
|
||||
WHERE collection IN ('app.twi.post','app.bsky.feed.post')
|
||||
AND did = ANY($2::text[])
|
||||
AND (indexed_at, uri) < ($3, $4)
|
||||
ORDER BY indexed_at DESC, uri DESC
|
||||
r#"SELECT t.uri, t.did, t.handle, t.rkey, t.collection,
|
||||
t.text, t.cid, t.parent_uri, t.root_uri,
|
||||
t.embed, t.langs, t.created_at, t.indexed_at
|
||||
FROM unnest($2::text[]) AS f(did)
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT p.uri, p.did, p.handle, p.rkey, p.collection,
|
||||
p.text, p.cid, p.parent_uri, p.root_uri,
|
||||
p.embed, p.langs, p.created_at, p.indexed_at
|
||||
FROM posts p
|
||||
WHERE p.did = f.did
|
||||
AND p.collection IN ('app.twi.post','app.bsky.feed.post')
|
||||
AND (p.indexed_at, p.uri) < ($3, $4)
|
||||
ORDER BY p.indexed_at DESC, p.uri DESC
|
||||
LIMIT $1
|
||||
) t
|
||||
ORDER BY t.indexed_at DESC, t.uri DESC
|
||||
LIMIT $1"#,
|
||||
)
|
||||
.bind(fetch)
|
||||
@@ -254,13 +373,21 @@ async fn timeline_home(
|
||||
.await
|
||||
.map_err(db_err)?,
|
||||
None => sqlx::query_as::<_, PostRowWithIndexed>(
|
||||
r#"SELECT uri, did, handle, rkey, collection, text, cid,
|
||||
parent_uri, root_uri, embed, langs, created_at,
|
||||
indexed_at
|
||||
FROM posts
|
||||
WHERE collection IN ('app.twi.post','app.bsky.feed.post')
|
||||
AND did = ANY($2::text[])
|
||||
ORDER BY indexed_at DESC, uri DESC
|
||||
r#"SELECT t.uri, t.did, t.handle, t.rkey, t.collection,
|
||||
t.text, t.cid, t.parent_uri, t.root_uri,
|
||||
t.embed, t.langs, t.created_at, t.indexed_at
|
||||
FROM unnest($2::text[]) AS f(did)
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT p.uri, p.did, p.handle, p.rkey, p.collection,
|
||||
p.text, p.cid, p.parent_uri, p.root_uri,
|
||||
p.embed, p.langs, p.created_at, p.indexed_at
|
||||
FROM posts p
|
||||
WHERE p.did = f.did
|
||||
AND p.collection IN ('app.twi.post','app.bsky.feed.post')
|
||||
ORDER BY p.indexed_at DESC, p.uri DESC
|
||||
LIMIT $1
|
||||
) t
|
||||
ORDER BY t.indexed_at DESC, t.uri DESC
|
||||
LIMIT $1"#,
|
||||
)
|
||||
.bind(fetch)
|
||||
@@ -1046,13 +1173,19 @@ struct NotificationsQuery {
|
||||
/// of the list. The tiebreak here is the row's `id` rather than a URI
|
||||
/// (a notification has no URI of its own), which the shared
|
||||
/// [`cursor`] codec carries in its string slot.
|
||||
///
|
||||
/// **Authenticated**: `did` is the recipient, so the caller has to be
|
||||
/// that recipient. This is the endpoint that made authentication
|
||||
/// necessary in the first place.
|
||||
async fn notifications(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthedDid,
|
||||
Query(q): Query<NotificationsQuery>,
|
||||
) -> Result<Json<NotificationsResponse>, (StatusCode, Json<Value>)> {
|
||||
if q.did.is_empty() {
|
||||
return Err(bad_request("did is required"));
|
||||
}
|
||||
auth.ensure_matches(&q.did)?;
|
||||
let limit = clamp_limit(q.limit);
|
||||
let keyset = decode_cursor(q.cursor.as_deref())?;
|
||||
// The cursor's string slot holds the row id. A client that hands
|
||||
@@ -1121,13 +1254,18 @@ struct NotificationCountQuery {
|
||||
/// scales with the number of *unread* rows, not the user's lifetime
|
||||
/// notification history. That matters because the client polls this
|
||||
/// for its tray badge.
|
||||
///
|
||||
/// **Authenticated**, same rule as the list itself: an unread count is
|
||||
/// still information about someone else's inbox.
|
||||
async fn notifications_count(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthedDid,
|
||||
Query(q): Query<NotificationCountQuery>,
|
||||
) -> Result<Json<NotificationCountResponse>, (StatusCode, Json<Value>)> {
|
||||
if q.did.is_empty() {
|
||||
return Err(bad_request("did is required"));
|
||||
}
|
||||
auth.ensure_matches(&q.did)?;
|
||||
let count: i64 = sqlx::query_scalar(
|
||||
"SELECT COUNT(*)::BIGINT FROM notifications \
|
||||
WHERE recipient_did = $1 AND read_at IS NULL",
|
||||
@@ -1165,13 +1303,20 @@ struct NotificationsSeenReq {
|
||||
/// updates nothing and reports `updated: 0`. `read_at` is set to
|
||||
/// `now()` (when we recorded the ack), not to `seenAt` (which is a
|
||||
/// client-supplied watermark and could be arbitrarily far in the past).
|
||||
///
|
||||
/// **Authenticated**, and the only *write* among the private
|
||||
/// endpoints: without the check anyone could clear another user's
|
||||
/// unread badge. `AuthedDid` runs before `Json` because the body
|
||||
/// extractor consumes the request — axum requires body extractors last.
|
||||
async fn notifications_seen(
|
||||
State(state): State<AppState>,
|
||||
auth: AuthedDid,
|
||||
Json(req): Json<NotificationsSeenReq>,
|
||||
) -> Result<Json<NotificationSeenResponse>, (StatusCode, Json<Value>)> {
|
||||
if req.did.is_empty() {
|
||||
return Err(bad_request("did is required"));
|
||||
}
|
||||
auth.ensure_matches(&req.did)?;
|
||||
let res = sqlx::query(
|
||||
r#"UPDATE notifications
|
||||
SET read_at = now()
|
||||
@@ -1393,6 +1538,16 @@ async fn actor_list(
|
||||
|
||||
// -- healthz ----------------------------------------------------------------
|
||||
|
||||
/// Liveness probe.
|
||||
///
|
||||
/// Both ingest streams report separately. `jetstream_connected` is the
|
||||
/// public network's view; `pds_firehose_connected` is the local PDS's
|
||||
/// guaranteed delivery path for our own users' records. A deployment
|
||||
/// can be perfectly healthy for reads with the first one down, but a
|
||||
/// `pds_firehose_enabled: true, pds_firehose_connected: false` pair
|
||||
/// means local commits are riding on the best-effort push alone — which
|
||||
/// is exactly the state an operator wants to see in a probe rather than
|
||||
/// discover from a missing post.
|
||||
async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let stats = &state.stats;
|
||||
Json(json!({
|
||||
@@ -1400,6 +1555,10 @@ async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
|
||||
"lag_ms": stats.lag_ms(),
|
||||
"events_processed": stats.events_processed(),
|
||||
"jetstream_connected": stats.jetstream_connected(),
|
||||
"pds_firehose_enabled": state.cfg.pds_firehose_enabled,
|
||||
"pds_firehose_connected": stats.pds_connected(),
|
||||
"pds_firehose_frames": stats.pds_frames_processed(),
|
||||
"pds_firehose_seq": stats.pds_last_seq(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
@@ -2,18 +2,30 @@ use at_shared::config::AppConfig;
|
||||
use sqlx::PgPool;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::auth::PdsKeys;
|
||||
use crate::firehose::Stats;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
#[allow(dead_code)]
|
||||
pub cfg: AppConfig,
|
||||
pub db: PgPool,
|
||||
pub stats: Arc<Stats>,
|
||||
/// Cache of the PDS's published signing key, used by the
|
||||
/// [`crate::auth::AuthedDid`] extractor. Shared (`Arc`) so every
|
||||
/// handler verifies against the same cached key and one HTTP
|
||||
/// client, and so a key rotation picked up by one request is
|
||||
/// immediately visible to the rest.
|
||||
pub pds_keys: Arc<PdsKeys>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub fn new(cfg: AppConfig, db: PgPool, stats: Arc<Stats>) -> Self {
|
||||
Self { cfg, db, stats }
|
||||
let pds_keys = Arc::new(PdsKeys::from_config(&cfg));
|
||||
Self {
|
||||
cfg,
|
||||
db,
|
||||
stats,
|
||||
pds_keys,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,9 @@
|
||||
//! rather than panicking — so `cargo test --workspace` stays green in
|
||||
//! environments where the appview hasn't been started.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestAuth;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -46,6 +49,36 @@ async fn db_reachable() -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// How this suite authenticates against `/api/timeline/home`, which is
|
||||
/// no longer public. `None` means the AppView enforces auth but the
|
||||
/// test process cannot mint a token (no `PDS_JWT_SECRET`), in which
|
||||
/// case the test skips like it does for a missing service.
|
||||
async fn auth_or_skip() -> Option<TestAuth> {
|
||||
TestAuth::probe(&client().await, APPVIEW_URL).await
|
||||
}
|
||||
|
||||
/// `GET /api/timeline/home` as `did`, with the bearer token attached
|
||||
/// when the instance requires one. The seeded DIDs are synthetic, so
|
||||
/// the token is minted from the PDS's own signing secret — see
|
||||
/// `tests/common/mod.rs`.
|
||||
async fn get_timeline(
|
||||
c: &reqwest::Client,
|
||||
auth: &TestAuth,
|
||||
did: &str,
|
||||
extra: &[(&str, &str)],
|
||||
) -> reqwest::Response {
|
||||
let mut params: Vec<(&str, &str)> = vec![("did", did)];
|
||||
params.extend_from_slice(extra);
|
||||
auth.apply(
|
||||
c.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(¶ms),
|
||||
did,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn post_ingest(c: &reqwest::Client, body: Value) -> reqwest::Response {
|
||||
c.post(format!("{APPVIEW_URL}/internal/ingest-commit"))
|
||||
.json(&body)
|
||||
@@ -120,6 +153,7 @@ async fn timeline_returns_seeded_posts() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let did = did_for_test("tl");
|
||||
|
||||
// Seed 3 posts with distinct rkeys.
|
||||
@@ -157,12 +191,7 @@ async fn timeline_returns_seeded_posts() {
|
||||
// machine that has run this suite twice) the three rows we just
|
||||
// seeded fall outside a 10-row window and the assertions below
|
||||
// fail for reasons that have nothing to do with the timeline.
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "100")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "100")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().expect("posts is array");
|
||||
@@ -222,6 +251,7 @@ async fn timeline_paginates_with_cursor() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let did = did_for_test("pg");
|
||||
|
||||
// Seed 50 posts.
|
||||
@@ -246,28 +276,14 @@ async fn timeline_paginates_with_cursor() {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
// Page 1: limit=20.
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "20")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "20")]).await;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let page1 = body["posts"].as_array().unwrap().clone();
|
||||
let cursor1 = body["cursor"].as_str().expect("page1 cursor");
|
||||
assert_eq!(page1.len(), 20, "page1 should be exactly 20");
|
||||
|
||||
// Page 2: with cursor.
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[
|
||||
("did", did.as_str()),
|
||||
("limit", "20"),
|
||||
("cursor", cursor1),
|
||||
])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "20"), ("cursor", cursor1)]).await;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let page2 = body["posts"].as_array().unwrap().clone();
|
||||
assert_eq!(page2.len(), 20, "page2 should be exactly 20");
|
||||
@@ -285,16 +301,7 @@ async fn timeline_paginates_with_cursor() {
|
||||
|
||||
// Page 3: tail — fewer than 20 expected, cursor=null.
|
||||
let cursor2 = body["cursor"].as_str().expect("page2 cursor");
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[
|
||||
("did", did.as_str()),
|
||||
("limit", "20"),
|
||||
("cursor", cursor2),
|
||||
])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "20"), ("cursor", cursor2)]).await;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let page3 = body["posts"].as_array().unwrap().clone();
|
||||
assert!(page3.len() <= 20, "page3 should be <= 20");
|
||||
@@ -484,6 +491,7 @@ async fn timeline_filters_to_followees() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let url = std::env::var("DATABASE_URL_APPVIEW").unwrap();
|
||||
let pool = sqlx::PgPool::connect(&url).await.unwrap();
|
||||
|
||||
@@ -503,12 +511,7 @@ async fn timeline_filters_to_followees() {
|
||||
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", alice.as_str()), ("limit", "100")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &alice, &[("limit", "100")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().expect("posts is array");
|
||||
@@ -555,18 +558,14 @@ async fn timeline_includes_own_posts() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let alice = did_for_test("alone");
|
||||
|
||||
// Alice posts without seeding any follows.
|
||||
seed_posts(&c, &alice, &["alice's first post", "alice's second post"]).await;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", alice.as_str()), ("limit", "100")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &alice, &[("limit", "100")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().expect("posts is array");
|
||||
@@ -605,6 +604,7 @@ async fn timeline_caps_followee_list() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let url = std::env::var("DATABASE_URL_APPVIEW").unwrap();
|
||||
let pool = sqlx::PgPool::connect(&url).await.unwrap();
|
||||
|
||||
@@ -624,12 +624,7 @@ async fn timeline_caps_followee_list() {
|
||||
seed_posts(&c, &alice, &["poweruser post"]).await;
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", alice.as_str()), ("limit", "50")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &alice, &[("limit", "50")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().expect("posts is array");
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
//! End-to-end tests for AppView authentication.
|
||||
//!
|
||||
//! Unlike the other suites in this directory these use a **real**
|
||||
//! token: the test creates an account on the running PDS
|
||||
//! (`com.atproto.server.createAccount`, the same call the desktop
|
||||
//! client makes at signup) and presents the access JWT it gets back to
|
||||
//! the AppView. That is the whole point — it exercises the actual key
|
||||
//! distribution path (PDS signs → publishes its public key at
|
||||
//! `/.well-known/did.json` → AppView fetches and verifies), which a
|
||||
//! self-minted token would bypass.
|
||||
//!
|
||||
//! Fail-open like the rest of the suites: if the PDS or the AppView
|
||||
//! isn't running, or the AppView runs with `APPVIEW_AUTH_REQUIRED=false`,
|
||||
//! the test prints a notice and returns successfully.
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use std::time::Duration;
|
||||
|
||||
const PDS_URL: &str = "http://127.0.0.1:2583";
|
||||
|
||||
fn appview_url() -> String {
|
||||
std::env::var("APPVIEW_TEST_URL").unwrap_or_else(|_| "http://127.0.0.1:2584".to_string())
|
||||
}
|
||||
|
||||
/// 30 s, not the 5 s the sibling suites use.
|
||||
///
|
||||
/// The authenticated happy path for `/api/timeline/home` runs the
|
||||
/// cold-start query (a brand-new account follows nobody, so the handler
|
||||
/// falls back to the global recent feed). On a developer machine whose
|
||||
/// AppView has been indexing the public firehose for a while that scan
|
||||
/// takes seconds — a timeout there would look like an auth failure and
|
||||
/// is nothing of the sort.
|
||||
fn client() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn up(c: &reqwest::Client, base: &str) -> bool {
|
||||
for _ in 0..20 {
|
||||
if let Ok(r) = c.get(format!("{base}/healthz")).send().await {
|
||||
if r.status().is_success() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// A freshly created PDS account.
|
||||
struct Account {
|
||||
did: String,
|
||||
access_jwt: String,
|
||||
refresh_jwt: String,
|
||||
}
|
||||
|
||||
/// Guard for every test here: both services up **and** the AppView
|
||||
/// actually enforcing auth. Returns the client plus a new account.
|
||||
async fn ready() -> Option<(reqwest::Client, Account)> {
|
||||
let c = client();
|
||||
if !up(&c, &appview_url()).await {
|
||||
eprintln!("appview not running, skipping");
|
||||
return None;
|
||||
}
|
||||
if !up(&c, PDS_URL).await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return None;
|
||||
}
|
||||
// Probe: does this instance enforce auth? An operator running with
|
||||
// `APPVIEW_AUTH_REQUIRED=false` (the mode the other suites use)
|
||||
// would otherwise see every assertion here fail for the one reason
|
||||
// that isn't a bug.
|
||||
let probe = c
|
||||
.get(format!("{}/api/notifications/count", appview_url()))
|
||||
.query(&[("did", "did:plc:auth_probe")])
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if probe.status().as_u16() != 401 {
|
||||
eprintln!(
|
||||
"appview does not enforce auth (probe returned {}), skipping",
|
||||
probe.status()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
let acc = create_account(&c).await?;
|
||||
Some((c, acc))
|
||||
}
|
||||
|
||||
async fn create_account(c: &reqwest::Client) -> Option<Account> {
|
||||
let handle = format!("auth_{}.maarcadetweet.local", uuid::Uuid::new_v4().simple());
|
||||
let r: Value = c
|
||||
.post(format!("{PDS_URL}/xrpc/com.atproto.server.createAccount"))
|
||||
.json(&json!({ "handle": handle, "password": "hunter2hunter2" }))
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.json()
|
||||
.await
|
||||
.ok()?;
|
||||
Some(Account {
|
||||
did: r["did"].as_str()?.to_string(),
|
||||
access_jwt: r["access_jwt"].as_str()?.to_string(),
|
||||
refresh_jwt: r["refresh_jwt"].as_str()?.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// `GET /api/notifications?did=…` with an optional bearer token.
|
||||
async fn get_notifications(
|
||||
c: &reqwest::Client,
|
||||
did: &str,
|
||||
token: Option<&str>,
|
||||
) -> reqwest::Response {
|
||||
let mut rb = c
|
||||
.get(format!("{}/api/notifications", appview_url()))
|
||||
.query(&[("did", did)]);
|
||||
if let Some(t) = token {
|
||||
rb = rb.bearer_auth(t);
|
||||
}
|
||||
rb.send().await.unwrap()
|
||||
}
|
||||
|
||||
/// Assert the AppView's standard error envelope: the status, and the
|
||||
/// `error` code the desktop client branches on.
|
||||
async fn assert_error(r: reqwest::Response, status: u16, code: &str) {
|
||||
let got = r.status().as_u16();
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert_eq!(got, status, "unexpected status; body = {body}");
|
||||
assert_eq!(body["error"], json!(code), "unexpected error code: {body}");
|
||||
assert!(
|
||||
body["message"].is_string(),
|
||||
"error body must carry a message: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_token_reads_own_notifications() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let r = get_notifications(&c, &acc.did, Some(&acc.access_jwt)).await;
|
||||
assert_eq!(r.status().as_u16(), 200);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
// A brand-new account has no notifications, but the shape must be
|
||||
// the normal list response, not an error.
|
||||
assert!(body["notifications"].is_array(), "body = {body}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn no_token_is_401_auth_missing() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let r = get_notifications(&c, &acc.did, None).await;
|
||||
assert_error(r, 401, "AuthMissing").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreign_did_is_403() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
// A valid token, but asking about somebody else's inbox. This is
|
||||
// the case the endpoint used to answer with a 200.
|
||||
let r = get_notifications(&c, "did:plc:somebodyelse", Some(&acc.access_jwt)).await;
|
||||
assert_error(r, 403, "Forbidden").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn garbage_token_is_401_token_invalid() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let r = get_notifications(&c, &acc.did, Some("not.a.jwt")).await;
|
||||
assert_error(r, 401, "TokenInvalid").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_token_is_not_accepted() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
// Signed by the same key and valid for 90 days — only the `scope`
|
||||
// claim separates it from a session credential.
|
||||
let r = get_notifications(&c, &acc.did, Some(&acc.refresh_jwt)).await;
|
||||
assert_error(r, 401, "TokenInvalid").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn seen_write_requires_matching_token() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let url = format!("{}/api/notifications/seen", appview_url());
|
||||
|
||||
// Somebody else's unread badge: 403, nothing written.
|
||||
let r = c
|
||||
.post(&url)
|
||||
.bearer_auth(&acc.access_jwt)
|
||||
.json(&json!({ "did": "did:plc:somebodyelse" }))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_error(r, 403, "Forbidden").await;
|
||||
|
||||
// No credential at all: 401.
|
||||
let r = c
|
||||
.post(&url)
|
||||
.json(&json!({ "did": &acc.did }))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_error(r, 401, "AuthMissing").await;
|
||||
|
||||
// Own inbox: allowed (zero rows updated — the account is new).
|
||||
let r = c
|
||||
.post(&url)
|
||||
.bearer_auth(&acc.access_jwt)
|
||||
.json(&json!({ "did": &acc.did }))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status().as_u16(), 200);
|
||||
let body: Value = r.json().await.unwrap();
|
||||
assert_eq!(body["ok"], json!(true));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timeline_home_is_authenticated() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let base = appview_url();
|
||||
|
||||
let r = c
|
||||
.get(format!("{base}/api/timeline/home"))
|
||||
.query(&[("did", acc.did.as_str())])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_error(r, 401, "AuthMissing").await;
|
||||
|
||||
let r = c
|
||||
.get(format!("{base}/api/timeline/home"))
|
||||
.query(&[("did", "did:plc:somebodyelse")])
|
||||
.bearer_auth(&acc.access_jwt)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_error(r, 403, "Forbidden").await;
|
||||
|
||||
let r = c
|
||||
.get(format!("{base}/api/timeline/home"))
|
||||
.query(&[("did", acc.did.as_str())])
|
||||
.bearer_auth(&acc.access_jwt)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(r.status().as_u16(), 200);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn public_endpoints_stay_public() {
|
||||
let Some((c, acc)) = ready().await else { return };
|
||||
let base = appview_url();
|
||||
|
||||
// Profiles, search, follower lists and the health probe are public
|
||||
// records in AT Proto. Putting them behind auth would be a
|
||||
// behaviour change no protocol rule asks for — assert they still
|
||||
// answer without a token.
|
||||
for (path, params) in [
|
||||
("/api/profile", vec![("did", acc.did.as_str())]),
|
||||
("/api/search", vec![("q", "hello")]),
|
||||
("/api/followers", vec![("did", acc.did.as_str())]),
|
||||
("/api/following", vec![("did", acc.did.as_str())]),
|
||||
("/healthz", vec![]),
|
||||
] {
|
||||
let r = c
|
||||
.get(format!("{base}{path}"))
|
||||
.query(¶ms)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
r.status().as_u16(),
|
||||
200,
|
||||
"{path} must remain public, got {}",
|
||||
r.status()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The AppView can only verify anything because the PDS publishes its
|
||||
/// public key. If this document ever stops carrying a usable key, every
|
||||
/// authenticated request degrades to `503 AuthUnavailable` — so assert
|
||||
/// the shape the AppView parses.
|
||||
#[tokio::test]
|
||||
async fn pds_publishes_a_usable_signing_key() {
|
||||
let c = client();
|
||||
if !up(&c, PDS_URL).await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let doc: Value = c
|
||||
.get(format!("{PDS_URL}/.well-known/did.json"))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let id = doc["id"].as_str().expect("did document needs an id");
|
||||
assert!(id.starts_with("did:web:"), "id = {id}");
|
||||
let vm = &doc["verificationMethod"][0];
|
||||
assert_eq!(vm["type"], json!("Multikey"));
|
||||
assert_eq!(vm["controller"], json!(id));
|
||||
let key = vm["publicKeyMultibase"]
|
||||
.as_str()
|
||||
.expect("verificationMethod needs publicKeyMultibase");
|
||||
// base58-btc multibase: the `z` prefix is what the AppView's
|
||||
// decoder expects.
|
||||
assert!(key.starts_with('z'), "key = {key}");
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Shared test support for the AppView integration suites.
|
||||
//!
|
||||
//! ## Why the suites need this
|
||||
//!
|
||||
//! `/api/notifications*` and `/api/timeline/home` require a
|
||||
//! PDS-issued access token whose `sub` equals the `did` in the request.
|
||||
//! The suites, however, seed synthetic DIDs (`did:plc:ntf_…`) through
|
||||
//! `/internal/ingest-commit` — accounts the PDS has never heard of, so
|
||||
//! there is no `createSession` that would hand out a token for them.
|
||||
//!
|
||||
//! The way out is that a token is just an ES256 JWT signed with the
|
||||
//! server key derived from `PDS_JWT_SECRET`. A test that can read that
|
||||
//! secret (from the process environment, or from the repo `.env` the
|
||||
//! dev stack itself was started with) can mint a token for any DID it
|
||||
//! likes — the same thing `pds-server/src/jwt_issuer.rs` does.
|
||||
//!
|
||||
//! ## Fail-open, like the rest of the suites
|
||||
//!
|
||||
//! [`TestAuth::probe`] asks the running AppView whether it enforces
|
||||
//! auth at all:
|
||||
//!
|
||||
//! - not enforcing (`APPVIEW_AUTH_REQUIRED=false`) → no header needed;
|
||||
//! - enforcing and we have the secret → mint per-DID tokens;
|
||||
//! - enforcing and we don't → `None`, and the caller skips, exactly as
|
||||
//! it already skips when the service or the database is down.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use at_crypto::ecdsa::P256Keypair;
|
||||
use at_crypto::jwt::{issue_jwt, JwtClaims};
|
||||
|
||||
/// Audience the PDS stamps into access tokens — and, since the
|
||||
/// audience check landed, the value the AppView insists on: its own
|
||||
/// service DID, derived from `APPVIEW_PUBLIC_URL`. A token minted with
|
||||
/// anything else is rejected as `TokenInvalid`, which is exactly what
|
||||
/// we want a wrong value here to look like.
|
||||
///
|
||||
/// Derived the same way `AppConfig::appview_did()` does it, from the
|
||||
/// same environment variable, so this helper can't drift from the
|
||||
/// service it's impersonating the PDS for.
|
||||
fn appview_aud() -> String {
|
||||
let url = std::env::var("APPVIEW_PUBLIC_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:2584".to_string());
|
||||
at_shared::config::did_web_from_url(&url)
|
||||
}
|
||||
|
||||
/// The scope the AppView insists on. A token with any other scope —
|
||||
/// `com.atproto.refresh`, say — is rejected with `TokenInvalid`.
|
||||
pub const ACCESS_SCOPE: &str = "com.atproto.access";
|
||||
|
||||
/// How the suite should authenticate against the AppView under test.
|
||||
///
|
||||
/// `Clone` because a test that pages through results in a closure has
|
||||
/// to hand each iteration its own copy, exactly like the client and the
|
||||
/// base URL next to it.
|
||||
#[derive(Clone)]
|
||||
pub enum TestAuth {
|
||||
/// `APPVIEW_AUTH_REQUIRED=false`: send no `Authorization` header.
|
||||
Disabled,
|
||||
/// Auth is enforced; mint tokens with this hex secret.
|
||||
Secret(String),
|
||||
}
|
||||
|
||||
impl TestAuth {
|
||||
/// Decide how (or whether) this suite can talk to the AppView.
|
||||
///
|
||||
/// Returns `None` when the AppView enforces auth but no
|
||||
/// `PDS_JWT_SECRET` is reachable — the caller should print a notice
|
||||
/// and return, keeping `cargo test --workspace` green on a machine
|
||||
/// without the dev stack's environment.
|
||||
pub async fn probe(c: &reqwest::Client, base_url: &str) -> Option<Self> {
|
||||
// An unauthenticated probe against a private endpoint. We only
|
||||
// look at the status: 401 means the extractor is active. A DID
|
||||
// that doesn't exist is fine — the auth check runs first.
|
||||
let status = c
|
||||
.get(format!("{base_url}/api/notifications/count"))
|
||||
.query(&[("did", "did:plc:auth_probe")])
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.status()
|
||||
.as_u16();
|
||||
if status != 401 && status != 503 {
|
||||
return Some(TestAuth::Disabled);
|
||||
}
|
||||
match pds_jwt_secret() {
|
||||
Some(secret) => Some(TestAuth::Secret(secret)),
|
||||
None => {
|
||||
eprintln!(
|
||||
"appview enforces auth (probe returned {status}) but PDS_JWT_SECRET \
|
||||
is not set and no .env was found — skipping"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach an `Authorization: Bearer` header for `did`, if needed.
|
||||
pub fn apply(&self, rb: reqwest::RequestBuilder, did: &str) -> reqwest::RequestBuilder {
|
||||
match self {
|
||||
TestAuth::Disabled => rb,
|
||||
TestAuth::Secret(secret) => match mint_access_jwt(secret, did, ACCESS_SCOPE, 3600) {
|
||||
Some(token) => rb.bearer_auth(token),
|
||||
None => rb,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// A token for `did` — for tests that want to send a *wrong* one on
|
||||
/// purpose. `None` when auth is disabled, in which case the test
|
||||
/// that needs it should skip.
|
||||
pub fn token_for(&self, did: &str) -> Option<String> {
|
||||
match self {
|
||||
TestAuth::Disabled => None,
|
||||
TestAuth::Secret(secret) => mint_access_jwt(secret, did, ACCESS_SCOPE, 3600),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `PDS_JWT_SECRET` from the environment, falling back to the repo
|
||||
/// `.env` — the same file the running dev stack loaded at startup, so
|
||||
/// the minted tokens verify against the key the PDS actually publishes.
|
||||
pub fn pds_jwt_secret() -> Option<String> {
|
||||
if let Ok(v) = std::env::var("PDS_JWT_SECRET") {
|
||||
if !v.trim().is_empty() {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
// `dotenvy::dotenv` walks up from the current directory, which for
|
||||
// a test binary is the crate root — so this finds the workspace
|
||||
// `.env` two levels up. It never overrides a real env var.
|
||||
let _ = dotenvy::dotenv();
|
||||
std::env::var("PDS_JWT_SECRET")
|
||||
.ok()
|
||||
.filter(|v| !v.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Mint an access JWT for `did`, signed with the PDS's server key.
|
||||
///
|
||||
/// Mirrors `pds-server/src/jwt_issuer.rs`: the P-256 secret scalar is
|
||||
/// the **first 32 bytes** of `PDS_JWT_SECRET` (the config allows a
|
||||
/// longer secret), i.e. the first 64 hex characters.
|
||||
///
|
||||
/// `ttl_secs` may be negative to build a deliberately expired token.
|
||||
pub fn mint_access_jwt(
|
||||
secret_hex: &str,
|
||||
did: &str,
|
||||
scope: &str,
|
||||
ttl_secs: i64,
|
||||
) -> Option<String> {
|
||||
let hex = secret_hex.trim().trim_start_matches("0x");
|
||||
if hex.len() < 64 || !hex.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
eprintln!("PDS_JWT_SECRET is not ≥32 bytes of hex; cannot mint a test token");
|
||||
return None;
|
||||
}
|
||||
let kp = P256Keypair {
|
||||
secret_hex: hex[..64].to_string(),
|
||||
// Only the signing half is used by `issue_jwt`; the verifier
|
||||
// fetches the public key from the PDS's DID document.
|
||||
public_multibase: String::new(),
|
||||
};
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
issue_jwt(
|
||||
&kp,
|
||||
&JwtClaims {
|
||||
iss: "did:web:test".into(),
|
||||
sub: did.to_string(),
|
||||
aud: appview_aud(),
|
||||
iat: now - 1,
|
||||
exp: now + ttl_secs,
|
||||
jti: None,
|
||||
scope: Some(scope.to_string()),
|
||||
},
|
||||
)
|
||||
.ok()
|
||||
}
|
||||
@@ -15,6 +15,9 @@
|
||||
//! and returns rather than panicking. The point of the tests is to
|
||||
//! catch regressions in CI where the service IS up.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestAuth;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -50,6 +53,32 @@ async fn db_reachable() -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// `/api/timeline/home` requires a token whose `sub` is the requested
|
||||
/// DID. The DIDs here are synthetic, so the token is minted from the
|
||||
/// PDS signing secret — see `tests/common/mod.rs`. `None` → skip.
|
||||
async fn auth_or_skip() -> Option<TestAuth> {
|
||||
TestAuth::probe(&client().await, APPVIEW_URL).await
|
||||
}
|
||||
|
||||
/// `GET /api/timeline/home` as `did`, authenticated when required.
|
||||
async fn get_timeline(
|
||||
c: &reqwest::Client,
|
||||
auth: &TestAuth,
|
||||
did: &str,
|
||||
extra: &[(&str, &str)],
|
||||
) -> reqwest::Response {
|
||||
let mut params: Vec<(&str, &str)> = vec![("did", did)];
|
||||
params.extend_from_slice(extra);
|
||||
auth.apply(
|
||||
c.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(¶ms),
|
||||
did,
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn post_ingest(c: &reqwest::Client, body: Value) -> reqwest::Response {
|
||||
c.post(format!("{APPVIEW_URL}/internal/ingest-commit"))
|
||||
.json(&body)
|
||||
@@ -125,6 +154,7 @@ async fn timeline_includes_embed() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let did = did_for_test("img");
|
||||
let uri = seed_post(
|
||||
&c,
|
||||
@@ -160,12 +190,7 @@ async fn timeline_includes_embed() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "10")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "10")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().unwrap();
|
||||
@@ -199,6 +224,7 @@ async fn timeline_includes_external_embed() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let did = did_for_test("ext");
|
||||
let uri = seed_post(
|
||||
&c,
|
||||
@@ -222,12 +248,7 @@ async fn timeline_includes_external_embed() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "10")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "10")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let posts = body["posts"].as_array().unwrap();
|
||||
@@ -241,12 +262,7 @@ async fn timeline_includes_external_embed() {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "10")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "10")]).await;
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
our = body["posts"]
|
||||
.as_array()
|
||||
@@ -410,6 +426,7 @@ async fn timeline_post_without_embed_has_null_embed() {
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let Some(auth) = auth_or_skip().await else { return };
|
||||
let did = did_for_test("plain");
|
||||
let uri = seed_post(
|
||||
&c,
|
||||
@@ -421,12 +438,7 @@ async fn timeline_post_without_embed_has_null_embed() {
|
||||
)
|
||||
.await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{APPVIEW_URL}/api/timeline/home"))
|
||||
.query(&[("did", did.as_str()), ("limit", "10")])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let resp = get_timeline(&c, &auth, &did, &[("limit", "10")]).await;
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let our = body["posts"]
|
||||
|
||||
@@ -175,8 +175,13 @@ async fn sync_resolves_known_did() {
|
||||
seed_post(&db, &did, "rkb", "", "second").await.unwrap();
|
||||
assert_eq!(count_empty_handle_for(&db, &did).await.unwrap(), 2);
|
||||
|
||||
// Drive the resolve half with our own DID. `run_once()` scans
|
||||
// globally, ordered by DID and capped at BATCH_SIZE, so on a
|
||||
// database that a live indexer keeps topping up, a freshly seeded
|
||||
// DID isn't guaranteed to make the batch — the assertions below
|
||||
// would then be measuring someone else's rows.
|
||||
let worker = worker_with(db.clone(), stub.clone());
|
||||
let report: SyncReport = worker.run_once().await.unwrap();
|
||||
let report: SyncReport = worker.resolve_batch(vec![did.clone()]).await.unwrap();
|
||||
assert_eq!(report.resolved, 2, "{report:?}");
|
||||
assert_eq!(report.failed, 0);
|
||||
assert_eq!(report.skipped, 0);
|
||||
@@ -224,9 +229,16 @@ async fn sync_skips_already_resolved() {
|
||||
.into_arc();
|
||||
|
||||
let worker = worker_with(db.clone(), stub.clone());
|
||||
let report = worker.run_once().await.unwrap();
|
||||
assert_eq!(report.resolved, 0, "{report:?}");
|
||||
assert_eq!(report.failed, 0);
|
||||
// This test is about the SELECT: a DID whose rows already carry a
|
||||
// handle must never reach a resolver in the first place. So assert
|
||||
// on `select_candidates()` rather than forcing the DID through
|
||||
// `resolve_batch` — that would consult the resolver by definition
|
||||
// and defeat the `query_count == 0` check below.
|
||||
let candidates = worker.select_candidates().await.unwrap();
|
||||
assert!(
|
||||
!candidates.contains(&did),
|
||||
"a DID that already has a handle must not be selected"
|
||||
);
|
||||
|
||||
// Both rows must still carry the pre-existing handle.
|
||||
let (cnt,): (i64,) = sqlx::query_as(
|
||||
@@ -279,7 +291,16 @@ async fn sync_respects_limit() {
|
||||
let stub = StubResolver::new(mapping).into_arc();
|
||||
|
||||
let worker = worker_with(db.clone(), stub.clone());
|
||||
let report = worker.run_once().await.unwrap();
|
||||
// The cap lives in the SELECT, so assert it there; the report of a
|
||||
// full `run_once()` depends on what else is pending database-wide.
|
||||
let candidates = worker.select_candidates().await.unwrap();
|
||||
assert!(
|
||||
candidates.len() as i64 <= BATCH_SIZE,
|
||||
"select must never exceed BATCH_SIZE, got {}",
|
||||
candidates.len()
|
||||
);
|
||||
let batch: Vec<String> = all_dids.iter().take(BATCH_SIZE as usize).cloned().collect();
|
||||
let report = worker.resolve_batch(batch).await.unwrap();
|
||||
assert_eq!(
|
||||
report.resolved as i64,
|
||||
BATCH_SIZE,
|
||||
@@ -339,7 +360,7 @@ async fn sync_skips_unresolvable_dids() {
|
||||
let stub = StubResolver::new(HashMap::new()).into_arc();
|
||||
|
||||
let worker = worker_with(db.clone(), stub.clone());
|
||||
let report = worker.run_once().await.unwrap();
|
||||
let report = worker.resolve_batch(vec![did.clone()]).await.unwrap();
|
||||
assert_eq!(report.resolved, 0);
|
||||
assert_eq!(report.failed, 0);
|
||||
assert_eq!(report.skipped, 1, "{report:?}");
|
||||
@@ -388,14 +409,21 @@ async fn sync_resolves_did_web_via_web_resolver() {
|
||||
let plc_arc: Arc<dyn DidHandleResolver> = plc.into_arc();
|
||||
let web_arc: Arc<dyn DidHandleResolver> = web.into_arc();
|
||||
|
||||
// The local PDS is consulted before the method dispatch and does
|
||||
// not host a foreign `did:web:` — wiring one of the other stubs in
|
||||
// here would make it claim a DID it doesn't have, and the test
|
||||
// would assert against the PDS-first rule instead of the dispatch.
|
||||
let pds_arc: Arc<dyn DidHandleResolver> =
|
||||
StubResolver::new(HashMap::new()).into_arc();
|
||||
|
||||
let worker = HandleSyncWorker {
|
||||
db: db.clone(),
|
||||
pds_resolver: Arc::clone(&plc_arc),
|
||||
pds_resolver: pds_arc,
|
||||
plc_resolver: plc_arc,
|
||||
web_resolver: web_arc,
|
||||
interval_secs: 999,
|
||||
};
|
||||
let report = worker.run_once().await.unwrap();
|
||||
let report = worker.resolve_batch(vec![did.clone()]).await.unwrap();
|
||||
assert_eq!(
|
||||
report.resolved, 1,
|
||||
"did:web must resolve through the web resolver, got {report:?}"
|
||||
@@ -442,14 +470,19 @@ async fn sync_resolves_did_plc_via_plc_resolver() {
|
||||
let plc_arc: Arc<dyn DidHandleResolver> = plc.into_arc();
|
||||
let web_arc: Arc<dyn DidHandleResolver> = web.into_arc();
|
||||
|
||||
// Same reasoning as the did:web test: the PDS doesn't host this
|
||||
// DID, so the method dispatch is what's under test.
|
||||
let pds_arc: Arc<dyn DidHandleResolver> =
|
||||
StubResolver::new(HashMap::new()).into_arc();
|
||||
|
||||
let worker = HandleSyncWorker {
|
||||
db: db.clone(),
|
||||
pds_resolver: Arc::clone(&plc_arc),
|
||||
pds_resolver: pds_arc,
|
||||
plc_resolver: plc_arc,
|
||||
web_resolver: web_arc,
|
||||
interval_secs: 999,
|
||||
};
|
||||
let report = worker.run_once().await.unwrap();
|
||||
let report = worker.resolve_batch(vec![did.clone()]).await.unwrap();
|
||||
assert_eq!(
|
||||
report.resolved, 1,
|
||||
"did:plc must resolve through the PLC resolver, got {report:?}"
|
||||
|
||||
@@ -14,6 +14,9 @@
|
||||
//! API. That's deliberate: it's the only way to catch a mismatch
|
||||
//! between what the write path stores and what the read path joins.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::TestAuth;
|
||||
use serde_json::{json, Value};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -55,8 +58,12 @@ async fn db_pool() -> Option<sqlx::PgPool> {
|
||||
}
|
||||
|
||||
/// Guard used at the top of every test. Returns `None` (→ skip) unless
|
||||
/// both the HTTP service and the database are up.
|
||||
async fn ready() -> Option<(reqwest::Client, sqlx::PgPool)> {
|
||||
/// the HTTP service is up, the database is reachable, **and** we know
|
||||
/// how to authenticate: `/api/notifications*` now requires a token
|
||||
/// whose `sub` is the requested DID, and the DIDs seeded here are
|
||||
/// synthetic, so the token has to be minted from the PDS's signing
|
||||
/// secret. See `tests/common/mod.rs`.
|
||||
async fn ready() -> Option<(reqwest::Client, sqlx::PgPool, TestAuth)> {
|
||||
if !wait_for_appview_db().await {
|
||||
eprintln!("appview not running, skipping");
|
||||
return None;
|
||||
@@ -65,7 +72,32 @@ async fn ready() -> Option<(reqwest::Client, sqlx::PgPool)> {
|
||||
eprintln!("appview DB unreachable, skipping");
|
||||
return None;
|
||||
};
|
||||
Some((client().await, pool))
|
||||
let c = client().await;
|
||||
let auth = TestAuth::probe(&c, &appview_url()).await?;
|
||||
Some((c, pool, auth))
|
||||
}
|
||||
|
||||
/// `GET <url>` against a private endpoint, carrying the token for
|
||||
/// `did` when the instance enforces auth.
|
||||
fn authed_get(
|
||||
c: &reqwest::Client,
|
||||
auth: &TestAuth,
|
||||
url: String,
|
||||
did: &str,
|
||||
) -> reqwest::RequestBuilder {
|
||||
auth.apply(c.get(url), did)
|
||||
}
|
||||
|
||||
/// `POST <url>` against a private endpoint. Same rule as
|
||||
/// [`authed_get`] — `/api/notifications/seen` is a write into one
|
||||
/// user's read state.
|
||||
fn authed_post(
|
||||
c: &reqwest::Client,
|
||||
auth: &TestAuth,
|
||||
url: String,
|
||||
did: &str,
|
||||
) -> reqwest::RequestBuilder {
|
||||
auth.apply(c.post(url), did)
|
||||
}
|
||||
|
||||
async fn post_ingest(c: &reqwest::Client, body: Value) -> reqwest::Response {
|
||||
@@ -181,7 +213,7 @@ async fn seed_follow(c: &reqwest::Client, follower: &str, subject: &str) {
|
||||
#[tokio::test]
|
||||
async fn notifications_list_count_and_seen() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let alice = did_for_test("alice");
|
||||
@@ -196,8 +228,7 @@ async fn notifications_list_count_and_seen() {
|
||||
let reply_uri = seed_reply(&c, &carol, &post_uri, &post_uri, "carol's reply").await;
|
||||
seed_follow(&c, &bob, &alice).await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str()), ("limit", "50")])
|
||||
.send()
|
||||
.await
|
||||
@@ -253,8 +284,7 @@ async fn notifications_list_count_and_seen() {
|
||||
}
|
||||
|
||||
// The unread count agrees with the list.
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications/count"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications/count"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -264,8 +294,7 @@ async fn notifications_list_count_and_seen() {
|
||||
assert_eq!(body["count"], json!(3));
|
||||
|
||||
// Mark everything seen.
|
||||
let resp = c
|
||||
.post(format!("{base}/api/notifications/seen"))
|
||||
let resp = authed_post(&c, &auth, format!("{base}/api/notifications/seen"), &alice)
|
||||
.json(&json!({ "did": alice }))
|
||||
.send()
|
||||
.await
|
||||
@@ -276,8 +305,7 @@ async fn notifications_list_count_and_seen() {
|
||||
assert_eq!(body["updated"], json!(3));
|
||||
|
||||
// Idempotent: a second call updates nothing and still succeeds.
|
||||
let resp = c
|
||||
.post(format!("{base}/api/notifications/seen"))
|
||||
let resp = authed_post(&c, &auth, format!("{base}/api/notifications/seen"), &alice)
|
||||
.json(&json!({ "did": alice }))
|
||||
.send()
|
||||
.await
|
||||
@@ -286,8 +314,7 @@ async fn notifications_list_count_and_seen() {
|
||||
assert_eq!(body["updated"], json!(0));
|
||||
|
||||
// Count is now zero and the rows carry a read_at.
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications/count"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications/count"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -295,8 +322,7 @@ async fn notifications_list_count_and_seen() {
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["count"], json!(0));
|
||||
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -311,8 +337,7 @@ async fn notifications_list_count_and_seen() {
|
||||
"/api/notifications",
|
||||
"/api/notifications/count",
|
||||
] {
|
||||
let resp = c
|
||||
.get(format!("{base}{path}"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}{path}"), &alice)
|
||||
.query(&[("did", "")])
|
||||
.send()
|
||||
.await
|
||||
@@ -326,7 +351,7 @@ async fn notifications_list_count_and_seen() {
|
||||
#[tokio::test]
|
||||
async fn notifications_skip_self_interactions() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let alice = did_for_test("solo");
|
||||
@@ -335,8 +360,7 @@ async fn notifications_skip_self_interactions() {
|
||||
seed_reply(&c, &alice, &post_uri, &post_uri, "and replying too").await;
|
||||
seed_follow(&c, &alice, &alice).await;
|
||||
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -355,7 +379,7 @@ async fn notifications_skip_self_interactions() {
|
||||
#[tokio::test]
|
||||
async fn notifications_paginate_with_cursor() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let alice = did_for_test("popular");
|
||||
@@ -372,9 +396,9 @@ async fn notifications_paginate_with_cursor() {
|
||||
let c = c.clone();
|
||||
let alice = alice.clone();
|
||||
let base = base.clone();
|
||||
let auth = auth.clone();
|
||||
async move {
|
||||
let mut req = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let mut req = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str()), ("limit", "5")]);
|
||||
if let Some(cur) = cursor {
|
||||
req = req.query(&[("cursor", cur.as_str())]);
|
||||
@@ -417,8 +441,7 @@ async fn notifications_paginate_with_cursor() {
|
||||
assert_eq!(all.len(), 12);
|
||||
|
||||
// A mangled cursor is a 400, not a silent restart at page 1.
|
||||
let resp = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let resp = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str()), ("cursor", "!!!garbage!!!")])
|
||||
.send()
|
||||
.await
|
||||
@@ -431,7 +454,7 @@ async fn notifications_paginate_with_cursor() {
|
||||
#[tokio::test]
|
||||
async fn notifications_seen_respects_watermark() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let alice = did_for_test("watermark");
|
||||
@@ -442,8 +465,7 @@ async fn notifications_seen_respects_watermark() {
|
||||
|
||||
// Read back the first notification's indexed_at — that's the
|
||||
// watermark a client would echo after rendering page 1.
|
||||
let body: Value = c
|
||||
.get(format!("{base}/api/notifications"))
|
||||
let body: Value = authed_get(&c, &auth, format!("{base}/api/notifications"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -461,8 +483,7 @@ async fn notifications_seen_respects_watermark() {
|
||||
let second = did_for_test("late");
|
||||
seed_like(&c, &second, &post_uri).await;
|
||||
|
||||
let resp = c
|
||||
.post(format!("{base}/api/notifications/seen"))
|
||||
let resp = authed_post(&c, &auth, format!("{base}/api/notifications/seen"), &alice)
|
||||
.json(&json!({ "did": alice, "seenAt": watermark }))
|
||||
.send()
|
||||
.await
|
||||
@@ -475,8 +496,7 @@ async fn notifications_seen_respects_watermark() {
|
||||
);
|
||||
|
||||
// The later one is still unread.
|
||||
let body: Value = c
|
||||
.get(format!("{base}/api/notifications/count"))
|
||||
let body: Value = authed_get(&c, &auth, format!("{base}/api/notifications/count"), &alice)
|
||||
.query(&[("did", alice.as_str())])
|
||||
.send()
|
||||
.await
|
||||
@@ -487,8 +507,7 @@ async fn notifications_seen_respects_watermark() {
|
||||
assert_eq!(body["count"], json!(1));
|
||||
|
||||
// snake_case spelling must work identically.
|
||||
let resp = c
|
||||
.post(format!("{base}/api/notifications/seen"))
|
||||
let resp = authed_post(&c, &auth, format!("{base}/api/notifications/seen"), &alice)
|
||||
.json(&json!({ "did": alice, "seen_at": null }))
|
||||
.send()
|
||||
.await
|
||||
@@ -502,7 +521,7 @@ async fn notifications_seen_respects_watermark() {
|
||||
#[tokio::test]
|
||||
async fn followers_and_following_lists() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, _auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let hub = did_for_test("hub");
|
||||
@@ -591,7 +610,8 @@ async fn followers_and_following_lists() {
|
||||
assert_eq!(unique.len(), seen.len(), "paged followers repeat: {seen:?}");
|
||||
assert_eq!(seen.len(), 3, "paging lost a follower: {seen:?}");
|
||||
|
||||
// `did` is mandatory.
|
||||
// `did` is mandatory. These two stay public — a follow edge is a
|
||||
// public record — so no token is involved.
|
||||
for path in ["/api/followers", "/api/following"] {
|
||||
let resp = c
|
||||
.get(format!("{base}{path}"))
|
||||
@@ -610,7 +630,7 @@ async fn followers_and_following_lists() {
|
||||
#[tokio::test]
|
||||
async fn thread_returns_parents_and_replies() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, _auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let a = did_for_test("root");
|
||||
@@ -689,7 +709,7 @@ async fn thread_returns_parents_and_replies() {
|
||||
#[tokio::test]
|
||||
async fn post_by_uri_stays_backwards_compatible() {
|
||||
let base = appview_url();
|
||||
let Some((c, _pool)) = ready().await else {
|
||||
let Some((c, _pool, _auth)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let a = did_for_test("compat_a");
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
//! End-to-end tests for the local PDS firehose consumer.
|
||||
//!
|
||||
//! These are the only tests that put *real* PDS bytes through
|
||||
//! [`appview::pds_firehose`]: everything else in the module's own
|
||||
//! `#[cfg(test)]` section builds frames from a hand-written DAG-CBOR
|
||||
//! encoder, which proves the decoder matches our reading of the
|
||||
//! contract but not that the PDS writes what we think it writes.
|
||||
//!
|
||||
//! Fail-open, like every other suite in this directory. Each test
|
||||
//! prints a notice and returns successfully when a precondition is
|
||||
//! missing:
|
||||
//!
|
||||
//! - the PDS isn't running on `:2583`;
|
||||
//! - `DATABASE_URL_APPVIEW` is unset or the database is unreachable;
|
||||
//! - **`com.atproto.sync.subscribeRepos` does not exist yet.** The
|
||||
//! endpoint is being built in `crates/pds-server` in parallel with
|
||||
//! this consumer. Until it lands, the WebSocket upgrade fails and
|
||||
//! these tests skip with a message saying so — they are not proof of
|
||||
//! anything while that line appears in the output.
|
||||
//!
|
||||
//! What they cover once the endpoint is live:
|
||||
//!
|
||||
//! - `frame_from_the_local_pds_indexes_a_post` — subscribe, create a
|
||||
//! record over `com.atproto.repo.createRecord`, and drive the frame
|
||||
//! the PDS emits through the real decode → CAR → indexer path,
|
||||
//! asserting the row lands in `posts`.
|
||||
//! - `replaying_the_same_frame_changes_nothing` — the same frame
|
||||
//! applied twice leaves exactly one row, which is what makes the
|
||||
//! overlap with the `/internal/ingest-commit` push safe.
|
||||
//! - `car_reader_parses_a_real_repo_export` — the AppView's CAR reader
|
||||
//! against a CAR the PDS's *writer* produced (`getRepo`).
|
||||
//! - `healthz_reports_the_firehose_state` — the running AppView's
|
||||
//! probe carries the new fields.
|
||||
|
||||
use futures::StreamExt;
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::PgPool;
|
||||
use std::time::Duration;
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
use appview::pds_firehose::{self, Frame};
|
||||
|
||||
const PDS_URL: &str = "http://127.0.0.1:2583";
|
||||
const PDS_WS: &str = "ws://127.0.0.1:2583";
|
||||
|
||||
fn appview_url() -> String {
|
||||
std::env::var("APPVIEW_TEST_URL").unwrap_or_else(|_| "http://127.0.0.1:2584".to_string())
|
||||
}
|
||||
|
||||
fn client() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn service_up(c: &reqwest::Client, base: &str) -> bool {
|
||||
for _ in 0..12 {
|
||||
if let Ok(r) = c.get(format!("{base}/healthz")).send().await {
|
||||
if r.status().is_success() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn appview_db() -> Option<PgPool> {
|
||||
let url = std::env::var("DATABASE_URL_APPVIEW").ok()?;
|
||||
match tokio::time::timeout(Duration::from_secs(2), PgPool::connect(&url)).await {
|
||||
Ok(Ok(pool)) => Some(pool),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
struct Account {
|
||||
did: String,
|
||||
access_jwt: String,
|
||||
}
|
||||
|
||||
async fn create_account(c: &reqwest::Client) -> Option<Account> {
|
||||
let handle = format!("fh_{}.maarcadetweet.local", uuid::Uuid::new_v4().simple());
|
||||
let r: Value = c
|
||||
.post(format!("{PDS_URL}/xrpc/com.atproto.server.createAccount"))
|
||||
.json(&json!({ "handle": handle, "password": "hunter2hunter2" }))
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.json()
|
||||
.await
|
||||
.ok()?;
|
||||
Some(Account {
|
||||
did: r["did"].as_str()?.to_string(),
|
||||
access_jwt: r["access_jwt"].as_str()?.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn create_post(c: &reqwest::Client, acc: &Account, text: &str) -> Option<String> {
|
||||
let r: Value = c
|
||||
.post(format!("{PDS_URL}/xrpc/com.atproto.repo.createRecord"))
|
||||
.bearer_auth(&acc.access_jwt)
|
||||
.json(&json!({
|
||||
"repo": acc.did,
|
||||
"collection": "app.twi.post",
|
||||
"record": { "text": text, "createdAt": "2026-09-10T12:00:00Z" },
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.ok()?
|
||||
.json()
|
||||
.await
|
||||
.ok()?;
|
||||
r["uri"].as_str().map(str::to_string)
|
||||
}
|
||||
|
||||
type Ws = tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>;
|
||||
|
||||
/// Subscribe to the PDS firehose, or `None` if the endpoint isn't there
|
||||
/// yet (see the module docs).
|
||||
async fn subscribe() -> Option<Ws> {
|
||||
let url = pds_firehose::subscribe_url(PDS_WS, None);
|
||||
match tokio::time::timeout(Duration::from_secs(5), tokio_tungstenite::connect_async(&url)).await
|
||||
{
|
||||
Ok(Ok((ws, _))) => Some(ws),
|
||||
Ok(Err(e)) => {
|
||||
eprintln!(
|
||||
"cannot subscribe to {url}: {e} — the PDS endpoint com.atproto.sync.\
|
||||
subscribeRepos is probably not implemented yet; skipping"
|
||||
);
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("timed out connecting to {url}; skipping");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read frames until one is a `#commit` for `did`, or the deadline
|
||||
/// passes. `#info` frames along the way are tolerated (a fresh
|
||||
/// subscription may legitimately be told its cursor is outdated).
|
||||
async fn next_commit_for(
|
||||
ws: &mut Ws,
|
||||
did: &str,
|
||||
timeout: Duration,
|
||||
) -> Option<pds_firehose::CommitFrame> {
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if remaining.is_zero() {
|
||||
return None;
|
||||
}
|
||||
let msg = match tokio::time::timeout(remaining, ws.next()).await {
|
||||
Ok(Some(Ok(m))) => m,
|
||||
Ok(Some(Err(e))) => {
|
||||
eprintln!("firehose read error: {e}");
|
||||
return None;
|
||||
}
|
||||
Ok(None) | Err(_) => return None,
|
||||
};
|
||||
let Message::Binary(bytes) = msg else { continue };
|
||||
match pds_firehose::decode_frame(&bytes) {
|
||||
Ok(Frame::Commit(commit)) if commit.repo == did => return Some(*commit),
|
||||
Ok(Frame::Commit(_)) => continue,
|
||||
Ok(Frame::Info { name, .. }) => {
|
||||
eprintln!("firehose #info: {name}");
|
||||
continue;
|
||||
}
|
||||
Ok(Frame::Error { error, message }) => {
|
||||
eprintln!("firehose error frame: {error} {message:?}");
|
||||
return None;
|
||||
}
|
||||
Ok(Frame::Other { .. }) => continue,
|
||||
Err(e) => {
|
||||
// A frame we cannot decode is a contract failure worth
|
||||
// failing the test over — but only once we know the
|
||||
// endpoint exists, which we do by this point.
|
||||
panic!("could not decode a real PDS firehose frame: {e:#}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Everything a live test needs, or `None` with a printed reason.
|
||||
async fn ready() -> Option<(reqwest::Client, PgPool, Account, Ws)> {
|
||||
let c = client();
|
||||
if !service_up(&c, PDS_URL).await {
|
||||
eprintln!("pds not running on {PDS_URL}, skipping");
|
||||
return None;
|
||||
}
|
||||
let Some(db) = appview_db().await else {
|
||||
eprintln!("DATABASE_URL_APPVIEW unset or unreachable, skipping");
|
||||
return None;
|
||||
};
|
||||
let Some(acc) = create_account(&c).await else {
|
||||
eprintln!("could not create a PDS account, skipping");
|
||||
return None;
|
||||
};
|
||||
// Subscribe *before* writing anything, so the commit we are about
|
||||
// to make is guaranteed to fall inside the subscription window.
|
||||
let ws = subscribe().await?;
|
||||
Some((c, db, acc, ws))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn frame_from_the_local_pds_indexes_a_post() {
|
||||
let Some((c, db, acc, mut ws)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let text = format!("firehose e2e {}", uuid::Uuid::new_v4().simple());
|
||||
let Some(uri) = create_post(&c, &acc, &text).await else {
|
||||
eprintln!("createRecord failed, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let Some(commit) = next_commit_for(&mut ws, &acc.did, Duration::from_secs(15)).await else {
|
||||
eprintln!("no #commit frame for {} arrived in time, skipping", acc.did);
|
||||
return;
|
||||
};
|
||||
|
||||
// The frame itself must carry what the contract promises.
|
||||
assert!(commit.seq > 0, "seq must be a positive sequence number");
|
||||
assert_eq!(commit.repo, acc.did);
|
||||
assert!(!commit.rev.is_empty(), "commit frames carry a rev");
|
||||
assert!(
|
||||
!commit.blocks.is_empty(),
|
||||
"a create commit must inline its record block"
|
||||
);
|
||||
let create = commit
|
||||
.ops
|
||||
.iter()
|
||||
.find(|op| op.action == "create" && op.collection() == Some("app.twi.post"))
|
||||
.expect("the frame must contain the post create op");
|
||||
assert!(create.cid.is_some(), "a create op carries the record CID");
|
||||
|
||||
// The blocks field must be a CAR our reader understands, and the
|
||||
// op's CID must resolve inside it.
|
||||
let car = appview::car::parse(&commit.blocks).expect("blocks must be a readable CAR v1");
|
||||
assert!(
|
||||
car.block_map().contains_key(&create.cid.unwrap()),
|
||||
"the record block must be present in the CAR"
|
||||
);
|
||||
|
||||
// And the whole path — frame → CAR → indexer — must land the row.
|
||||
let events = pds_firehose::events_from_frame(&commit).expect("events");
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|e| e.commit.as_ref().unwrap()["record"]["text"] == json!(text)),
|
||||
"the decoded record must carry the text we posted"
|
||||
);
|
||||
|
||||
// Remove whatever the AppView's own push path already wrote, so the
|
||||
// assertion below is about *this* code applying *this* frame.
|
||||
sqlx::query("DELETE FROM posts WHERE uri = $1")
|
||||
.bind(&uri)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
pds_firehose::apply_frame(&db, &commit)
|
||||
.await
|
||||
.expect("apply_frame");
|
||||
|
||||
let stored: Option<String> = sqlx::query_scalar("SELECT text FROM posts WHERE uri = $1")
|
||||
.bind(&uri)
|
||||
.fetch_optional(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
stored.as_deref(),
|
||||
Some(text.as_str()),
|
||||
"the firehose frame must index the post at {uri}"
|
||||
);
|
||||
|
||||
sqlx::query("DELETE FROM posts WHERE uri = $1")
|
||||
.bind(&uri)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replaying_the_same_frame_changes_nothing() {
|
||||
let Some((c, db, acc, mut ws)) = ready().await else {
|
||||
return;
|
||||
};
|
||||
let text = format!("firehose replay {}", uuid::Uuid::new_v4().simple());
|
||||
let Some(uri) = create_post(&c, &acc, &text).await else {
|
||||
eprintln!("createRecord failed, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(commit) = next_commit_for(&mut ws, &acc.did, Duration::from_secs(15)).await else {
|
||||
eprintln!("no #commit frame for {} arrived in time, skipping", acc.did);
|
||||
return;
|
||||
};
|
||||
|
||||
// Three applications: the push already ran, then the firehose, then
|
||||
// a post-restart replay of the same seq.
|
||||
for _ in 0..3 {
|
||||
pds_firehose::apply_frame(&db, &commit)
|
||||
.await
|
||||
.expect("apply_frame");
|
||||
}
|
||||
|
||||
let rows: i64 = sqlx::query_scalar("SELECT count(*) FROM posts WHERE uri = $1")
|
||||
.bind(&uri)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(rows, 1, "replay must not duplicate {uri}");
|
||||
|
||||
sqlx::query("DELETE FROM posts WHERE uri = $1")
|
||||
.bind(&uri)
|
||||
.execute(&db)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn car_reader_parses_a_real_repo_export() {
|
||||
// This one needs no firehose: `getRepo` has always served a CAR
|
||||
// produced by the PDS's own writer, which is exactly the encoder
|
||||
// the firehose's `blocks` field reuses.
|
||||
let c = client();
|
||||
if !service_up(&c, PDS_URL).await {
|
||||
eprintln!("pds not running on {PDS_URL}, skipping");
|
||||
return;
|
||||
}
|
||||
let Some(acc) = create_account(&c).await else {
|
||||
eprintln!("could not create a PDS account, skipping");
|
||||
return;
|
||||
};
|
||||
if create_post(&c, &acc, "car reader fixture").await.is_none() {
|
||||
eprintln!("createRecord failed, skipping");
|
||||
return;
|
||||
}
|
||||
|
||||
let resp = c
|
||||
.get(format!("{PDS_URL}/xrpc/com.atproto.sync.getRepo"))
|
||||
.query(&[("did", acc.did.as_str())])
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
if !resp.status().is_success() {
|
||||
eprintln!("getRepo returned {}, skipping", resp.status());
|
||||
return;
|
||||
}
|
||||
let bytes = resp.bytes().await.unwrap();
|
||||
let car = appview::car::parse(&bytes).expect("getRepo must return a readable CAR v1");
|
||||
assert_eq!(car.header.version, 1);
|
||||
assert!(
|
||||
!car.blocks.is_empty(),
|
||||
"a repo with one record has blocks (commit + MST + record)"
|
||||
);
|
||||
// Every block must hash to the CID the file declares — the strongest
|
||||
// available statement that the reader's section framing is right.
|
||||
appview::car::verify_block_cids(&car).expect("block CIDs must verify");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn healthz_reports_the_firehose_state() {
|
||||
let c = client();
|
||||
if !service_up(&c, &appview_url()).await {
|
||||
eprintln!("appview not running, skipping");
|
||||
return;
|
||||
}
|
||||
let body: Value = c
|
||||
.get(format!("{}/healthz", appview_url()))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A binary built before this feature has none of these keys; say so
|
||||
// rather than failing, because "restart the AppView" is the fix.
|
||||
let Some(enabled) = body.get("pds_firehose_enabled").and_then(Value::as_bool) else {
|
||||
eprintln!(
|
||||
"the running AppView predates the PDS firehose (no pds_firehose_enabled \
|
||||
in /healthz) — rebuild and restart it; skipping"
|
||||
);
|
||||
return;
|
||||
};
|
||||
assert!(
|
||||
body.get("pds_firehose_connected")
|
||||
.and_then(Value::as_bool)
|
||||
.is_some(),
|
||||
"/healthz must report pds_firehose_connected: {body}"
|
||||
);
|
||||
assert!(
|
||||
body.get("pds_firehose_seq").and_then(Value::as_i64).is_some(),
|
||||
"/healthz must report pds_firehose_seq: {body}"
|
||||
);
|
||||
|
||||
if !enabled {
|
||||
eprintln!("PDS_FIREHOSE_ENABLED=false on the running AppView; nothing more to check");
|
||||
return;
|
||||
}
|
||||
if !body["pds_firehose_connected"].as_bool().unwrap() {
|
||||
eprintln!(
|
||||
"the AppView is not connected to the PDS firehose — expected while \
|
||||
com.atproto.sync.subscribeRepos is still being implemented; skipping \
|
||||
the live-consumption check"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Connected: a new record must move the sequence number the AppView
|
||||
// reports, which is the end-to-end proof that the *service* (not
|
||||
// just this test process) consumes the stream.
|
||||
if !service_up(&c, PDS_URL).await {
|
||||
eprintln!("pds not running, skipping the live-consumption check");
|
||||
return;
|
||||
}
|
||||
let Some(acc) = create_account(&c).await else {
|
||||
eprintln!("could not create a PDS account, skipping");
|
||||
return;
|
||||
};
|
||||
let before = body["pds_firehose_seq"].as_i64().unwrap_or(0);
|
||||
if create_post(&c, &acc, "healthz seq probe").await.is_none() {
|
||||
eprintln!("createRecord failed, skipping");
|
||||
return;
|
||||
}
|
||||
for _ in 0..40 {
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
let now: Value = c
|
||||
.get(format!("{}/healthz", appview_url()))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
if now["pds_firehose_seq"].as_i64().unwrap_or(0) > before {
|
||||
return; // consumed
|
||||
}
|
||||
}
|
||||
panic!("the AppView reports the firehose connected but its seq never advanced");
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use serde_json::json;
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
@@ -33,6 +32,43 @@ impl JetstreamConsumer {
|
||||
}
|
||||
}
|
||||
|
||||
/// The URL actually dialled: base URL plus `wantedCollections` and
|
||||
/// `cursor` as query parameters.
|
||||
///
|
||||
/// This used to connect to the bare URL and then send
|
||||
/// `{"type": "options", "wantedCollections": [...]}` as a text frame.
|
||||
/// Jetstream ignores that, and silently: filters are query parameters,
|
||||
/// and the only message-based path (`options_update`) requires the
|
||||
/// connection to have been opened with `requireHello=true`. So every
|
||||
/// deployment that thought it was subscribing to six collections was in
|
||||
/// fact taking the entire public firehose — measured against
|
||||
/// jetstream1.us-east: 3119 events in 8 s unfiltered versus 520 for a
|
||||
/// single collection. On the dev database that quietly grew to 3.3 M
|
||||
/// posts; on the production instance the AppView had to be switched off
|
||||
/// to stop it filling the disk.
|
||||
///
|
||||
/// Note what fixing this does *not* solve: the collections this project
|
||||
/// wants (`app.bsky.feed.post` / `like` / `repost` / `graph.follow`) are
|
||||
/// ~97 % of the firehose by volume. Correct filtering is necessary, not
|
||||
/// sufficient — an instance that does not want the whole public network
|
||||
/// in its index wants `wantedDids`, or no Jetstream at all.
|
||||
pub fn subscribe_url(&self) -> String {
|
||||
let mut url = self.url.trim_end_matches('&').to_string();
|
||||
let mut sep = if url.contains('?') { '&' } else { '?' };
|
||||
for c in &self.collections {
|
||||
url.push(sep);
|
||||
url.push_str("wantedCollections=");
|
||||
url.push_str(&urlencode(c));
|
||||
sep = '&';
|
||||
}
|
||||
if self.cursor_us > 0 {
|
||||
url.push(sep);
|
||||
url.push_str("cursor=");
|
||||
url.push_str(&self.cursor_us.to_string());
|
||||
}
|
||||
url
|
||||
}
|
||||
|
||||
/// Build a consumer that shares a connection-state flag with the caller.
|
||||
pub fn with_connected_flag(mut self, flag: Arc<AtomicBool>) -> Self {
|
||||
self.connected = Some(flag);
|
||||
@@ -81,23 +117,13 @@ impl JetstreamConsumer {
|
||||
F: FnMut(JetstreamEvent) -> Fut + Send,
|
||||
Fut: std::future::Future<Output = Result<()>> + Send,
|
||||
{
|
||||
let (mut ws, _) = tokio_tungstenite::connect_async(&self.url).await?;
|
||||
info!("connected to jetstream: {}", self.url);
|
||||
let url = self.subscribe_url();
|
||||
let (mut ws, _) = tokio_tungstenite::connect_async(&url).await?;
|
||||
info!("connected to jetstream: {url}");
|
||||
if let Some(flag) = &self.connected {
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
if !self.collections.is_empty() || self.cursor_us > 0 {
|
||||
let mut options = json!({ "type": "options" });
|
||||
if !self.collections.is_empty() {
|
||||
options["wantedCollections"] = json!(self.collections);
|
||||
}
|
||||
if self.cursor_us > 0 {
|
||||
options["cursor"] = json!(self.cursor_us);
|
||||
}
|
||||
ws.send(Message::Text(options.to_string())).await?;
|
||||
}
|
||||
|
||||
while let Some(msg) = ws.next().await {
|
||||
let msg = msg?;
|
||||
if let Message::Text(text) = msg {
|
||||
@@ -109,3 +135,61 @@ impl JetstreamConsumer {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Percent-encode everything outside the unreserved set. Collection NSIDs are
|
||||
/// dots and letters today, but a `wantedDids` value carries `:` — encoding
|
||||
/// unconditionally keeps this correct if the caller passes one.
|
||||
fn urlencode(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| match c {
|
||||
'A'..='Z' | 'a'..='z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
|
||||
other => {
|
||||
let mut buf = [0u8; 4];
|
||||
other
|
||||
.encode_utf8(&mut buf)
|
||||
.bytes()
|
||||
.map(|b| format!("%{b:02X}"))
|
||||
.collect()
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod url_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collections_go_into_the_query_string() {
|
||||
let c = JetstreamConsumer::new(
|
||||
"wss://jetstream1.us-east.bsky.network/subscribe",
|
||||
vec!["app.twi.post".into(), "app.bsky.feed.like".into()],
|
||||
);
|
||||
assert_eq!(
|
||||
c.subscribe_url(),
|
||||
"wss://jetstream1.us-east.bsky.network/subscribe\
|
||||
?wantedCollections=app.twi.post&wantedCollections=app.bsky.feed.like"
|
||||
.replace(' ', "")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_is_appended_and_respects_an_existing_query() {
|
||||
let mut c = JetstreamConsumer::new("wss://host/subscribe?compress=false", vec![]);
|
||||
c.cursor_us = 1234;
|
||||
assert_eq!(c.subscribe_url(), "wss://host/subscribe?compress=false&cursor=1234");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_filters_leaves_the_url_alone() {
|
||||
let c = JetstreamConsumer::new("wss://host/subscribe", vec![]);
|
||||
assert_eq!(c.subscribe_url(), "wss://host/subscribe");
|
||||
}
|
||||
|
||||
/// A DID contains `:`, which has to survive as `%3A` in a query value.
|
||||
#[test]
|
||||
fn values_are_percent_encoded() {
|
||||
assert_eq!(urlencode("did:plc:abc"), "did%3Aplc%3Aabc");
|
||||
assert_eq!(urlencode("app.bsky.feed.post"), "app.bsky.feed.post");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,62 @@ fn default_handle_sync_interval() -> u64 {
|
||||
300
|
||||
}
|
||||
|
||||
/// Default for `APPVIEW_AUTH_REQUIRED`.
|
||||
///
|
||||
/// `true` — the AppView's private endpoints (notifications, home
|
||||
/// timeline) reject unauthenticated requests. Fail closed: an operator
|
||||
/// who forgets the variable gets the safe behaviour, and the only way
|
||||
/// to serve another user's notifications to an anonymous caller is to
|
||||
/// opt out explicitly.
|
||||
fn default_auth_required() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Default for `PDS_FIREHOSE_ENABLED`.
|
||||
///
|
||||
/// `true` — the AppView consumes the local PDS's
|
||||
/// `com.atproto.sync.subscribeRepos` stream. That stream is the only
|
||||
/// *guaranteed* path for a local user's own records: the fast
|
||||
/// `POST /internal/ingest-commit` push is best effort, and the public
|
||||
/// Jetstream never sees this PDS, so a lost push means a permanently
|
||||
/// missing post. Defaulting to on means an operator who never heard of
|
||||
/// the variable gets the durable behaviour; switching it off is the
|
||||
/// explicit choice (e.g. a PDS too old to serve the endpoint, or a
|
||||
/// second AppView instance that should not double-index).
|
||||
fn default_pds_firehose_enabled() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Default for `PDS_INVITE_REQUIRED`.
|
||||
///
|
||||
/// `false` — `com.atproto.server.createAccount` stays open unless the
|
||||
/// operator says otherwise. This is the one security switch in this file
|
||||
/// that fails *open*, and it does so for a concrete reason: dozens of
|
||||
/// integration tests across `pds-server` and `appview` create throwaway
|
||||
/// accounts against a locally running PDS, and every dev instance is
|
||||
/// bootstrapped the same way. Defaulting to `true` would break all of
|
||||
/// them on the next `cargo test`, and the usual reflex to a suite that
|
||||
/// suddenly fails is to switch the new thing off — which lands you at
|
||||
/// `false` anyway, only with the flag now looking like the thing that
|
||||
/// was in the way rather than the thing that protects the server.
|
||||
///
|
||||
/// The cost of that choice is that an operator who exposes the PDS
|
||||
/// publicly without setting the variable gets an open registration
|
||||
/// endpoint. That is paid for at startup: `pds-server` logs a loud
|
||||
/// warning on every boot where this is `false`, in the same spirit as
|
||||
/// `appview`'s `log_startup_posture`. A warning you have to read once
|
||||
/// per restart is the trade for a test suite that keeps working.
|
||||
///
|
||||
/// That warning also covers the other way this fails open:
|
||||
/// [`parse_bool_env`] reads anything it doesn't recognise as `false`, so
|
||||
/// `PDS_INVITE_REQUIRED=ture` leaves registration open. The operator
|
||||
/// who typed it sees the same startup warning as the operator who never
|
||||
/// set the variable at all, which is the only signal that distinguishes
|
||||
/// "I meant to leave it open" from "I thought I had closed it".
|
||||
fn default_pds_invite_required() -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct AppConfig {
|
||||
pub pds_host: String,
|
||||
@@ -46,6 +102,51 @@ pub struct AppConfig {
|
||||
/// directory. Default: 300s (5 minutes).
|
||||
#[serde(default = "default_handle_sync_interval")]
|
||||
pub appview_handle_sync_interval_secs: u64,
|
||||
/// Whether the AppView enforces bearer-token auth on the endpoints
|
||||
/// that serve a single user's private data (`/api/notifications*`,
|
||||
/// `/api/timeline/home`). Default `true`.
|
||||
///
|
||||
/// Set `APPVIEW_AUTH_REQUIRED=false` to get the pre-auth behaviour
|
||||
/// (every endpoint public). That mode exists for two callers:
|
||||
/// the fail-open integration suites, which seed synthetic DIDs the
|
||||
/// PDS has never issued a token for, and an instance that is
|
||||
/// already isolated at the network layer (VPN / private subnet).
|
||||
/// The AppView warns loudly at startup when it is off.
|
||||
#[serde(default = "default_auth_required")]
|
||||
pub appview_auth_required: bool,
|
||||
/// Browser origins allowed to call the AppView's `/api/*` routes,
|
||||
/// from the comma-separated `APPVIEW_CORS_ORIGINS`. Empty means
|
||||
/// "no allowlist configured" — the AppView then keeps the historic
|
||||
/// `Access-Control-Allow-Origin: *` behaviour and warns at startup.
|
||||
///
|
||||
/// Example (Tauri webview origins differ per platform):
|
||||
/// `APPVIEW_CORS_ORIGINS=tauri://localhost,http://127.0.0.1:1430`
|
||||
#[serde(default)]
|
||||
pub appview_cors_origins: Vec<String>,
|
||||
/// Whether the AppView subscribes to the local PDS firehose
|
||||
/// (`com.atproto.sync.subscribeRepos` on
|
||||
/// [`AppConfig::pds_base_url`]). Default `true` — see
|
||||
/// [`default_pds_firehose_enabled`] for why.
|
||||
#[serde(default = "default_pds_firehose_enabled")]
|
||||
pub pds_firehose_enabled: bool,
|
||||
/// Whether `com.atproto.server.createAccount` demands a valid invite
|
||||
/// code. Default `false` — see [`default_pds_invite_required`] for
|
||||
/// why this switch, alone among the security switches here, fails
|
||||
/// open.
|
||||
///
|
||||
/// When `true`, a request without an `invite_code` (or its
|
||||
/// camelCase `inviteCode` spelling), or with one that is unknown,
|
||||
/// disabled or already used up, is rejected with
|
||||
/// `400 InvalidInviteCode`. The value is also what
|
||||
/// `describeServer` reports as `invite_code_required`, so a client
|
||||
/// can find out before it asks the user for a handle.
|
||||
///
|
||||
/// Codes are minted out of band with `pds-server invite create`;
|
||||
/// there is no HTTP endpoint that creates them, on purpose — an
|
||||
/// open PDS's registration gate should not come with a second
|
||||
/// public surface that hands out keys to it.
|
||||
#[serde(default = "default_pds_invite_required")]
|
||||
pub pds_invite_required: bool,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
@@ -82,6 +183,204 @@ impl AppConfig {
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or_else(default_handle_sync_interval),
|
||||
appview_auth_required: std::env::var("APPVIEW_AUTH_REQUIRED")
|
||||
.ok()
|
||||
.map(|s| parse_bool_env(&s))
|
||||
.unwrap_or_else(default_auth_required),
|
||||
appview_cors_origins: std::env::var("APPVIEW_CORS_ORIGINS")
|
||||
.ok()
|
||||
.map(|s| parse_csv_env(&s))
|
||||
.unwrap_or_default(),
|
||||
pds_firehose_enabled: std::env::var("PDS_FIREHOSE_ENABLED")
|
||||
.ok()
|
||||
.map(|s| parse_bool_env(&s))
|
||||
.unwrap_or_else(default_pds_firehose_enabled),
|
||||
pds_invite_required: std::env::var("PDS_INVITE_REQUIRED")
|
||||
.ok()
|
||||
.map(|s| parse_bool_env(&s))
|
||||
.unwrap_or_else(default_pds_invite_required),
|
||||
})
|
||||
}
|
||||
|
||||
/// The `did:web:` DID of *this* PDS, derived from `PDS_PUBLIC_URL`.
|
||||
///
|
||||
/// One derivation, two consumers: `com.atproto.server.describeServer`
|
||||
/// (which used to return a hardcoded `did:web:pds.maarcadetweet.local`
|
||||
/// no matter what the operator configured) and
|
||||
/// `GET /.well-known/did.json`, which publishes the server's signing
|
||||
/// key under exactly this id. If those two ever disagreed, a client
|
||||
/// that trusts `describeServer` would fetch the key document of a
|
||||
/// different identity.
|
||||
pub fn pds_did(&self) -> String {
|
||||
did_web_from_url(&self.pds_public_url)
|
||||
}
|
||||
|
||||
/// The AppView's own service DID, derived from `APPVIEW_PUBLIC_URL`.
|
||||
///
|
||||
/// Also one derivation, two consumers: the PDS stamps it into the
|
||||
/// `aud` of every access token it issues, and the AppView checks
|
||||
/// incoming tokens against it. A token minted for a *different*
|
||||
/// AppView must not be usable here — that's the whole point of an
|
||||
/// audience — so both sides have to agree on the spelling, and the
|
||||
/// only way to guarantee that is to compute it the same way from
|
||||
/// the same configuration.
|
||||
pub fn appview_did(&self) -> String {
|
||||
did_web_from_url(&self.appview_public_url)
|
||||
}
|
||||
|
||||
/// Base URL the AppView uses to reach the PDS.
|
||||
///
|
||||
/// `PDS_INTERNAL_URL` when set (the cluster-internal hostname),
|
||||
/// otherwise `PDS_PUBLIC_URL`. Both the handle-sync resolver and the
|
||||
/// signing-key fetch go through here, so the two can't end up
|
||||
/// talking to different PDS instances.
|
||||
pub fn pds_base_url(&self) -> String {
|
||||
self.pds_internal_url
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.pds_public_url.clone())
|
||||
}
|
||||
}
|
||||
|
||||
/// Interpret an environment variable as a boolean.
|
||||
///
|
||||
/// Accepts the spellings people actually type in a `.env` file. Anything
|
||||
/// unrecognised counts as `false` for an explicitly-set variable — the
|
||||
/// caller decides what an *absent* variable means (see
|
||||
/// [`default_auth_required`]), and a typo like `APPVIEW_AUTH_REQUIRED=ture`
|
||||
/// must never silently read as "on" when the operator's intent was to
|
||||
/// switch something off... nor as "off" for a security switch. Since
|
||||
/// this is only reached when the variable *is* set, and the only
|
||||
/// security-relevant user of it defaults to `true` when unset, we treat
|
||||
/// unknown values as `false` and rely on the startup warning to make a
|
||||
/// disabled auth switch impossible to miss in the logs.
|
||||
fn parse_bool_env(raw: &str) -> bool {
|
||||
matches!(
|
||||
raw.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
)
|
||||
}
|
||||
|
||||
/// Split a comma-separated environment variable into trimmed,
|
||||
/// non-empty entries. `"a, b,,c "` → `["a", "b", "c"]`.
|
||||
fn parse_csv_env(raw: &str) -> Vec<String> {
|
||||
raw.split(',')
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Turn an `http(s)://host[:port][/path]` URL into a `did:web:` DID.
|
||||
///
|
||||
/// The did:web method spec maps the authority to the method-specific
|
||||
/// id, with two wrinkles that matter here:
|
||||
///
|
||||
/// - a port is **percent-encoded** (`:` → `%3A`), because a bare colon
|
||||
/// already separates the DID's own segments. `http://127.0.0.1:2583`
|
||||
/// is therefore `did:web:127.0.0.1%3A2583`, *not*
|
||||
/// `did:web:127.0.0.1:2583` (which would parse as host `127.0.0.1`
|
||||
/// plus a path segment `2583`).
|
||||
/// - path segments, if any, are appended separated by `:`.
|
||||
///
|
||||
/// The default ports (80/443) are kept rather than stripped: the
|
||||
/// resolution rule is a textual one, and a client that reverses this
|
||||
/// mapping has to end up at the same URL we serve the document from.
|
||||
pub fn did_web_from_url(url: &str) -> String {
|
||||
// Strip the scheme. We accept a bare `host:port` too, which is what
|
||||
// a misconfigured `PDS_PUBLIC_URL` often contains.
|
||||
let rest = url
|
||||
.trim()
|
||||
.trim_start_matches("https://")
|
||||
.trim_start_matches("http://")
|
||||
.trim_end_matches('/');
|
||||
// Drop any userinfo (`user@host`) and query/fragment — neither has
|
||||
// a place in a did:web identifier.
|
||||
let rest = rest.split(['?', '#']).next().unwrap_or(rest);
|
||||
let rest = rest.rsplit('@').next().unwrap_or(rest);
|
||||
|
||||
let mut parts = rest.split('/');
|
||||
let authority = parts.next().unwrap_or("");
|
||||
let host = authority.replacen(':', "%3A", 1);
|
||||
let mut did = format!("did:web:{host}");
|
||||
for segment in parts.filter(|s| !s.is_empty()) {
|
||||
did.push(':');
|
||||
did.push_str(segment);
|
||||
}
|
||||
did
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn did_web_encodes_port_as_percent_3a() {
|
||||
// The dev default. A literal colon here would be read as a
|
||||
// did:web path segment, so it has to be percent-encoded.
|
||||
assert_eq!(
|
||||
did_web_from_url("http://127.0.0.1:2583"),
|
||||
"did:web:127.0.0.1%3A2583"
|
||||
);
|
||||
assert_eq!(
|
||||
did_web_from_url("https://pds.example.com:8443"),
|
||||
"did:web:pds.example.com%3A8443"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn did_web_without_port_is_plain_host() {
|
||||
assert_eq!(
|
||||
did_web_from_url("https://pds.maarcadetweet.local"),
|
||||
"did:web:pds.maarcadetweet.local"
|
||||
);
|
||||
// Trailing slash must not produce an empty path segment.
|
||||
assert_eq!(
|
||||
did_web_from_url("https://pds.example.com/"),
|
||||
"did:web:pds.example.com"
|
||||
);
|
||||
// Scheme-less input is tolerated.
|
||||
assert_eq!(did_web_from_url("pds.example.com"), "did:web:pds.example.com");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn did_web_appends_path_segments_with_colons() {
|
||||
assert_eq!(
|
||||
did_web_from_url("https://example.com/user/alice"),
|
||||
"did:web:example.com:user:alice"
|
||||
);
|
||||
// Port + path together: only the port gets percent-encoded.
|
||||
assert_eq!(
|
||||
did_web_from_url("http://example.com:2583/pds"),
|
||||
"did:web:example.com%3A2583:pds"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn did_web_ignores_userinfo_query_and_fragment() {
|
||||
assert_eq!(
|
||||
did_web_from_url("https://user@example.com?x=1#frag"),
|
||||
"did:web:example.com"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bool_env_accepts_common_spellings() {
|
||||
for on in ["1", "true", "TRUE", " yes ", "on"] {
|
||||
assert!(parse_bool_env(on), "{on} should parse as true");
|
||||
}
|
||||
for off in ["0", "false", "no", "off", "", "nonsense"] {
|
||||
assert!(!parse_bool_env(off), "{off} should parse as false");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_env_trims_and_drops_empties() {
|
||||
assert_eq!(
|
||||
parse_csv_env("tauri://localhost, http://127.0.0.1:1430 ,,"),
|
||||
vec![
|
||||
"tauri://localhost".to_string(),
|
||||
"http://127.0.0.1:1430".to_string()
|
||||
]
|
||||
);
|
||||
assert!(parse_csv_env(" ").is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ path = "src/main.rs"
|
||||
[dependencies]
|
||||
tokio = { workspace = true }
|
||||
dotenvy = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
axum = { workspace = true, features = ["ws"] }
|
||||
tower = { workspace = true }
|
||||
tower-http = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
@@ -39,6 +39,7 @@ hex = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
cid = { workspace = true }
|
||||
k256 = { workspace = true }
|
||||
p256 = { workspace = true }
|
||||
@@ -56,3 +57,6 @@ sha2 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
sqlx = { workspace = true }
|
||||
at-crypto = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
tokio-tungstenite = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
+131
-157
@@ -17,79 +17,51 @@
|
||||
//! The header is `{ version: 1, roots: [CID, ...] }` encoded as DAG-CBOR. In
|
||||
//! DAG-CBOR CID links carry the IANA-registered CBOR tag `42`, which the
|
||||
//! `ciborium` crate does not emit for `cid::Cid` (it uses serde newtype-struct
|
||||
//! tagging instead). We hand-encode the header bytes to keep the file
|
||||
//! spec-compliant: a `Map(2)` with text keys `"version"` and `"roots"`, an
|
||||
//! unsigned int `1` for the version, and a tagged byte string for each root
|
||||
//! CID.
|
||||
//! tagging instead). We hand-encode the header bytes through the shared
|
||||
//! primitives in [`crate::dag_cbor`]: a `Map(2)` with text keys `"version"`
|
||||
//! and `"roots"`, an unsigned int `1` for the version, and a tagged byte
|
||||
//! string for each root CID.
|
||||
//!
|
||||
//! One documented deviation from the DAG-CBOR spec lives in
|
||||
//! [`encode_header`] — the root CIDs are tagged but not identity-prefixed.
|
||||
//! See the note there; the firehose frames in [`crate::firehose`] do it the
|
||||
//! spec-correct way via [`crate::dag_cbor::write_link`].
|
||||
//!
|
||||
//! Per the spec, CAR v1 stores the raw CID bytes (varint version + codec +
|
||||
//! multihash) prefixed to every block, with a leading varint giving the total
|
||||
//! length of the section (CID + block).
|
||||
|
||||
use crate::dag_cbor::{read_head, write_bytes, write_head, write_text};
|
||||
use anyhow::Result;
|
||||
use cid::Cid;
|
||||
|
||||
/// Encode an unsigned CBOR head (major type in upper 3 bits) with a value.
|
||||
///
|
||||
/// Supports values up to `u32::MAX` which is more than enough for any realistic
|
||||
/// header or array length.
|
||||
fn cbor_head(out: &mut Vec<u8>, major: u8, n: u64) {
|
||||
let m = (major & 0x07) << 5;
|
||||
if n < 24 {
|
||||
out.push(m | n as u8);
|
||||
} else if n < 0x100 {
|
||||
out.push(m | 24);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x10000 {
|
||||
out.push(m | 25);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x100_0000 {
|
||||
out.push(m | 26);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else {
|
||||
out.push(m | 27);
|
||||
out.push((n >> 24) as u8);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a CBOR text string.
|
||||
fn cbor_text(out: &mut Vec<u8>, s: &str) {
|
||||
cbor_head(out, 3, s.len() as u64);
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
/// Append a CBOR byte string.
|
||||
fn cbor_bytes(out: &mut Vec<u8>, b: &[u8]) {
|
||||
cbor_head(out, 2, b.len() as u64);
|
||||
out.extend_from_slice(b);
|
||||
}
|
||||
|
||||
/// Append a CBOR tag wrapping the following value.
|
||||
fn cbor_tag(out: &mut Vec<u8>, tag: u64) {
|
||||
cbor_head(out, 6, tag);
|
||||
}
|
||||
|
||||
/// Encode the CAR v1 DAG-CBOR header `{ version: 1, roots: [CID, ...] }`.
|
||||
///
|
||||
/// CIDs are encoded as `tag(42) + bytes(<raw-cid-bytes>)` per the DAG-CBOR
|
||||
/// spec. This is the canonical IPLD CID-link form.
|
||||
/// ### Known deviation: the root CIDs carry no identity prefix
|
||||
///
|
||||
/// A spec-conformant DAG-CBOR CID link is `tag(42)` wrapping a byte string of
|
||||
/// `0x00 || <binary CID>` — the `0x00` being the multibase identity prefix.
|
||||
///
|
||||
/// Older builds of this server omitted that byte and tagged the bare CID,
|
||||
/// which no conformant CAR reader can follow: it reads the first byte as the
|
||||
/// CID version and gives up. Since the header is not content-addressed —
|
||||
/// nothing hashes it, and no CID anywhere depends on its bytes — fixing it
|
||||
/// changes only what goes out on the wire, never an identifier. So it is
|
||||
/// fixed, via [`crate::dag_cbor::write_link`], the same writer the firehose
|
||||
/// frames use.
|
||||
///
|
||||
/// [`decode_header`] accepts both spellings, so a CAR captured from an older
|
||||
/// build still parses.
|
||||
pub fn encode_header(roots: &[Cid]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
// Map(2): { "version": 1, "roots": [...] }
|
||||
cbor_head(&mut out, 5, 2);
|
||||
cbor_text(&mut out, "version");
|
||||
cbor_head(&mut out, 0, 1);
|
||||
cbor_text(&mut out, "roots");
|
||||
cbor_head(&mut out, 4, roots.len() as u64);
|
||||
write_head(&mut out, 5, 2);
|
||||
write_text(&mut out, "version");
|
||||
write_head(&mut out, 0, 1);
|
||||
write_text(&mut out, "roots");
|
||||
write_head(&mut out, 4, roots.len() as u64);
|
||||
for cid in roots {
|
||||
cbor_tag(&mut out, 42);
|
||||
cbor_bytes(&mut out, &cid.to_bytes());
|
||||
crate::dag_cbor::write_link(&mut out, cid);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -261,43 +233,83 @@ fn read_section(section: &[u8]) -> Result<(Cid, Vec<u8>)> {
|
||||
Ok((cid, data))
|
||||
}
|
||||
|
||||
/// Decode the CAR header written by [`encode_header`].
|
||||
///
|
||||
/// Structural only, and deliberately *not* routed through
|
||||
/// [`crate::dag_cbor::decode`]: that decoder enforces the `0x00` multibase
|
||||
/// identity prefix on every tag-42 link, which our own header does not carry
|
||||
/// (see the deviation note on [`encode_header`]). It does share the CBOR head
|
||||
/// reader with it, so there is exactly one implementation of that.
|
||||
#[allow(dead_code)]
|
||||
fn decode_header(bytes: &[u8]) -> Result<CarHeader> {
|
||||
// The header is a tiny DAG-CBOR map. We decode only the structure we emit.
|
||||
let mut p = 0usize;
|
||||
let (n_items, consumed) = read_head_and_uint(bytes, p, 5)?;
|
||||
p += consumed;
|
||||
let (major, n_items, next) = read_head(bytes, p)?;
|
||||
if major != 5 {
|
||||
anyhow::bail!("CAR header must be a CBOR map, got major type {major}");
|
||||
}
|
||||
if n_items != 2 {
|
||||
anyhow::bail!("CAR header must have 2 keys, got {n_items}");
|
||||
}
|
||||
p = next;
|
||||
|
||||
let mut version: Option<u64> = None;
|
||||
let mut roots: Vec<Cid> = Vec::new();
|
||||
|
||||
for _ in 0..2 {
|
||||
let (key, consumed) = read_head_and_text(bytes, p)?;
|
||||
p += consumed;
|
||||
let (major, len, next) = read_head(bytes, p)?;
|
||||
if major != 3 {
|
||||
anyhow::bail!("CAR header key must be text, got major type {major}");
|
||||
}
|
||||
p = next;
|
||||
if p + len as usize > bytes.len() {
|
||||
anyhow::bail!("CAR header key exceeds header");
|
||||
}
|
||||
let key = std::str::from_utf8(&bytes[p..p + len as usize])
|
||||
.map_err(|e| anyhow::anyhow!("invalid UTF-8 in CAR header key: {e}"))?
|
||||
.to_string();
|
||||
p += len as usize;
|
||||
|
||||
match key.as_str() {
|
||||
"version" => {
|
||||
let (v, c) = read_head_and_uint(bytes, p, 0)?;
|
||||
p += c;
|
||||
let (major, v, next) = read_head(bytes, p)?;
|
||||
if major != 0 {
|
||||
anyhow::bail!("CAR header `version` must be an unsigned int");
|
||||
}
|
||||
p = next;
|
||||
version = Some(v);
|
||||
}
|
||||
"roots" => {
|
||||
let (n_roots, c) = read_head_and_uint(bytes, p, 4)?;
|
||||
p += c;
|
||||
let (major, n_roots, next) = read_head(bytes, p)?;
|
||||
if major != 4 {
|
||||
anyhow::bail!("CAR header `roots` must be an array");
|
||||
}
|
||||
p = next;
|
||||
for _ in 0..n_roots {
|
||||
// tag(42)
|
||||
let (_, c) = read_head_and_uint(bytes, p, 6)?;
|
||||
p += c;
|
||||
// bytes
|
||||
let (n, c) = read_head_and_uint(bytes, p, 2)?;
|
||||
p += c;
|
||||
let (major, tag, next) = read_head(bytes, p)?;
|
||||
if major != 6 || tag != 42 {
|
||||
anyhow::bail!("CAR root must be CBOR tag 42, got major {major} tag {tag}");
|
||||
}
|
||||
p = next;
|
||||
let (major, n, next) = read_head(bytes, p)?;
|
||||
if major != 2 {
|
||||
anyhow::bail!("CAR root CID must be a byte string");
|
||||
}
|
||||
p = next;
|
||||
if p + n as usize > bytes.len() {
|
||||
anyhow::bail!("CAR root CID bytes exceed header");
|
||||
}
|
||||
let cid_bytes = &bytes[p..p + n as usize];
|
||||
let cid = Cid::read_bytes(cid_bytes)
|
||||
// Tolerate both spellings: the conformant
|
||||
// `0x00 || cid` this server writes today, and the bare
|
||||
// CID older builds wrote (see `encode_header`). A real
|
||||
// CID never starts with 0x00 — that byte position holds
|
||||
// the version varint, and version 0 does not exist — so
|
||||
// stripping it is unambiguous, not a guess.
|
||||
let raw = &bytes[p..p + n as usize];
|
||||
let raw = match raw.first() {
|
||||
Some(0x00) => &raw[1..],
|
||||
_ => raw,
|
||||
};
|
||||
let cid = Cid::read_bytes(raw)
|
||||
.map_err(|e| anyhow::anyhow!("invalid root CID bytes: {e}"))?;
|
||||
p += n as usize;
|
||||
roots.push(cid);
|
||||
@@ -313,92 +325,54 @@ fn decode_header(bytes: &[u8]) -> Result<CarHeader> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Read a CBOR head (single byte for value < 24, otherwise head + varint
|
||||
/// extension) and decode its value. Validates that the major type is
|
||||
/// `expected_major`. Returns the decoded value and the number of bytes
|
||||
/// consumed (head + any extension).
|
||||
#[allow(dead_code)]
|
||||
fn read_head_and_uint(
|
||||
bytes: &[u8],
|
||||
offset: usize,
|
||||
expected_major: u8,
|
||||
) -> Result<(u64, usize)> {
|
||||
if offset >= bytes.len() {
|
||||
anyhow::bail!("CBOR read past end of input");
|
||||
}
|
||||
let first = bytes[offset];
|
||||
let major = first >> 5;
|
||||
if major != expected_major {
|
||||
anyhow::bail!(
|
||||
"expected CBOR major {}, got {}",
|
||||
expected_major,
|
||||
major
|
||||
);
|
||||
}
|
||||
let low = first & 0x1f;
|
||||
let (value, extra) = match low {
|
||||
0..=23 => (low as u64, 0usize),
|
||||
24 => {
|
||||
if offset + 2 > bytes.len() {
|
||||
anyhow::bail!("truncated CBOR uint8");
|
||||
}
|
||||
(bytes[offset + 1] as u64, 1)
|
||||
}
|
||||
25 => {
|
||||
if offset + 3 > bytes.len() {
|
||||
anyhow::bail!("truncated CBOR uint16");
|
||||
}
|
||||
(
|
||||
((bytes[offset + 1] as u64) << 8) | (bytes[offset + 2] as u64),
|
||||
2,
|
||||
)
|
||||
}
|
||||
26 => {
|
||||
if offset + 5 > bytes.len() {
|
||||
anyhow::bail!("truncated CBOR uint32");
|
||||
}
|
||||
let n = ((bytes[offset + 1] as u64) << 24)
|
||||
| ((bytes[offset + 2] as u64) << 16)
|
||||
| ((bytes[offset + 3] as u64) << 8)
|
||||
| (bytes[offset + 4] as u64);
|
||||
(n, 4)
|
||||
}
|
||||
27 => {
|
||||
if offset + 9 > bytes.len() {
|
||||
anyhow::bail!("truncated CBOR uint64");
|
||||
}
|
||||
let mut n = 0u64;
|
||||
for i in 0..8 {
|
||||
n = (n << 8) | (bytes[offset + 1 + i] as u64);
|
||||
}
|
||||
(n, 8)
|
||||
}
|
||||
other => anyhow::bail!("unsupported CBOR uint tag {other}"),
|
||||
};
|
||||
Ok((value, 1 + extra))
|
||||
}
|
||||
|
||||
/// Read a CBOR text string with major type 3, returning the string and the
|
||||
/// total number of bytes consumed.
|
||||
#[allow(dead_code)]
|
||||
fn read_head_and_text(
|
||||
bytes: &[u8],
|
||||
offset: usize,
|
||||
) -> Result<(String, usize)> {
|
||||
let (n, c) = read_head_and_uint(bytes, offset, 3)?;
|
||||
if offset + c + n as usize > bytes.len() {
|
||||
anyhow::bail!("CBOR text string exceeds buffer");
|
||||
}
|
||||
let s = std::str::from_utf8(&bytes[offset + c..offset + c + n as usize])
|
||||
.map_err(|e| anyhow::anyhow!("invalid UTF-8 in CBOR text: {e}"))?;
|
||||
Ok((s.to_string(), c + n as usize))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
/// The identity prefix is what makes a root readable by a stock CAR
|
||||
/// library, so assert on the bytes rather than only on the round trip
|
||||
/// through our own parser — which would pass either way.
|
||||
#[test]
|
||||
fn header_roots_carry_the_identity_prefix() {
|
||||
let c = cid_for_cbor(b"a").unwrap();
|
||||
let bytes = encode_header(&[c]);
|
||||
let raw = c.to_bytes();
|
||||
// tag(42) is 0xD8 0x2A, then a byte string one longer than the CID,
|
||||
// whose first content byte is the 0x00 multibase identity prefix.
|
||||
let tag_at = bytes
|
||||
.windows(2)
|
||||
.position(|w| w == [0xD8, 0x2A])
|
||||
.expect("tag(42) must be present");
|
||||
let after_tag = &bytes[tag_at + 2..];
|
||||
let (major, len, next) = read_head(after_tag, 0).unwrap();
|
||||
assert_eq!(major, 2, "a link wraps a byte string");
|
||||
assert_eq!(len as usize, raw.len() + 1, "one byte longer than the CID");
|
||||
assert_eq!(after_tag[next], 0x00, "multibase identity prefix");
|
||||
assert_eq!(&after_tag[next + 1..next + 1 + raw.len()], &raw[..]);
|
||||
}
|
||||
|
||||
/// A CAR captured from an older build tagged the bare CID. Those bytes
|
||||
/// must keep parsing — otherwise upgrading the server would strand
|
||||
/// anything that stored a repo export.
|
||||
#[test]
|
||||
fn header_without_identity_prefix_still_parses() {
|
||||
let c = cid_for_cbor(b"legacy").unwrap();
|
||||
// Hand-build the old shape: map(2), "version", 1, "roots", [tag(42)
|
||||
// bytes(<bare cid>)].
|
||||
let mut old = Vec::new();
|
||||
write_head(&mut old, 5, 2);
|
||||
write_text(&mut old, "version");
|
||||
write_head(&mut old, 0, 1);
|
||||
write_text(&mut old, "roots");
|
||||
write_head(&mut old, 4, 1);
|
||||
write_head(&mut old, 6, 42);
|
||||
write_bytes(&mut old, &c.to_bytes());
|
||||
|
||||
let h = decode_header(&old).unwrap();
|
||||
assert_eq!(h.roots, vec![c], "legacy root must still decode");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_encodes_cids_with_tag_42() {
|
||||
let c1 = cid_for_cbor(b"a").unwrap();
|
||||
|
||||
@@ -0,0 +1,557 @@
|
||||
//! A small, self-contained DAG-CBOR encoder + decoder.
|
||||
//!
|
||||
//! Why this exists
|
||||
//!
|
||||
//! `ciborium` (the CBOR crate the rest of the workspace uses) speaks plain
|
||||
//! CBOR through serde. It has no notion of an IPLD *CID link*, which DAG-CBOR
|
||||
//! encodes as the IANA-registered tag `42` wrapping a byte string whose first
|
||||
//! byte is the multibase-identity prefix `0x00` followed by the binary CID.
|
||||
//! Serde has no representation for a CBOR tag, so `ciborium` silently encodes
|
||||
//! `cid::Cid` as a newtype struct instead — which is *not* DAG-CBOR and which
|
||||
//! no atproto consumer can read.
|
||||
//!
|
||||
//! `car.rs` already hand-rolled the handful of primitives needed for the CAR
|
||||
//! v1 header (`{version, roots: [<tag 42 link>]}`). The firehose frames need
|
||||
//! exactly the same primitives plus a couple more (arrays of maps, nullable
|
||||
//! links, byte strings, i64). Rather than write the encoder twice, both
|
||||
//! callers now go through this module.
|
||||
//!
|
||||
//! ## What "correct DAG-CBOR" means here
|
||||
//!
|
||||
//! * Map keys are text strings, sorted in the DAG-CBOR canonical order:
|
||||
//! shorter keys first, then bytewise-ascending within a length. This is the
|
||||
//! ordering `@ipld/dag-cbor` (and therefore the reference atproto
|
||||
//! implementation) emits, so a frame produced here is byte-identical to one
|
||||
//! produced by a Typescript PDS for the same logical value.
|
||||
//! * Integers use the shortest possible head. Byte and text strings likewise.
|
||||
//! * A CID link is `tag(42) || bytes(0x00 || <cid.to_bytes()>)`. The leading
|
||||
//! `0x00` is the multibase identity prefix mandated by the DAG-CBOR spec for
|
||||
//! binary CIDs; forgetting it is the single most common interop bug, so the
|
||||
//! decoder asserts on it too.
|
||||
//! * Floats are deliberately *not* supported. DAG-CBOR permits them but
|
||||
//! nothing in this codebase emits one, and accepting them would mean
|
||||
//! deciding on a canonical float encoding we would never exercise.
|
||||
//!
|
||||
//! ## Scope
|
||||
//!
|
||||
//! This is not a general CBOR library. It handles definite-length items only
|
||||
//! (DAG-CBOR forbids indefinite lengths anyway) and rejects everything it does
|
||||
//! not understand rather than guessing. It is used for the *envelope* of
|
||||
//! things — CAR headers and firehose frames — never for repo blocks, which
|
||||
//! are produced by `at_repo`/`at_crypto` with their own (see
|
||||
//! `crate::firehose`) conventions.
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use cid::Cid;
|
||||
|
||||
/// The IPLD CID-link tag. See <https://ipld.io/specs/codecs/dag-cbor/spec/>.
|
||||
pub const CID_TAG: u64 = 42;
|
||||
|
||||
/// A decoded (or to-be-encoded) DAG-CBOR value.
|
||||
///
|
||||
/// `Link` is kept distinct from `Bytes` so a round-trip through
|
||||
/// [`decode`] / [`encode`] preserves the tag rather than flattening a link
|
||||
/// into an anonymous byte string.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Value {
|
||||
Null,
|
||||
Bool(bool),
|
||||
/// Signed integer. CBOR major types 0 (non-negative) and 1 (negative).
|
||||
Int(i64),
|
||||
Bytes(Vec<u8>),
|
||||
Text(String),
|
||||
Array(Vec<Value>),
|
||||
/// Map with text keys. Insertion order is irrelevant — [`encode`] sorts
|
||||
/// into the canonical DAG-CBOR order, and [`decode`] returns keys in the
|
||||
/// order they appeared on the wire.
|
||||
Map(Vec<(String, Value)>),
|
||||
/// An IPLD CID link — `tag(42)` wrapping the identity-prefixed CID bytes.
|
||||
Link(Cid),
|
||||
}
|
||||
|
||||
impl Value {
|
||||
/// Convenience: build a `Map` from an iterator of pairs.
|
||||
pub fn map<I, K>(pairs: I) -> Value
|
||||
where
|
||||
I: IntoIterator<Item = (K, Value)>,
|
||||
K: Into<String>,
|
||||
{
|
||||
Value::Map(pairs.into_iter().map(|(k, v)| (k.into(), v)).collect())
|
||||
}
|
||||
|
||||
/// Convenience: a text value from anything string-ish.
|
||||
pub fn text(s: impl Into<String>) -> Value {
|
||||
Value::Text(s.into())
|
||||
}
|
||||
|
||||
/// Look up a key in a `Map`. Returns `None` for a non-map or a
|
||||
/// missing key. Used by the tests and by frame consumers.
|
||||
pub fn get(&self, key: &str) -> Option<&Value> {
|
||||
match self {
|
||||
Value::Map(entries) => entries.iter().find(|(k, _)| k == key).map(|(_, v)| v),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_i64(&self) -> Option<i64> {
|
||||
match self {
|
||||
Value::Int(i) => Some(*i),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> Option<&str> {
|
||||
match self {
|
||||
Value::Text(s) => Some(s.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bool(&self) -> Option<bool> {
|
||||
match self {
|
||||
Value::Bool(b) => Some(*b),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Value::Bytes(b) => Some(b.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_array(&self) -> Option<&[Value]> {
|
||||
match self {
|
||||
Value::Array(a) => Some(a.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_link(&self) -> Option<&Cid> {
|
||||
match self {
|
||||
Value::Link(c) => Some(c),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_null(&self) -> bool {
|
||||
matches!(self, Value::Null)
|
||||
}
|
||||
}
|
||||
|
||||
// -- encoding ---------------------------------------------------------------
|
||||
|
||||
/// Write a CBOR head: the 3-bit major type plus the argument, using the
|
||||
/// shortest encoding that fits.
|
||||
///
|
||||
/// Public because `car.rs` builds its length-prefixed sections around the same
|
||||
/// primitive and there is no reason to have two copies.
|
||||
pub fn write_head(out: &mut Vec<u8>, major: u8, n: u64) {
|
||||
let m = (major & 0x07) << 5;
|
||||
if n < 24 {
|
||||
out.push(m | n as u8);
|
||||
} else if n <= u8::MAX as u64 {
|
||||
out.push(m | 24);
|
||||
out.push(n as u8);
|
||||
} else if n <= u16::MAX as u64 {
|
||||
out.push(m | 25);
|
||||
out.extend_from_slice(&(n as u16).to_be_bytes());
|
||||
} else if n <= u32::MAX as u64 {
|
||||
out.push(m | 26);
|
||||
out.extend_from_slice(&(n as u32).to_be_bytes());
|
||||
} else {
|
||||
out.push(m | 27);
|
||||
out.extend_from_slice(&n.to_be_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a CBOR text string (major type 3).
|
||||
pub fn write_text(out: &mut Vec<u8>, s: &str) {
|
||||
write_head(out, 3, s.len() as u64);
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
/// Append a CBOR byte string (major type 2).
|
||||
pub fn write_bytes(out: &mut Vec<u8>, b: &[u8]) {
|
||||
write_head(out, 2, b.len() as u64);
|
||||
out.extend_from_slice(b);
|
||||
}
|
||||
|
||||
/// Append a CID as a DAG-CBOR link: `tag(42) || bytes(0x00 || cid)`.
|
||||
///
|
||||
/// The `0x00` is the multibase identity prefix. Binary CIDs inside DAG-CBOR
|
||||
/// always carry it; the textual form (`bafy…`) never does.
|
||||
pub fn write_link(out: &mut Vec<u8>, cid: &Cid) {
|
||||
write_head(out, 6, CID_TAG);
|
||||
let raw = cid.to_bytes();
|
||||
let mut prefixed = Vec::with_capacity(raw.len() + 1);
|
||||
prefixed.push(0x00);
|
||||
prefixed.extend_from_slice(&raw);
|
||||
write_bytes(out, &prefixed);
|
||||
}
|
||||
|
||||
/// DAG-CBOR canonical map-key order: shorter keys sort first; equal-length
|
||||
/// keys sort bytewise ascending.
|
||||
///
|
||||
/// This is RFC 7049's "canonical CBOR" rule, which DAG-CBOR inherited and
|
||||
/// which `@ipld/dag-cbor` implements. (RFC 8949 later switched the *core*
|
||||
/// deterministic profile to plain bytewise ordering, but DAG-CBOR did not
|
||||
/// follow — using 8949's rule here would produce frames that differ from the
|
||||
/// reference implementation's for keys like `"op"` vs `"t"`.)
|
||||
fn canonical_key_cmp(a: &str, b: &str) -> std::cmp::Ordering {
|
||||
a.len()
|
||||
.cmp(&b.len())
|
||||
.then_with(|| a.as_bytes().cmp(b.as_bytes()))
|
||||
}
|
||||
|
||||
/// Encode a value into `out`.
|
||||
pub fn encode_into(out: &mut Vec<u8>, value: &Value) {
|
||||
match value {
|
||||
Value::Null => out.push(0xF6),
|
||||
Value::Bool(false) => out.push(0xF4),
|
||||
Value::Bool(true) => out.push(0xF5),
|
||||
Value::Int(i) => {
|
||||
if *i >= 0 {
|
||||
write_head(out, 0, *i as u64);
|
||||
} else {
|
||||
// CBOR major type 1 stores -1-n, so n = -(i+1). Computed on
|
||||
// i64 via `i128` to stay correct at `i64::MIN`, where
|
||||
// `-(i + 1)` would overflow.
|
||||
let n = (-((*i as i128) + 1)) as u64;
|
||||
write_head(out, 1, n);
|
||||
}
|
||||
}
|
||||
Value::Bytes(b) => write_bytes(out, b),
|
||||
Value::Text(s) => write_text(out, s),
|
||||
Value::Array(items) => {
|
||||
write_head(out, 4, items.len() as u64);
|
||||
for item in items {
|
||||
encode_into(out, item);
|
||||
}
|
||||
}
|
||||
Value::Map(entries) => {
|
||||
let mut sorted: Vec<&(String, Value)> = entries.iter().collect();
|
||||
sorted.sort_by(|a, b| canonical_key_cmp(&a.0, &b.0));
|
||||
write_head(out, 5, sorted.len() as u64);
|
||||
for (k, v) in sorted {
|
||||
write_text(out, k);
|
||||
encode_into(out, v);
|
||||
}
|
||||
}
|
||||
Value::Link(cid) => write_link(out, cid),
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a value to a fresh `Vec<u8>`.
|
||||
///
|
||||
/// `#[allow(dead_code)]`: the server itself always appends into an existing
|
||||
/// buffer via [`encode_into`] (a frame is two values in one allocation), so
|
||||
/// this convenience wrapper is exercised only by the tests that assert on
|
||||
/// exact byte sequences. It is kept because the decoder half needs a matching
|
||||
/// encoder half to be testable at all.
|
||||
#[allow(dead_code)]
|
||||
pub fn encode(value: &Value) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
encode_into(&mut out, value);
|
||||
out
|
||||
}
|
||||
|
||||
// -- decoding ---------------------------------------------------------------
|
||||
|
||||
// -- Why a decoder lives in a server that only encodes ----------------------
|
||||
//
|
||||
// The PDS never parses a firehose frame in production — it writes them. The
|
||||
// decoder exists so the frame *contract* can be tested from the outside: a
|
||||
// unit test that only checks "the encoder produced these bytes" locks in
|
||||
// whatever the encoder happens to do, including its bugs. Decoding the bytes
|
||||
// back and asserting on the structure is what actually verifies that a tag-42
|
||||
// link is a link and not a byte string, that the two frame halves are
|
||||
// separable, and that a `null` `since` is `null` rather than absent.
|
||||
//
|
||||
// The integration test uses it for the same reason from the client side, and
|
||||
// `crates/appview` builds its consumer against the same shape.
|
||||
|
||||
/// Decode exactly one value, requiring it to consume the whole input.
|
||||
#[allow(dead_code)]
|
||||
pub fn decode(bytes: &[u8]) -> Result<Value> {
|
||||
let (v, used) = decode_one(bytes)?;
|
||||
if used != bytes.len() {
|
||||
bail!(
|
||||
"trailing bytes after DAG-CBOR value: consumed {used} of {}",
|
||||
bytes.len()
|
||||
);
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Decode one value from the front of `bytes`, returning it along with the
|
||||
/// number of bytes consumed.
|
||||
///
|
||||
/// This is the entry point the firehose frame reader needs: a frame is two
|
||||
/// concatenated DAG-CBOR values (header then body) with no length prefix
|
||||
/// between them, so the only way to find the body is to decode the header and
|
||||
/// see where it ended.
|
||||
#[allow(dead_code)]
|
||||
pub fn decode_one(bytes: &[u8]) -> Result<(Value, usize)> {
|
||||
decode_at(bytes, 0).map(|(v, end)| (v, end))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn decode_at(bytes: &[u8], offset: usize) -> Result<(Value, usize)> {
|
||||
let (major, arg, mut p) = read_head(bytes, offset)?;
|
||||
match major {
|
||||
0 => {
|
||||
let i = i64::try_from(arg).map_err(|_| anyhow!("CBOR uint {arg} exceeds i64"))?;
|
||||
Ok((Value::Int(i), p))
|
||||
}
|
||||
1 => {
|
||||
let v = -(arg as i128) - 1;
|
||||
let i = i64::try_from(v).map_err(|_| anyhow!("CBOR nint {v} exceeds i64"))?;
|
||||
Ok((Value::Int(i), p))
|
||||
}
|
||||
2 => {
|
||||
let end = p + arg as usize;
|
||||
if end > bytes.len() {
|
||||
bail!("CBOR byte string exceeds input");
|
||||
}
|
||||
Ok((Value::Bytes(bytes[p..end].to_vec()), end))
|
||||
}
|
||||
3 => {
|
||||
let end = p + arg as usize;
|
||||
if end > bytes.len() {
|
||||
bail!("CBOR text string exceeds input");
|
||||
}
|
||||
let s = std::str::from_utf8(&bytes[p..end])
|
||||
.map_err(|e| anyhow!("invalid UTF-8 in CBOR text: {e}"))?;
|
||||
Ok((Value::Text(s.to_string()), end))
|
||||
}
|
||||
4 => {
|
||||
let mut items = Vec::with_capacity(arg.min(1024) as usize);
|
||||
for _ in 0..arg {
|
||||
let (v, next) = decode_at(bytes, p)?;
|
||||
items.push(v);
|
||||
p = next;
|
||||
}
|
||||
Ok((Value::Array(items), p))
|
||||
}
|
||||
5 => {
|
||||
let mut entries = Vec::with_capacity(arg.min(1024) as usize);
|
||||
for _ in 0..arg {
|
||||
let (k, next) = decode_at(bytes, p)?;
|
||||
p = next;
|
||||
let key = match k {
|
||||
Value::Text(s) => s,
|
||||
other => bail!("DAG-CBOR map keys must be text, got {other:?}"),
|
||||
};
|
||||
let (v, next) = decode_at(bytes, p)?;
|
||||
p = next;
|
||||
entries.push((key, v));
|
||||
}
|
||||
Ok((Value::Map(entries), p))
|
||||
}
|
||||
6 => {
|
||||
if arg != CID_TAG {
|
||||
bail!("unsupported CBOR tag {arg}; DAG-CBOR allows only 42");
|
||||
}
|
||||
let (inner, end) = decode_at(bytes, p)?;
|
||||
let raw = match inner {
|
||||
Value::Bytes(b) => b,
|
||||
other => bail!("CBOR tag 42 must wrap a byte string, got {other:?}"),
|
||||
};
|
||||
let stripped = raw
|
||||
.split_first()
|
||||
.filter(|(first, _)| **first == 0x00)
|
||||
.map(|(_, rest)| rest)
|
||||
.ok_or_else(|| {
|
||||
anyhow!("CID link missing the 0x00 multibase identity prefix")
|
||||
})?;
|
||||
let cid = Cid::read_bytes(stripped)
|
||||
.map_err(|e| anyhow!("invalid CID inside tag 42: {e}"))?;
|
||||
Ok((Value::Link(cid), end))
|
||||
}
|
||||
7 => match arg {
|
||||
20 => Ok((Value::Bool(false), p)),
|
||||
21 => Ok((Value::Bool(true), p)),
|
||||
22 => Ok((Value::Null, p)),
|
||||
// 23 is `undefined`, 25/26/27 are floats. DAG-CBOR forbids
|
||||
// `undefined`; floats are out of scope (see the module header).
|
||||
other => bail!("unsupported CBOR simple/float value {other}"),
|
||||
},
|
||||
other => bail!("unsupported CBOR major type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a CBOR head at `offset`, returning `(major, argument, next_offset)`.
|
||||
///
|
||||
/// Public so `car.rs` can drive its own (deliberately non-conformant, see
|
||||
/// there) header parser off the same primitive instead of keeping a second
|
||||
/// copy.
|
||||
pub fn read_head(bytes: &[u8], offset: usize) -> Result<(u8, u64, usize)> {
|
||||
let first = *bytes
|
||||
.get(offset)
|
||||
.ok_or_else(|| anyhow!("CBOR read past end of input at {offset}"))?;
|
||||
let major = first >> 5;
|
||||
let low = first & 0x1f;
|
||||
let (arg, extra) = match low {
|
||||
0..=23 => (low as u64, 0usize),
|
||||
24 => (read_uint(bytes, offset + 1, 1)?, 1),
|
||||
25 => (read_uint(bytes, offset + 1, 2)?, 2),
|
||||
26 => (read_uint(bytes, offset + 1, 4)?, 4),
|
||||
27 => (read_uint(bytes, offset + 1, 8)?, 8),
|
||||
// 28..=30 are reserved; 31 is the indefinite-length marker, which
|
||||
// DAG-CBOR forbids outright.
|
||||
other => bail!("invalid or indefinite CBOR head 0x{other:02x}"),
|
||||
};
|
||||
Ok((major, arg, offset + 1 + extra))
|
||||
}
|
||||
|
||||
fn read_uint(bytes: &[u8], offset: usize, width: usize) -> Result<u64> {
|
||||
if offset + width > bytes.len() {
|
||||
bail!("truncated CBOR integer of width {width}");
|
||||
}
|
||||
let mut n: u64 = 0;
|
||||
for b in &bytes[offset..offset + width] {
|
||||
n = (n << 8) | *b as u64;
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
fn round_trip(v: Value) {
|
||||
let bytes = encode(&v);
|
||||
let back = decode(&bytes).expect("decode");
|
||||
// Maps come back in canonical (encoded) order, so compare the
|
||||
// re-encoding rather than the structure for map-bearing values.
|
||||
assert_eq!(encode(&back), bytes, "re-encode must be stable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalars_round_trip() {
|
||||
for v in [
|
||||
Value::Null,
|
||||
Value::Bool(true),
|
||||
Value::Bool(false),
|
||||
Value::Int(0),
|
||||
Value::Int(23),
|
||||
Value::Int(24),
|
||||
Value::Int(255),
|
||||
Value::Int(256),
|
||||
Value::Int(65_535),
|
||||
Value::Int(65_536),
|
||||
Value::Int(i64::MAX),
|
||||
Value::Int(-1),
|
||||
Value::Int(-24),
|
||||
Value::Int(-1000),
|
||||
Value::Int(i64::MIN),
|
||||
Value::Text("hello".into()),
|
||||
Value::Text(String::new()),
|
||||
Value::Bytes(vec![1, 2, 3]),
|
||||
Value::Bytes(Vec::new()),
|
||||
] {
|
||||
let bytes = encode(&v);
|
||||
assert_eq!(decode(&bytes).unwrap(), v, "round trip of {v:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_heads_are_shortest_form() {
|
||||
assert_eq!(encode(&Value::Int(1)), vec![0x01]);
|
||||
assert_eq!(encode(&Value::Int(24)), vec![0x18, 24]);
|
||||
assert_eq!(encode(&Value::Int(-1)), vec![0x20]);
|
||||
assert_eq!(encode(&Value::Int(-25)), vec![0x38, 24]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cid_link_carries_tag_42_and_identity_prefix() {
|
||||
let cid = cid_for_cbor(b"a block").unwrap();
|
||||
let bytes = encode(&Value::Link(cid));
|
||||
// 0xD8 0x2A == tag(42) in two-byte form.
|
||||
assert_eq!(&bytes[0..2], &[0xD8, 0x2A]);
|
||||
// Then a byte string whose first content byte is the 0x00 prefix.
|
||||
let (_major, len, p) = read_head(&bytes, 2).unwrap();
|
||||
assert_eq!(len as usize, cid.to_bytes().len() + 1);
|
||||
assert_eq!(bytes[p], 0x00);
|
||||
assert_eq!(decode(&bytes).unwrap(), Value::Link(cid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn link_without_identity_prefix_is_rejected() {
|
||||
let cid = cid_for_cbor(b"x").unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
write_head(&mut bytes, 6, CID_TAG);
|
||||
// Deliberately omit the leading 0x00.
|
||||
write_bytes(&mut bytes, &cid.to_bytes());
|
||||
let e = decode(&bytes).unwrap_err().to_string();
|
||||
assert!(e.contains("identity prefix"), "got: {e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_keys_are_sorted_length_first() {
|
||||
// The firehose header is exactly this shape, and the reference
|
||||
// implementation emits `t` before `op` because it is shorter.
|
||||
let v = Value::map([("op", Value::Int(1)), ("t", Value::text("#commit"))]);
|
||||
let bytes = encode(&v);
|
||||
assert_eq!(bytes[0], 0xA2, "map(2)");
|
||||
assert_eq!(bytes[1], 0x61, "text(1)");
|
||||
assert_eq!(bytes[2], b't');
|
||||
// …and `op` follows after the "#commit" value.
|
||||
let decoded = decode(&bytes).unwrap();
|
||||
match &decoded {
|
||||
Value::Map(entries) => {
|
||||
assert_eq!(entries[0].0, "t");
|
||||
assert_eq!(entries[1].0, "op");
|
||||
}
|
||||
other => panic!("expected map, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_structures_round_trip() {
|
||||
let cid = cid_for_cbor(b"nested").unwrap();
|
||||
round_trip(Value::map([
|
||||
("seq", Value::Int(42)),
|
||||
("commit", Value::Link(cid)),
|
||||
(
|
||||
"ops",
|
||||
Value::Array(vec![Value::map([
|
||||
("action", Value::text("create")),
|
||||
("path", Value::text("app.twi.post/3l")),
|
||||
("cid", Value::Link(cid)),
|
||||
])]),
|
||||
),
|
||||
("blobs", Value::Array(vec![])),
|
||||
("since", Value::Null),
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_one_stops_at_the_value_boundary() {
|
||||
// Two concatenated values — exactly how a firehose frame is laid out.
|
||||
let mut buf = encode(&Value::map([("op", Value::Int(1))]));
|
||||
let header_len = buf.len();
|
||||
buf.extend_from_slice(&encode(&Value::map([("seq", Value::Int(7))])));
|
||||
let (header, used) = decode_one(&buf).unwrap();
|
||||
assert_eq!(used, header_len);
|
||||
assert_eq!(header.get("op").and_then(Value::as_i64), Some(1));
|
||||
let body = decode(&buf[used..]).unwrap();
|
||||
assert_eq!(body.get("seq").and_then(Value::as_i64), Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn indefinite_length_is_rejected() {
|
||||
// 0x9F == array(*) — legal CBOR, illegal DAG-CBOR.
|
||||
assert!(decode(&[0x9F, 0x01, 0xFF]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_bytes_are_rejected() {
|
||||
let mut bytes = encode(&Value::Int(1));
|
||||
bytes.push(0x01);
|
||||
assert!(decode(&bytes).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,946 @@
|
||||
//! `com.atproto.sync.subscribeRepos` — the event log, the frame codec and the
|
||||
//! in-process broadcast channel.
|
||||
//!
|
||||
//! # What this is for
|
||||
//!
|
||||
//! Before this module the PDS emitted no firehose. A locally created record
|
||||
//! reached the AppView through exactly one channel: the best-effort HTTP push
|
||||
//! in [`crate::appview_push`], a detached `tokio::spawn` whose failure branch
|
||||
//! logs "jetstream will replay". For records that only exist on this PDS
|
||||
//! there *is* no Jetstream to replay them, so a lost push meant the post was
|
||||
//! never indexed and nothing would ever notice. The firehose replaces that
|
||||
//! hope with a durable, ordered log: every repo write appends one row inside
|
||||
//! the same transaction as the commit, and a consumer can ask for everything
|
||||
//! after a cursor at any later time.
|
||||
//!
|
||||
//! # Frame format
|
||||
//!
|
||||
//! One WebSocket **binary** message is two DAG-CBOR values written back to
|
||||
//! back with nothing between them — a header, then a body. There is no length
|
||||
//! prefix; the reader decodes the header and continues the body at the offset
|
||||
//! where the header ended (that is what [`crate::dag_cbor::decode_one`] is
|
||||
//! for).
|
||||
//!
|
||||
//! ```text
|
||||
//! regular: {"op": 1, "t": "#commit"} {"seq": …, "repo": …, …}
|
||||
//! {"op": 1, "t": "#info"} {"name": …, "message": …}
|
||||
//! error: {"op": -1} {"error": "<Name>", "message": "<Text>"}
|
||||
//! ```
|
||||
//!
|
||||
//! The `#commit` body carries:
|
||||
//!
|
||||
//! | field | type |
|
||||
//! |-----------|---------------------------------------------------------|
|
||||
//! | `seq` | int — the cursor value for this event |
|
||||
//! | `rebase` | bool — always `false` (we never rebase a repo) |
|
||||
//! | `tooBig` | bool — always `false` (see the size note below) |
|
||||
//! | `repo` | text — the DID |
|
||||
//! | `commit` | **CID link (tag 42)** — the new commit block |
|
||||
//! | `rev` | text — the new commit's revision |
|
||||
//! | `since` | text or null — the previous commit's revision |
|
||||
//! | `blocks` | byte string — a CAR v1 file, commit block as root |
|
||||
//! | `ops` | array of `{action, path, cid}`; `cid` is a link or null |
|
||||
//! | `blobs` | array — always empty (blob refs live inside the record) |
|
||||
//! | `time` | text — RFC 3339, when the event was appended |
|
||||
//!
|
||||
//! Map keys are emitted in DAG-CBOR canonical order (shortest first, then
|
||||
//! bytewise), so the bytes match what a reference atproto implementation
|
||||
//! would produce for the same logical value.
|
||||
//!
|
||||
//! # Deliberate deviation from the atproto spec
|
||||
//!
|
||||
//! **The frame envelope is conformant. The blocks inside `blocks` are not.**
|
||||
//!
|
||||
//! This repository encodes CIDs *inside* commit blocks as CBOR text strings
|
||||
//! rather than as DAG-CBOR links with tag 42 — see `at_repo::commit` and
|
||||
//! `at_crypto::signing::sign_dag_cbor`. That convention predates this module
|
||||
//! and is load-bearing: the block bytes determine every CID in the system,
|
||||
//! including the `did:plc:` derivation, so changing it would re-address every
|
||||
//! repo in the database. It is explicitly out of scope here.
|
||||
//!
|
||||
//! The consequence, stated plainly: a foreign atproto consumer can connect,
|
||||
//! parse every frame, read `seq` / `repo` / `rev` / `ops`, and follow the
|
||||
//! stream. It will then fail when it tries to *validate* the payload — the
|
||||
//! CAR in `blocks` parses fine and the block CIDs hash correctly over their
|
||||
//! own bytes, but decoding a commit block as DAG-CBOR yields `"prev"` and
|
||||
//! `"data"` as strings where the spec demands links, and MST traversal
|
||||
//! against a stock implementation will not work. The AppView in this
|
||||
//! workspace reads the frames with the same conventions this crate writes
|
||||
//! them, which is why it works there.
|
||||
//!
|
||||
//! `tooBig` is therefore always `false`: it exists so a producer can say "the
|
||||
//! diff was too large, go fetch the repo yourself", and we never make that
|
||||
//! call — every commit here is one record change, and its CAR is small.
|
||||
//!
|
||||
//! # Lagging consumers
|
||||
//!
|
||||
//! The broadcast channel is bounded ([`FIREHOSE_CHANNEL_CAPACITY`]). A
|
||||
//! consumer that reads slower than the PDS writes will eventually be lapped,
|
||||
//! and `tokio::sync::broadcast` reports that as `RecvError::Lagged(n)`.
|
||||
//!
|
||||
//! The write path must never wait on a reader, so the channel cannot be made
|
||||
//! blocking. When a reader lags we send it an `#info` / `OutdatedCursor`
|
||||
//! frame and **fall back to the database replay** from the last sequence it
|
||||
//! actually received, then resume live. We do not disconnect it: the events
|
||||
//! are durable in `firehose_events`, so the fallback is lossless, whereas
|
||||
//! dropping the socket would force the client to reconnect and perform
|
||||
//! exactly the same replay after two extra round trips. The only thing a
|
||||
//! disconnect would buy is protection against a client that lags forever, and
|
||||
//! that is handled separately by capping consecutive recoveries
|
||||
//! ([`MAX_LAG_RECOVERIES`]) before closing with an error frame.
|
||||
//!
|
||||
//! # Retention
|
||||
//!
|
||||
//! Nothing prunes `firehose_events`. It grows by one row per repo write, each
|
||||
//! carrying the CAR of that commit's new blocks. That is unbounded, and this
|
||||
//! deployment has no retention job — an operator who wants one has to add it.
|
||||
//! Pruning is safe by design: the cursor handshake compares the requested
|
||||
//! cursor against the oldest surviving row and answers a too-old cursor with
|
||||
//! `#info` / `OutdatedCursor` followed by a replay from the oldest row that
|
||||
//! still exists, rather than pretending the gap is not there.
|
||||
|
||||
use crate::car::CarWriter;
|
||||
use crate::dag_cbor::{encode_into, Value};
|
||||
use anyhow::{anyhow, Result};
|
||||
use at_crypto::cid::cid_from_multihash_bytes;
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
use cid::Cid;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
/// How many events the live broadcast channel buffers per subscriber before
|
||||
/// the slowest one starts reporting `Lagged`.
|
||||
///
|
||||
/// 1024 is chosen so a consumer can stall for the length of a garbage
|
||||
/// collection or a slow network write without falling back to the database,
|
||||
/// while the memory ceiling stays bounded: the channel holds `Arc`s, so the
|
||||
/// cost is one CAR blob per queued event, shared across all subscribers.
|
||||
pub const FIREHOSE_CHANNEL_CAPACITY: usize = 1024;
|
||||
|
||||
/// How many times in a row a single connection may be rescued from a lag
|
||||
/// before we give up and close it. A client that cannot keep up even with a
|
||||
/// database replay in between is not going to start; at that point the honest
|
||||
/// answer is an error frame rather than an endless catch-up loop that burns
|
||||
/// queries on its behalf.
|
||||
pub const MAX_LAG_RECOVERIES: u32 = 5;
|
||||
|
||||
/// Rows returned per replay query. Bounded so a client reconnecting with
|
||||
/// `cursor=0` after a long uptime streams the backlog in chunks instead of
|
||||
/// materialising the whole table (and every CAR in it) in memory at once.
|
||||
pub const REPLAY_PAGE_SIZE: i64 = 200;
|
||||
|
||||
/// The advisory-lock key that serialises `firehose_events` INSERTs. Any
|
||||
/// constant works as long as every writer uses the same one; this is
|
||||
/// `"fhose"` read as ASCII, which makes it recognisable in `pg_locks`.
|
||||
///
|
||||
/// What it costs: the lock is global, not per-repo, so the tail of every
|
||||
/// repo write — INSERT plus COMMIT — is serialised across all accounts.
|
||||
/// That is deliberate (a per-repo lock would order each repo's events but
|
||||
/// not the shared `seq` a consumer paginates on), and it bounds write
|
||||
/// throughput to how fast Postgres can commit one small INSERT at a time.
|
||||
/// If that ever becomes the ceiling, the fix is a different sequence
|
||||
/// design — a per-repo cursor, or handing out `seq` from a single writer
|
||||
/// task — not a weaker lock: a gap in `seq` is silent data loss for every
|
||||
/// consumer replaying from a cursor.
|
||||
pub const FIREHOSE_ADVISORY_LOCK_KEY: i64 = 0x66_68_6f_73_65;
|
||||
|
||||
// -- ops --------------------------------------------------------------------
|
||||
|
||||
/// What a single repo operation did to one record.
|
||||
///
|
||||
/// `Update` exists separately from `Create` because a consumer that keeps a
|
||||
/// materialised view needs to know whether to insert or replace; the MST
|
||||
/// itself does not distinguish them, so the write path resolves it by looking
|
||||
/// the key up before writing.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum RepoOpAction {
|
||||
Create,
|
||||
Update,
|
||||
Delete,
|
||||
}
|
||||
|
||||
impl RepoOpAction {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
RepoOpAction::Create => "create",
|
||||
RepoOpAction::Update => "update",
|
||||
RepoOpAction::Delete => "delete",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Result<Self> {
|
||||
match s {
|
||||
"create" => Ok(RepoOpAction::Create),
|
||||
"update" => Ok(RepoOpAction::Update),
|
||||
"delete" => Ok(RepoOpAction::Delete),
|
||||
other => Err(anyhow!("unknown repo op action `{other}`")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of a commit frame's `ops` array.
|
||||
///
|
||||
/// `path` is `"<collection>/<rkey>"` — the MST key, not an `at://` URI. That
|
||||
/// is what the wire format specifies, and it is also what the MST is actually
|
||||
/// keyed by, so there is no reassembly step on either side.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RepoOp {
|
||||
pub action: RepoOpAction,
|
||||
pub path: String,
|
||||
/// The record value's CID for `create` / `update`; `None` for `delete`,
|
||||
/// where there is no resulting value to point at.
|
||||
pub cid: Option<Cid>,
|
||||
}
|
||||
|
||||
impl RepoOp {
|
||||
pub fn create(collection: &str, rkey: &str, cid: Cid) -> Self {
|
||||
Self {
|
||||
action: RepoOpAction::Create,
|
||||
path: format!("{collection}/{rkey}"),
|
||||
cid: Some(cid),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update(collection: &str, rkey: &str, cid: Cid) -> Self {
|
||||
Self {
|
||||
action: RepoOpAction::Update,
|
||||
path: format!("{collection}/{rkey}"),
|
||||
cid: Some(cid),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn delete(collection: &str, rkey: &str) -> Self {
|
||||
Self {
|
||||
action: RepoOpAction::Delete,
|
||||
path: format!("{collection}/{rkey}"),
|
||||
cid: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Pick `create` or `update` from whether the key already existed.
|
||||
pub fn put(collection: &str, rkey: &str, cid: Cid, existed: bool) -> Self {
|
||||
if existed {
|
||||
Self::update(collection, rkey, cid)
|
||||
} else {
|
||||
Self::create(collection, rkey, cid)
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON shape stored in `firehose_events.ops`. The CID is a string here
|
||||
/// (JSONB has no link type); it becomes a tag-42 link again on the wire.
|
||||
pub fn to_json(&self) -> serde_json::Value {
|
||||
json!({
|
||||
"action": self.action.as_str(),
|
||||
"path": self.path,
|
||||
"cid": self.cid.map(|c| c.to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_json(v: &serde_json::Value) -> Result<Self> {
|
||||
let action = RepoOpAction::parse(
|
||||
v.get("action")
|
||||
.and_then(|a| a.as_str())
|
||||
.ok_or_else(|| anyhow!("op missing `action`"))?,
|
||||
)?;
|
||||
let path = v
|
||||
.get("path")
|
||||
.and_then(|p| p.as_str())
|
||||
.ok_or_else(|| anyhow!("op missing `path`"))?
|
||||
.to_string();
|
||||
let cid = match v.get("cid") {
|
||||
None | Some(serde_json::Value::Null) => None,
|
||||
Some(serde_json::Value::String(s)) => {
|
||||
Some(s.parse::<Cid>().map_err(|e| anyhow!("op cid: {e}"))?)
|
||||
}
|
||||
Some(other) => return Err(anyhow!("op `cid` must be a string or null, got {other}")),
|
||||
};
|
||||
Ok(Self { action, path, cid })
|
||||
}
|
||||
|
||||
fn to_dag_cbor(&self) -> Value {
|
||||
Value::map([
|
||||
("action", Value::text(self.action.as_str())),
|
||||
("path", Value::text(&self.path)),
|
||||
(
|
||||
"cid",
|
||||
match self.cid {
|
||||
Some(c) => Value::Link(c),
|
||||
None => Value::Null,
|
||||
},
|
||||
),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
// -- events -----------------------------------------------------------------
|
||||
|
||||
/// One row of `firehose_events`, ready to be framed.
|
||||
///
|
||||
/// Both the live path (built at write time) and the replay path (read back
|
||||
/// from Postgres) produce this exact struct, which is what makes a replayed
|
||||
/// frame byte-identical to the live one — including `time`, which comes from
|
||||
/// the stored `created_at` rather than from the clock at send time.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FirehoseEvent {
|
||||
pub seq: i64,
|
||||
pub did: String,
|
||||
pub rev: String,
|
||||
pub since: Option<String>,
|
||||
pub commit: Cid,
|
||||
/// CAR v1 file: the commit block as root, plus the blocks this commit
|
||||
/// newly created.
|
||||
pub blocks: Vec<u8>,
|
||||
pub ops: Vec<RepoOp>,
|
||||
pub time: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl FirehoseEvent {
|
||||
/// The `#commit` body as a DAG-CBOR value.
|
||||
pub fn to_body(&self) -> Value {
|
||||
Value::map([
|
||||
("seq", Value::Int(self.seq)),
|
||||
("rebase", Value::Bool(false)),
|
||||
("tooBig", Value::Bool(false)),
|
||||
("repo", Value::text(&self.did)),
|
||||
("commit", Value::Link(self.commit)),
|
||||
("rev", Value::text(&self.rev)),
|
||||
(
|
||||
"since",
|
||||
match &self.since {
|
||||
Some(s) => Value::text(s),
|
||||
None => Value::Null,
|
||||
},
|
||||
),
|
||||
("blocks", Value::Bytes(self.blocks.clone())),
|
||||
(
|
||||
"ops",
|
||||
Value::Array(self.ops.iter().map(RepoOp::to_dag_cbor).collect()),
|
||||
),
|
||||
("blobs", Value::Array(Vec::new())),
|
||||
(
|
||||
"time",
|
||||
Value::text(self.time.to_rfc3339_opts(SecondsFormat::Micros, true)),
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
/// The full binary WebSocket payload: `#commit` header then body.
|
||||
pub fn to_frame(&self) -> Vec<u8> {
|
||||
frame(&header_value("#commit"), &self.to_body())
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the CAR that goes into a commit event's `blocks` field.
|
||||
///
|
||||
/// Root is the commit block; the remaining entries are the blocks this commit
|
||||
/// newly wrote (MST nodes and record values). Blocks that already existed in
|
||||
/// the repo are deliberately left out — that is the whole point of a diff
|
||||
/// stream, and a consumer that needs an ancestor block asks
|
||||
/// `com.atproto.sync.getBlocks` for it.
|
||||
pub fn build_blocks_car(
|
||||
commit_cid: Cid,
|
||||
commit_block: &[u8],
|
||||
new_blocks: &[(Cid, Vec<u8>)],
|
||||
) -> Vec<u8> {
|
||||
let mut w = CarWriter::new();
|
||||
w.append(commit_cid, commit_block);
|
||||
for (cid, data) in new_blocks {
|
||||
w.append(*cid, data);
|
||||
}
|
||||
w.finish(&[commit_cid])
|
||||
}
|
||||
|
||||
// -- frame encoding ---------------------------------------------------------
|
||||
|
||||
/// `{"op": 1, "t": "<t>"}` — the header of a regular frame.
|
||||
fn header_value(t: &str) -> Value {
|
||||
Value::map([("op", Value::Int(1)), ("t", Value::text(t))])
|
||||
}
|
||||
|
||||
/// Concatenate a header and a body into one binary WebSocket payload.
|
||||
fn frame(header: &Value, body: &Value) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
encode_into(&mut out, header);
|
||||
encode_into(&mut out, body);
|
||||
out
|
||||
}
|
||||
|
||||
/// An `#info` frame: `{"op":1,"t":"#info"}` + `{"name":…,"message":…}`.
|
||||
///
|
||||
/// Informational, not fatal — the stream continues after it. We send it when
|
||||
/// a cursor is older than anything we still have, and when a live subscriber
|
||||
/// lagged and is being put back on the database replay.
|
||||
pub fn encode_info_frame(name: &str, message: &str) -> Vec<u8> {
|
||||
frame(
|
||||
&header_value("#info"),
|
||||
&Value::map([("name", Value::text(name)), ("message", Value::text(message))]),
|
||||
)
|
||||
}
|
||||
|
||||
/// An error frame: `{"op":-1}` + `{"error":…,"message":…}`.
|
||||
///
|
||||
/// Terminal — the server closes the socket right after sending it.
|
||||
pub fn encode_error_frame(error: &str, message: &str) -> Vec<u8> {
|
||||
frame(
|
||||
&Value::map([("op", Value::Int(-1))]),
|
||||
&Value::map([
|
||||
("error", Value::text(error)),
|
||||
("message", Value::text(message)),
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
// -- broadcast --------------------------------------------------------------
|
||||
|
||||
/// The in-process fan-out from the write path to connected subscribers.
|
||||
///
|
||||
/// Cloneable and cheap: it is a `broadcast::Sender` plus nothing. Events are
|
||||
/// wrapped in an `Arc` so a burst of subscribers does not multiply the CAR
|
||||
/// blobs.
|
||||
#[derive(Clone)]
|
||||
pub struct Firehose {
|
||||
tx: broadcast::Sender<Arc<FirehoseEvent>>,
|
||||
}
|
||||
|
||||
impl Default for Firehose {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Firehose {
|
||||
pub fn new() -> Self {
|
||||
let (tx, _rx) = broadcast::channel(FIREHOSE_CHANNEL_CAPACITY);
|
||||
Self { tx }
|
||||
}
|
||||
|
||||
/// Publish an event to every live subscriber.
|
||||
///
|
||||
/// Never blocks and never fails in a way the caller must handle: with no
|
||||
/// subscribers the send returns `Err`, which is the normal state of a PDS
|
||||
/// nobody is watching. The event is already durable in Postgres by the
|
||||
/// time we get here, so a dropped broadcast costs a consumer nothing
|
||||
/// beyond having to replay by cursor.
|
||||
pub fn publish(&self, event: FirehoseEvent) {
|
||||
let _ = self.tx.send(Arc::new(event));
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<Arc<FirehoseEvent>> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
|
||||
/// Number of live subscribers. Used for logging / the healthz surface.
|
||||
#[allow(dead_code)]
|
||||
pub fn subscriber_count(&self) -> usize {
|
||||
self.tx.receiver_count()
|
||||
}
|
||||
}
|
||||
|
||||
// -- persistence ------------------------------------------------------------
|
||||
|
||||
/// Append one event inside an open transaction and return it with its
|
||||
/// assigned `seq` and `created_at`.
|
||||
///
|
||||
/// The advisory lock taken first is what makes the sequence usable as a
|
||||
/// cursor: without it two concurrent writers can be assigned seq 5 and 6 and
|
||||
/// commit in the other order, so a reader polling in between sees 6, records
|
||||
/// it as its cursor, and never learns about 5. Holding
|
||||
/// `pg_advisory_xact_lock` from just before the INSERT until COMMIT forces
|
||||
/// commit order to match seq order. It is taken *after* the per-repo
|
||||
/// `SELECT … FOR UPDATE` in [`crate::routes::helpers::apply_repo_write`], and
|
||||
/// every writer takes the two in that same order, so the pair cannot deadlock.
|
||||
pub async fn insert_event_in_tx(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
|
||||
did: &str,
|
||||
rev: &str,
|
||||
since: Option<&str>,
|
||||
commit: Cid,
|
||||
blocks: Vec<u8>,
|
||||
ops: Vec<RepoOp>,
|
||||
) -> Result<FirehoseEvent, sqlx::Error> {
|
||||
sqlx::query("SELECT pg_advisory_xact_lock($1)")
|
||||
.bind(FIREHOSE_ADVISORY_LOCK_KEY)
|
||||
.execute(&mut **tx)
|
||||
.await?;
|
||||
|
||||
let ops_json = serde_json::Value::Array(ops.iter().map(RepoOp::to_json).collect());
|
||||
|
||||
let (seq, created_at): (i64, DateTime<Utc>) = sqlx::query_as(
|
||||
r#"INSERT INTO firehose_events (did, rev, since, commit_cid, blocks, ops)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
RETURNING seq, created_at"#,
|
||||
)
|
||||
.bind(did)
|
||||
.bind(rev)
|
||||
.bind(since)
|
||||
.bind(commit.to_bytes())
|
||||
.bind(&blocks)
|
||||
.bind(&ops_json)
|
||||
.fetch_one(&mut **tx)
|
||||
.await?;
|
||||
|
||||
Ok(FirehoseEvent {
|
||||
seq,
|
||||
did: did.to_string(),
|
||||
rev: rev.to_string(),
|
||||
since: since.map(|s| s.to_string()),
|
||||
commit,
|
||||
blocks,
|
||||
ops,
|
||||
time: created_at,
|
||||
})
|
||||
}
|
||||
|
||||
/// The `(min_seq, max_seq)` currently in the table, or `None` when it is
|
||||
/// empty. Used by the cursor handshake to tell "from the future" apart from
|
||||
/// "too old to still have".
|
||||
pub async fn seq_bounds(db: &sqlx::PgPool) -> Result<Option<(i64, i64)>, sqlx::Error> {
|
||||
let row: (Option<i64>, Option<i64>) =
|
||||
sqlx::query_as("SELECT MIN(seq), MAX(seq) FROM firehose_events")
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
Ok(match row {
|
||||
(Some(min), Some(max)) => Some((min, max)),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Read up to [`REPLAY_PAGE_SIZE`] events with `seq > after`, oldest first.
|
||||
pub async fn load_events_after(
|
||||
db: &sqlx::PgPool,
|
||||
after: i64,
|
||||
limit: i64,
|
||||
) -> Result<Vec<FirehoseEvent>> {
|
||||
let rows: Vec<(
|
||||
i64,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Vec<u8>,
|
||||
Vec<u8>,
|
||||
serde_json::Value,
|
||||
DateTime<Utc>,
|
||||
)> = sqlx::query_as(
|
||||
r#"SELECT seq, did, rev, since, commit_cid, blocks, ops, created_at
|
||||
FROM firehose_events
|
||||
WHERE seq > $1
|
||||
ORDER BY seq ASC
|
||||
LIMIT $2"#,
|
||||
)
|
||||
.bind(after)
|
||||
.bind(limit)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.map_err(|e| anyhow!("firehose_events replay read: {e}"))?;
|
||||
|
||||
rows.into_iter()
|
||||
.map(|(seq, did, rev, since, commit_cid, blocks, ops, created_at)| {
|
||||
let commit = cid_from_multihash_bytes(&commit_cid)
|
||||
.map_err(|e| anyhow!("firehose_events.commit_cid at seq {seq}: {e}"))?;
|
||||
let ops = ops
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("firehose_events.ops at seq {seq} is not an array"))?
|
||||
.iter()
|
||||
.map(RepoOp::from_json)
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
Ok(FirehoseEvent {
|
||||
seq,
|
||||
did,
|
||||
rev,
|
||||
since,
|
||||
commit,
|
||||
blocks,
|
||||
ops,
|
||||
time: created_at,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
// -- cursor handshake -------------------------------------------------------
|
||||
|
||||
/// What the connection handler should do with the cursor the client sent.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum CursorPlan {
|
||||
/// No cursor: send live events only, nothing from the log.
|
||||
LiveOnly,
|
||||
/// Replay everything after `from`, then go live.
|
||||
Replay { from: i64 },
|
||||
/// The requested cursor predates the oldest row we still have. Warn with
|
||||
/// an `#info` frame, then replay from `from` (the oldest surviving row
|
||||
/// minus one) so the client at least gets everything that does exist.
|
||||
OutdatedCursor { from: i64, message: String },
|
||||
/// The cursor names an event that has not happened. This is a client bug
|
||||
/// (or a cursor from a different server's log), and continuing would
|
||||
/// silently strand it — so it is a terminal error frame.
|
||||
FutureCursor { message: String },
|
||||
}
|
||||
|
||||
/// Decide what to do with `cursor` given the log's current `(min, max)`.
|
||||
///
|
||||
/// Split out from the socket handler so the boundary conditions are testable
|
||||
/// without a database or a WebSocket.
|
||||
///
|
||||
/// Semantics of the cursor: it is the seq of the last event the client
|
||||
/// *already has*, so a replay yields `seq > cursor`. `cursor = 0` therefore
|
||||
/// means "everything", and `cursor = max` means "nothing yet, just go live" —
|
||||
/// which is a valid, empty replay rather than a future cursor.
|
||||
pub fn plan_cursor(cursor: Option<i64>, bounds: Option<(i64, i64)>) -> CursorPlan {
|
||||
let cursor = match cursor {
|
||||
None => return CursorPlan::LiveOnly,
|
||||
Some(c) => c,
|
||||
};
|
||||
if cursor < 0 {
|
||||
return CursorPlan::FutureCursor {
|
||||
message: format!("cursor {cursor} is negative"),
|
||||
};
|
||||
}
|
||||
let (min, max) = match bounds {
|
||||
// An empty log accepts only cursor 0 ("give me everything, there is
|
||||
// nothing"). Anything else refers to an event we never had.
|
||||
None => {
|
||||
return if cursor == 0 {
|
||||
CursorPlan::Replay { from: 0 }
|
||||
} else {
|
||||
CursorPlan::FutureCursor {
|
||||
message: format!("cursor {cursor} is ahead of an empty log"),
|
||||
}
|
||||
};
|
||||
}
|
||||
Some(b) => b,
|
||||
};
|
||||
if cursor > max {
|
||||
return CursorPlan::FutureCursor {
|
||||
message: format!("cursor {cursor} is ahead of the latest event {max}"),
|
||||
};
|
||||
}
|
||||
// `cursor >= min - 1` means the next event the client wants (cursor + 1)
|
||||
// is still on disk. Below that, rows have been pruned and the client has
|
||||
// a hole it can never fill.
|
||||
if cursor < min - 1 {
|
||||
return CursorPlan::OutdatedCursor {
|
||||
from: min - 1,
|
||||
message: format!(
|
||||
"cursor {cursor} predates the oldest retained event {min}; \
|
||||
resuming from {min} — events {}..{} are gone",
|
||||
cursor + 1,
|
||||
min - 1
|
||||
),
|
||||
};
|
||||
}
|
||||
CursorPlan::Replay { from: cursor }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dag_cbor::{decode, decode_one};
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
fn sample_event() -> FirehoseEvent {
|
||||
let commit = cid_for_cbor(b"commit block").unwrap();
|
||||
let value = cid_for_cbor(b"record value").unwrap();
|
||||
FirehoseEvent {
|
||||
seq: 7,
|
||||
did: "did:plc:alice".into(),
|
||||
rev: "3lxxxxxxxx2".into(),
|
||||
since: Some("3lxxxxxxxx1".into()),
|
||||
commit,
|
||||
blocks: build_blocks_car(
|
||||
commit,
|
||||
b"commit block",
|
||||
&[(value, b"record value".to_vec())],
|
||||
),
|
||||
ops: vec![RepoOp::create("app.twi.post", "3lrkey", value)],
|
||||
time: DateTime::parse_from_rfc3339("2026-09-10T12:00:00.123456Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc),
|
||||
}
|
||||
}
|
||||
|
||||
// -- frame encoding ----------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn commit_frame_header_then_body() {
|
||||
let ev = sample_event();
|
||||
let bytes = ev.to_frame();
|
||||
let (header, used) = decode_one(&bytes).unwrap();
|
||||
assert_eq!(header.get("op").and_then(Value::as_i64), Some(1));
|
||||
assert_eq!(header.get("t").and_then(Value::as_str), Some("#commit"));
|
||||
// Everything after the header is exactly one more value — no padding,
|
||||
// no length prefix.
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
assert_eq!(body.get("seq").and_then(Value::as_i64), Some(7));
|
||||
assert_eq!(
|
||||
body.get("repo").and_then(Value::as_str),
|
||||
Some("did:plc:alice")
|
||||
);
|
||||
assert_eq!(body.get("rev").and_then(Value::as_str), Some("3lxxxxxxxx2"));
|
||||
assert_eq!(
|
||||
body.get("since").and_then(Value::as_str),
|
||||
Some("3lxxxxxxxx1")
|
||||
);
|
||||
assert_eq!(body.get("rebase").and_then(Value::as_bool), Some(false));
|
||||
assert_eq!(body.get("tooBig").and_then(Value::as_bool), Some(false));
|
||||
assert!(body.get("blobs").unwrap().as_array().unwrap().is_empty());
|
||||
assert_eq!(
|
||||
body.get("time").and_then(Value::as_str),
|
||||
Some("2026-09-10T12:00:00.123456Z")
|
||||
);
|
||||
}
|
||||
|
||||
/// The header's exact bytes, locked in. The AppView builds its reader
|
||||
/// against this, and canonical key ordering means `t` precedes `op` —
|
||||
/// which is easy to get wrong and produces a frame a strict DAG-CBOR
|
||||
/// decoder rejects.
|
||||
#[test]
|
||||
fn commit_header_has_the_exact_expected_bytes() {
|
||||
let bytes = sample_event().to_frame();
|
||||
assert_eq!(
|
||||
&bytes[..15],
|
||||
&[
|
||||
0xA2, // map(2)
|
||||
0x61, b't', // text(1) "t"
|
||||
0x67, b'#', b'c', b'o', b'm', b'm', b'i', b't', // text(7) "#commit"
|
||||
0x62, b'o', b'p', // text(2) "op"
|
||||
0x01, // 1
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_and_error_headers_have_the_exact_expected_bytes() {
|
||||
let info = encode_info_frame("OutdatedCursor", "x");
|
||||
assert_eq!(
|
||||
&info[..12],
|
||||
&[0xA2, 0x61, b't', 0x65, b'#', b'i', b'n', b'f', b'o', 0x62, b'o', b'p']
|
||||
);
|
||||
assert_eq!(info[12], 0x01);
|
||||
// Error: map(1) { "op": -1 }. -1 is major type 1 with argument 0.
|
||||
let err = encode_error_frame("FutureCursor", "x");
|
||||
assert_eq!(&err[..5], &[0xA1, 0x62, b'o', b'p', 0x20]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commit_cid_is_a_tag_42_link_not_a_string() {
|
||||
// The whole point of the hand-rolled encoder: `ciborium` would have
|
||||
// written this as a string or a newtype struct.
|
||||
let ev = sample_event();
|
||||
let bytes = ev.to_frame();
|
||||
let (_h, used) = decode_one(&bytes).unwrap();
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
assert_eq!(body.get("commit").and_then(Value::as_link), Some(&ev.commit));
|
||||
// Locate the tag bytes directly, to prove it is not the decoder being
|
||||
// generous: 0xD8 0x2A is tag(42).
|
||||
assert!(
|
||||
bytes.windows(2).any(|w| w == [0xD8, 0x2A]),
|
||||
"frame must contain a tag-42 head"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ops_encode_action_path_and_link() {
|
||||
let ev = sample_event();
|
||||
let bytes = ev.to_frame();
|
||||
let (_h, used) = decode_one(&bytes).unwrap();
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
let ops = body.get("ops").unwrap().as_array().unwrap();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(ops[0].get("action").and_then(Value::as_str), Some("create"));
|
||||
assert_eq!(
|
||||
ops[0].get("path").and_then(Value::as_str),
|
||||
Some("app.twi.post/3lrkey")
|
||||
);
|
||||
assert!(ops[0].get("cid").unwrap().as_link().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_op_has_a_null_cid() {
|
||||
let mut ev = sample_event();
|
||||
ev.ops = vec![RepoOp::delete("app.bsky.feed.like", "3lrkey")];
|
||||
let bytes = ev.to_frame();
|
||||
let (_h, used) = decode_one(&bytes).unwrap();
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
let ops = body.get("ops").unwrap().as_array().unwrap();
|
||||
assert_eq!(ops[0].get("action").and_then(Value::as_str), Some("delete"));
|
||||
assert!(ops[0].get("cid").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_commit_has_a_null_since() {
|
||||
let mut ev = sample_event();
|
||||
ev.since = None;
|
||||
let bytes = ev.to_frame();
|
||||
let (_h, used) = decode_one(&bytes).unwrap();
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
assert!(body.get("since").unwrap().is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blocks_is_a_parsable_car_rooted_at_the_commit() {
|
||||
let ev = sample_event();
|
||||
let bytes = ev.to_frame();
|
||||
let (_h, used) = decode_one(&bytes).unwrap();
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
let car = body.get("blocks").and_then(Value::as_bytes).unwrap();
|
||||
let (header, blocks) = crate::car::parse(car).unwrap();
|
||||
assert_eq!(header.version, 1);
|
||||
assert_eq!(header.roots, vec![ev.commit]);
|
||||
assert_eq!(blocks.len(), 2, "commit block + one new record block");
|
||||
assert_eq!(blocks[0].cid, ev.commit);
|
||||
assert_eq!(blocks[0].data, b"commit block");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn info_frame_round_trips() {
|
||||
let bytes = encode_info_frame("OutdatedCursor", "resuming from 12");
|
||||
let (header, used) = decode_one(&bytes).unwrap();
|
||||
assert_eq!(header.get("op").and_then(Value::as_i64), Some(1));
|
||||
assert_eq!(header.get("t").and_then(Value::as_str), Some("#info"));
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
assert_eq!(
|
||||
body.get("name").and_then(Value::as_str),
|
||||
Some("OutdatedCursor")
|
||||
);
|
||||
assert_eq!(
|
||||
body.get("message").and_then(Value::as_str),
|
||||
Some("resuming from 12")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_frame_uses_op_minus_one_and_carries_no_t() {
|
||||
let bytes = encode_error_frame("FutureCursor", "cursor 99 is ahead");
|
||||
let (header, used) = decode_one(&bytes).unwrap();
|
||||
assert_eq!(header.get("op").and_then(Value::as_i64), Some(-1));
|
||||
assert!(
|
||||
header.get("t").is_none(),
|
||||
"an error header carries op only"
|
||||
);
|
||||
let body = decode(&bytes[used..]).unwrap();
|
||||
assert_eq!(
|
||||
body.get("error").and_then(Value::as_str),
|
||||
Some("FutureCursor")
|
||||
);
|
||||
assert_eq!(
|
||||
body.get("message").and_then(Value::as_str),
|
||||
Some("cursor 99 is ahead")
|
||||
);
|
||||
}
|
||||
|
||||
// -- ops derivation ----------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn put_picks_create_or_update_from_prior_existence() {
|
||||
let cid = cid_for_cbor(b"v").unwrap();
|
||||
assert_eq!(
|
||||
RepoOp::put("c", "r", cid, false).action,
|
||||
RepoOpAction::Create
|
||||
);
|
||||
assert_eq!(
|
||||
RepoOp::put("c", "r", cid, true).action,
|
||||
RepoOpAction::Update
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_path_is_collection_slash_rkey() {
|
||||
let cid = cid_for_cbor(b"v").unwrap();
|
||||
assert_eq!(
|
||||
RepoOp::create("app.bsky.feed.repost", "3lk", cid).path,
|
||||
"app.bsky.feed.repost/3lk"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_json_round_trips_through_the_jsonb_shape() {
|
||||
let cid = cid_for_cbor(b"v").unwrap();
|
||||
for op in [
|
||||
RepoOp::create("app.twi.post", "a", cid),
|
||||
RepoOp::update("app.bsky.actor.profile", "self", cid),
|
||||
RepoOp::delete("app.bsky.graph.follow", "b"),
|
||||
] {
|
||||
let back = RepoOp::from_json(&op.to_json()).unwrap();
|
||||
assert_eq!(back, op);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn op_json_rejects_an_unknown_action() {
|
||||
let v = json!({"action": "rebase", "path": "a/b", "cid": null});
|
||||
assert!(RepoOp::from_json(&v).is_err());
|
||||
}
|
||||
|
||||
// -- cursor edge cases -------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn no_cursor_is_live_only() {
|
||||
assert_eq!(plan_cursor(None, Some((1, 10))), CursorPlan::LiveOnly);
|
||||
assert_eq!(plan_cursor(None, None), CursorPlan::LiveOnly);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_zero_replays_everything() {
|
||||
assert_eq!(
|
||||
plan_cursor(Some(0), Some((1, 10))),
|
||||
CursorPlan::Replay { from: 0 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_at_the_head_is_an_empty_replay_not_an_error() {
|
||||
// The client is fully caught up. Replaying `seq > 10` yields nothing
|
||||
// and it goes straight live — that must not be a FutureCursor.
|
||||
assert_eq!(
|
||||
plan_cursor(Some(10), Some((1, 10))),
|
||||
CursorPlan::Replay { from: 10 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_past_the_head_is_a_future_cursor() {
|
||||
assert!(matches!(
|
||||
plan_cursor(Some(11), Some((1, 10))),
|
||||
CursorPlan::FutureCursor { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn negative_cursor_is_a_future_cursor() {
|
||||
assert!(matches!(
|
||||
plan_cursor(Some(-1), Some((1, 10))),
|
||||
CursorPlan::FutureCursor { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_log_accepts_zero_and_refuses_anything_else() {
|
||||
assert_eq!(plan_cursor(Some(0), None), CursorPlan::Replay { from: 0 });
|
||||
assert!(matches!(
|
||||
plan_cursor(Some(1), None),
|
||||
CursorPlan::FutureCursor { .. }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_exactly_one_below_the_oldest_row_is_still_exact() {
|
||||
// min = 5 means seq 5 is the oldest surviving event. A client whose
|
||||
// cursor is 4 wants 5 next — nothing is missing.
|
||||
assert_eq!(
|
||||
plan_cursor(Some(4), Some((5, 10))),
|
||||
CursorPlan::Replay { from: 4 }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cursor_below_the_pruned_window_is_outdated() {
|
||||
match plan_cursor(Some(2), Some((5, 10))) {
|
||||
CursorPlan::OutdatedCursor { from, message } => {
|
||||
assert_eq!(from, 4, "resume so the next delivered event is 5");
|
||||
assert!(message.contains('5'), "message should name the gap: {message}");
|
||||
}
|
||||
other => panic!("expected OutdatedCursor, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,684 @@
|
||||
//! Invite codes: minting, listing, and the single redeem operation that
|
||||
//! `com.atproto.server.createAccount` calls.
|
||||
//!
|
||||
//! # Why this exists
|
||||
//!
|
||||
//! `createAccount` had no gate at all. That was fine while the PDS only
|
||||
//! answered on `127.0.0.1:2583`; it is not fine on a public name, because
|
||||
//! each accepted account allocates a repo head, a server-held key pair, an
|
||||
//! MST that grows with every write, and a stream of firehose events every
|
||||
//! subscribed AppView is obliged to index. Invite codes are the smallest
|
||||
//! gate that turns "anyone with curl" into "anyone the operator handed a
|
||||
//! string to".
|
||||
//!
|
||||
//! The gate is off by default (`PDS_INVITE_REQUIRED`, see
|
||||
//! [`at_shared::config`]) so the existing test suites and dev instances
|
||||
//! keep creating throwaway accounts; `main` warns loudly on every boot
|
||||
//! where it is off.
|
||||
//!
|
||||
//! # Where codes come from
|
||||
//!
|
||||
//! Nowhere over HTTP. Minting lives in the `pds-server invite` subcommand
|
||||
//! ([`run_cli`]), which the operator runs on the box. Adding a
|
||||
//! `createInviteCode` endpoint would mean the thing that guards
|
||||
//! registration is itself reachable by whoever can reach registration —
|
||||
//! at which point it guards nothing, and the only question left is
|
||||
//! whether *its* auth has a hole. A subcommand has no attack surface to
|
||||
//! get wrong.
|
||||
//!
|
||||
//! # The one interesting piece of code in here
|
||||
//!
|
||||
//! [`redeem`] is a single conditional `UPDATE … RETURNING`, not a
|
||||
//! `SELECT`-then-`UPDATE`. See its docs and
|
||||
//! `migrations/pds/0004_invite_codes.sql` for why that distinction is the
|
||||
//! whole feature.
|
||||
|
||||
use rand::rngs::OsRng;
|
||||
use rand::RngCore;
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
|
||||
/// Alphabet the codes are drawn from: Crockford base32, lowercased.
|
||||
///
|
||||
/// Exactly 32 symbols, which is the property that matters — it lets each
|
||||
/// character consume exactly 5 bits of entropy with no modulo bias, so
|
||||
/// every code in the space is equally likely. A 31- or 36-character
|
||||
/// "human friendly" alphabet would need rejection sampling to say the
|
||||
/// same thing, and the usual `byte % len` shortcut would quietly make
|
||||
/// some characters more probable than others.
|
||||
///
|
||||
/// The excluded letters are Crockford's: `i`, `l`, `o` and `u`. The
|
||||
/// first three are the ones people mistype as `1`, `1` and `0` when
|
||||
/// copying a code out of a chat message; `u` is dropped so a random draw
|
||||
/// cannot spell something the operator has to apologise for.
|
||||
const CODE_ALPHABET: &[u8; 32] = b"0123456789abcdefghjkmnpqrstvwxyz";
|
||||
|
||||
/// Characters per group, and groups per code. Two groups of five is
|
||||
/// 50 bits of entropy — far past anything an online guesser can reach
|
||||
/// against a database round-trip per attempt, and short enough to read
|
||||
/// aloud.
|
||||
const GROUP_LEN: usize = 5;
|
||||
const GROUPS: usize = 2;
|
||||
|
||||
/// Fixed prefix so a code is recognisable as one when it turns up out of
|
||||
/// context (a support ticket, a pasted log line) and so it cannot be
|
||||
/// confused with a handle or a DID.
|
||||
const CODE_PREFIX: &str = "mt";
|
||||
|
||||
/// Why a redemption was refused.
|
||||
///
|
||||
/// Deliberately coarse. The route maps [`RedeemError::Invalid`] to a
|
||||
/// single `400 InvalidInviteCode` with one fixed message, so an
|
||||
/// unauthenticated caller cannot use the error text to distinguish
|
||||
/// "no such code" from "that code exists but is used up" — which would
|
||||
/// turn the endpoint into an oracle for probing the code space.
|
||||
#[derive(Debug)]
|
||||
pub enum RedeemError {
|
||||
/// Unknown, disabled, or already at its use limit. One variant on
|
||||
/// purpose: see the type docs.
|
||||
Invalid,
|
||||
/// The database itself failed. Distinct from [`RedeemError::Invalid`]
|
||||
/// because this is a `500`, not a `400` — refusing a legitimate code
|
||||
/// because Postgres hiccuped would be a lie to the user.
|
||||
Db(sqlx::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RedeemError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
RedeemError::Invalid => write!(f, "invite code is not valid"),
|
||||
RedeemError::Db(e) => write!(f, "invite lookup failed: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalise a client-supplied code into the form stored in the table.
|
||||
///
|
||||
/// Trims surrounding whitespace (people paste codes with a trailing
|
||||
/// newline out of a terminal) and lowercases. Every generated code is
|
||||
/// already lowercase ASCII, so this is an exact normalisation — which is
|
||||
/// what lets [`redeem`] look the code up with a plain `code = $1` and
|
||||
/// hit the primary-key index, instead of `lower(code) = $1`, which
|
||||
/// would force a sequential scan on the one query that runs per
|
||||
/// registration attempt.
|
||||
///
|
||||
/// Returns `None` for a code that is empty after trimming, so "field
|
||||
/// present but blank" and "field absent" reach the route as the same
|
||||
/// case.
|
||||
pub fn normalize(raw: &str) -> Option<String> {
|
||||
let trimmed = raw.trim();
|
||||
if trimmed.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(trimmed.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
/// Generate one cryptographically random invite code, e.g.
|
||||
/// `mt-7k3qw-z9d2m`.
|
||||
///
|
||||
/// Randomness comes from [`OsRng`] — the same source
|
||||
/// `keys::generate_user_keys` and `password::hash_password` already use
|
||||
/// in this crate, i.e. the OS CSPRNG, never a seeded or thread-local
|
||||
/// generator. A code is a bearer credential for creating an account on
|
||||
/// this server; a predictable one is the same bug as a predictable
|
||||
/// password-reset token.
|
||||
///
|
||||
/// Entropy: [`GROUPS`] × [`GROUP_LEN`] characters × 5 bits = 50 bits.
|
||||
pub fn generate_code() -> String {
|
||||
let total = GROUPS * GROUP_LEN;
|
||||
let mut bytes = vec![0u8; total];
|
||||
OsRng.fill_bytes(&mut bytes);
|
||||
|
||||
let mut out = String::with_capacity(CODE_PREFIX.len() + total + GROUPS);
|
||||
out.push_str(CODE_PREFIX);
|
||||
for chunk in bytes.chunks(GROUP_LEN) {
|
||||
out.push('-');
|
||||
for b in chunk {
|
||||
// Take the low 5 bits of a uniformly random byte. The
|
||||
// alphabet is exactly 32 symbols, so this is a bijection
|
||||
// from 5 bits onto it — no bias, no rejection loop.
|
||||
out.push(CODE_ALPHABET[(*b & 0b0001_1111) as usize] as char);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Consume one use of `code` on behalf of the account `did` / `handle`.
|
||||
///
|
||||
/// **This must be called with the same transaction that creates the
|
||||
/// account.** The whole point is that a code is spent if and only if an
|
||||
/// account was actually created: if the caller's transaction rolls back
|
||||
/// — handle taken, key generation failed, anything — the counter goes
|
||||
/// back with it and the code is still redeemable. `create_account`
|
||||
/// therefore begins its transaction, redeems here, inserts `users` and
|
||||
/// `repos`, and only then commits.
|
||||
///
|
||||
/// # The race, and why there isn't one
|
||||
///
|
||||
/// The tempting implementation is: `SELECT used_count, max_uses …`,
|
||||
/// compare in Rust, then `UPDATE`. That is a check-then-act. Two
|
||||
/// registrations arriving together on a code with one use left both read
|
||||
/// `used_count = 0`, both conclude they may proceed, and both write
|
||||
/// `used_count = 1`. Two accounts, one use — and the row afterwards
|
||||
/// claims it was redeemed once, so nothing even shows up as wrong.
|
||||
///
|
||||
/// Instead the check *is* the write:
|
||||
///
|
||||
/// ```sql
|
||||
/// UPDATE invite_codes
|
||||
/// SET used_count = used_count + 1
|
||||
/// WHERE code = $1 AND NOT disabled AND used_count < max_uses
|
||||
/// RETURNING used_count, max_uses
|
||||
/// ```
|
||||
///
|
||||
/// Postgres serialises the two statements on the row lock. The loser
|
||||
/// blocks until the winner commits, and then — this is the part that
|
||||
/// makes it work — does *not* continue with its old snapshot: it
|
||||
/// re-fetches the committed row and re-evaluates the `WHERE` clause
|
||||
/// against it (EvalPlanQual). `used_count` is now `1`, the predicate is
|
||||
/// false, the row is dropped from the update set, and the statement
|
||||
/// affects zero rows. Zero rows is the rejection. This function never
|
||||
/// forms an opinion about validity that could be stale by the time it
|
||||
/// acts on it, because it never looks before it writes.
|
||||
///
|
||||
/// The `invite_code_uses` insert that follows is inside the same
|
||||
/// transaction and the same row lock, so the counter and the audit rows
|
||||
/// cannot drift apart.
|
||||
pub async fn redeem(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
code: &str,
|
||||
did: &str,
|
||||
handle: &str,
|
||||
) -> Result<(), RedeemError> {
|
||||
let normalized = match normalize(code) {
|
||||
Some(c) => c,
|
||||
None => return Err(RedeemError::Invalid),
|
||||
};
|
||||
|
||||
// One statement, and its row count is the verdict.
|
||||
let claimed: Option<(i32, i32)> = sqlx::query_as(
|
||||
r#"UPDATE invite_codes
|
||||
SET used_count = used_count + 1
|
||||
WHERE code = $1
|
||||
AND NOT disabled
|
||||
AND used_count < max_uses
|
||||
RETURNING used_count, max_uses"#,
|
||||
)
|
||||
.bind(&normalized)
|
||||
.fetch_optional(&mut **tx)
|
||||
.await
|
||||
.map_err(RedeemError::Db)?;
|
||||
|
||||
if claimed.is_none() {
|
||||
return Err(RedeemError::Invalid);
|
||||
}
|
||||
|
||||
// Audit trail: which account this code produced. Same transaction,
|
||||
// so it lands exactly when the counter increment does.
|
||||
//
|
||||
// The `(code, did)` primary key makes a duplicate impossible; a
|
||||
// conflict here would mean the same DID redeemed the same code
|
||||
// twice in one registration, which cannot happen but would corrupt
|
||||
// the counter/uses agreement if it did — so let it be an error
|
||||
// rather than silently ignoring it.
|
||||
sqlx::query(
|
||||
r#"INSERT INTO invite_code_uses (code, did, handle)
|
||||
VALUES ($1, $2, $3)"#,
|
||||
)
|
||||
.bind(&normalized)
|
||||
.bind(did)
|
||||
.bind(handle)
|
||||
.execute(&mut **tx)
|
||||
.await
|
||||
.map_err(RedeemError::Db)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// =====================================================
|
||||
// CLI: `pds-server invite …`
|
||||
// =====================================================
|
||||
|
||||
/// One row of `invite list`, and the shape `create` hands back.
|
||||
#[derive(Debug)]
|
||||
pub struct InviteRow {
|
||||
pub code: String,
|
||||
pub max_uses: i32,
|
||||
pub used_count: i32,
|
||||
pub disabled: bool,
|
||||
pub note: Option<String>,
|
||||
pub created_at: chrono::DateTime<chrono::Utc>,
|
||||
}
|
||||
|
||||
/// What a parsed `invite` command line asks for.
|
||||
///
|
||||
/// Parsed out of `std::env::args` by hand. The workspace has no
|
||||
/// argument-parsing dependency and this subcommand is not worth adding
|
||||
/// one for: three verbs, three flags, and a hand-rolled parser that is
|
||||
/// small enough to unit-test exhaustively (which it is, below) beats a
|
||||
/// derive macro plus a new crate in the dependency tree of a server
|
||||
/// binary.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum InviteCommand {
|
||||
/// `invite create [--count N] [--uses N] [--note TEXT]`
|
||||
Create {
|
||||
count: u32,
|
||||
uses: i32,
|
||||
note: Option<String>,
|
||||
},
|
||||
/// `invite list [--all]` — without `--all`, spent and disabled codes
|
||||
/// are hidden, because the question the operator almost always has
|
||||
/// is "what can I still hand out".
|
||||
List { all: bool },
|
||||
/// `invite disable <code>` — stop honouring a code without losing
|
||||
/// the record of which accounts it already created.
|
||||
Disable { code: String },
|
||||
}
|
||||
|
||||
/// Usage text. Printed for `invite help`, and for anything that fails to
|
||||
/// parse.
|
||||
pub const INVITE_USAGE: &str = "\
|
||||
usage: pds-server invite <command>
|
||||
|
||||
create [--count N] [--uses N] [--note TEXT]
|
||||
Mint N codes (default 1), each good for `--uses` accounts
|
||||
(default 1). Prints one code per line and nothing else, so the
|
||||
output can be piped or pasted directly.
|
||||
|
||||
list [--all]
|
||||
Show codes that can still be redeemed. --all includes spent and
|
||||
disabled ones.
|
||||
|
||||
disable <code>
|
||||
Stop honouring a code. The record of accounts it already created
|
||||
is kept.
|
||||
|
||||
The database is the one named by DATABASE_URL_PDS (read from .env like
|
||||
the server does). Codes are only meaningful while PDS_INVITE_REQUIRED
|
||||
is true.";
|
||||
|
||||
/// Parse the arguments after the `invite` verb.
|
||||
///
|
||||
/// Returns `Err(message)` for anything malformed; the caller prints the
|
||||
/// message plus [`INVITE_USAGE`] and exits non-zero. Unknown flags are
|
||||
/// an error rather than being ignored — a typo'd `--uses` that silently
|
||||
/// became `1` would hand out the wrong codes and the operator would only
|
||||
/// find out when the second person to use one got a `400`.
|
||||
pub fn parse_invite_args(args: &[String]) -> Result<InviteCommand, String> {
|
||||
let verb = args
|
||||
.first()
|
||||
.map(|s| s.as_str())
|
||||
.ok_or_else(|| "missing invite command".to_string())?;
|
||||
let rest = &args[1..];
|
||||
|
||||
match verb {
|
||||
"create" => {
|
||||
let mut count: u32 = 1;
|
||||
let mut uses: i32 = 1;
|
||||
let mut note: Option<String> = None;
|
||||
let mut i = 0;
|
||||
while i < rest.len() {
|
||||
match rest[i].as_str() {
|
||||
"--count" => {
|
||||
let v = rest
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| "--count needs a value".to_string())?;
|
||||
count = v
|
||||
.parse()
|
||||
.map_err(|_| format!("--count: not a number: {v}"))?;
|
||||
if count == 0 {
|
||||
return Err("--count must be at least 1".to_string());
|
||||
}
|
||||
i += 2;
|
||||
}
|
||||
"--uses" => {
|
||||
let v = rest
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| "--uses needs a value".to_string())?;
|
||||
uses = v
|
||||
.parse()
|
||||
.map_err(|_| format!("--uses: not a number: {v}"))?;
|
||||
// Mirrors the table's CHECK (max_uses > 0). Caught
|
||||
// here so the operator gets a sentence instead of a
|
||||
// constraint-violation dump.
|
||||
if uses < 1 {
|
||||
return Err("--uses must be at least 1".to_string());
|
||||
}
|
||||
i += 2;
|
||||
}
|
||||
"--note" => {
|
||||
let v = rest
|
||||
.get(i + 1)
|
||||
.ok_or_else(|| "--note needs a value".to_string())?;
|
||||
note = Some(v.clone());
|
||||
i += 2;
|
||||
}
|
||||
other => return Err(format!("unknown option for `create`: {other}")),
|
||||
}
|
||||
}
|
||||
Ok(InviteCommand::Create { count, uses, note })
|
||||
}
|
||||
"list" => {
|
||||
let mut all = false;
|
||||
for a in rest {
|
||||
match a.as_str() {
|
||||
"--all" => all = true,
|
||||
other => return Err(format!("unknown option for `list`: {other}")),
|
||||
}
|
||||
}
|
||||
Ok(InviteCommand::List { all })
|
||||
}
|
||||
"disable" => {
|
||||
let code = rest
|
||||
.first()
|
||||
.ok_or_else(|| "disable needs a code".to_string())?;
|
||||
if rest.len() > 1 {
|
||||
return Err("disable takes exactly one code".to_string());
|
||||
}
|
||||
let code = normalize(code).ok_or_else(|| "disable needs a code".to_string())?;
|
||||
Ok(InviteCommand::Disable { code })
|
||||
}
|
||||
other => Err(format!("unknown invite command: {other}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Insert `count` freshly generated codes, each good for `uses`
|
||||
/// accounts.
|
||||
///
|
||||
/// Retries on a primary-key collision. With 50 bits per code a
|
||||
/// collision is not something that will happen, but "not something that
|
||||
/// will happen" is exactly the class of event that turns into a
|
||||
/// confusing `duplicate key` traceback at 2am, and the retry costs three
|
||||
/// lines.
|
||||
pub async fn create_codes(
|
||||
db: &PgPool,
|
||||
count: u32,
|
||||
uses: i32,
|
||||
note: Option<&str>,
|
||||
) -> anyhow::Result<Vec<String>> {
|
||||
let mut out = Vec::with_capacity(count as usize);
|
||||
for _ in 0..count {
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
let code = generate_code();
|
||||
let inserted = sqlx::query(
|
||||
r#"INSERT INTO invite_codes (code, max_uses, note)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (code) DO NOTHING"#,
|
||||
)
|
||||
.bind(&code)
|
||||
.bind(uses)
|
||||
.bind(note)
|
||||
.execute(db)
|
||||
.await?
|
||||
.rows_affected();
|
||||
if inserted == 1 {
|
||||
out.push(code);
|
||||
break;
|
||||
}
|
||||
attempt += 1;
|
||||
if attempt >= 5 {
|
||||
anyhow::bail!("could not find a free invite code after 5 attempts");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Read back codes for `invite list`.
|
||||
pub async fn list_codes(db: &PgPool, all: bool) -> anyhow::Result<Vec<InviteRow>> {
|
||||
// Two statements rather than one with a `$1`-toggled predicate:
|
||||
// the redeemable filter is exactly the redeem query's `WHERE`
|
||||
// clause, and keeping it spelled the same way makes it obvious that
|
||||
// `list` shows what `redeem` would accept.
|
||||
let sql = if all {
|
||||
r#"SELECT code, max_uses, used_count, disabled, note, created_at
|
||||
FROM invite_codes
|
||||
ORDER BY created_at DESC"#
|
||||
} else {
|
||||
r#"SELECT code, max_uses, used_count, disabled, note, created_at
|
||||
FROM invite_codes
|
||||
WHERE NOT disabled AND used_count < max_uses
|
||||
ORDER BY created_at DESC"#
|
||||
};
|
||||
let rows: Vec<(String, i32, i32, bool, Option<String>, chrono::DateTime<chrono::Utc>)> =
|
||||
sqlx::query_as(sql).fetch_all(db).await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(
|
||||
|(code, max_uses, used_count, disabled, note, created_at)| InviteRow {
|
||||
code,
|
||||
max_uses,
|
||||
used_count,
|
||||
disabled,
|
||||
note,
|
||||
created_at,
|
||||
},
|
||||
)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Flip `disabled` on one code. Returns `false` if there is no such
|
||||
/// code, so the CLI can say so instead of reporting a successful no-op.
|
||||
pub async fn disable_code(db: &PgPool, code: &str) -> anyhow::Result<bool> {
|
||||
let n = sqlx::query("UPDATE invite_codes SET disabled = TRUE WHERE code = $1")
|
||||
.bind(code)
|
||||
.execute(db)
|
||||
.await?
|
||||
.rows_affected();
|
||||
Ok(n == 1)
|
||||
}
|
||||
|
||||
/// Run the `invite` subcommand end to end: parse, connect, act, print.
|
||||
///
|
||||
/// Connects with the same `DATABASE_URL_PDS` and runs the same
|
||||
/// migrations as [`crate::main`], so `invite create` works on a fresh
|
||||
/// checkout before the server has ever been started — otherwise the
|
||||
/// first thing an operator does after deploying would fail with
|
||||
/// "relation invite_codes does not exist".
|
||||
pub async fn run_cli(args: &[String]) -> anyhow::Result<()> {
|
||||
if matches!(args.first().map(|s| s.as_str()), None | Some("help") | Some("-h") | Some("--help"))
|
||||
{
|
||||
println!("{INVITE_USAGE}");
|
||||
return Ok(());
|
||||
}
|
||||
let cmd = match parse_invite_args(args) {
|
||||
Ok(c) => c,
|
||||
Err(msg) => {
|
||||
eprintln!("pds-server invite: {msg}\n\n{INVITE_USAGE}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
|
||||
let cfg = at_shared::config::AppConfig::from_env()?;
|
||||
let db = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.acquire_timeout(std::time::Duration::from_secs(10))
|
||||
.connect(&cfg.database_url_pds)
|
||||
.await?;
|
||||
sqlx::migrate!("../../migrations/pds").run(&db).await?;
|
||||
|
||||
match cmd {
|
||||
InviteCommand::Create { count, uses, note } => {
|
||||
let codes = create_codes(&db, count, uses, note.as_deref()).await?;
|
||||
// Bare codes, one per line, nothing else on stdout — the
|
||||
// operator pipes this into a message or a file. Anything
|
||||
// decorative here would have to be stripped by hand.
|
||||
for c in &codes {
|
||||
println!("{c}");
|
||||
}
|
||||
if !cfg.pds_invite_required {
|
||||
// Not an error: minting codes before flipping the switch
|
||||
// is the correct order of operations. But an operator
|
||||
// who thinks they have just closed registration should
|
||||
// find out now.
|
||||
eprintln!(
|
||||
"note: PDS_INVITE_REQUIRED is not true — createAccount currently \
|
||||
accepts requests without any code."
|
||||
);
|
||||
}
|
||||
}
|
||||
InviteCommand::List { all } => {
|
||||
let rows = list_codes(&db, all).await?;
|
||||
if rows.is_empty() {
|
||||
eprintln!("no invite codes");
|
||||
}
|
||||
for r in rows {
|
||||
let state = if r.disabled {
|
||||
"disabled"
|
||||
} else if r.used_count >= r.max_uses {
|
||||
"spent"
|
||||
} else {
|
||||
"open"
|
||||
};
|
||||
println!(
|
||||
"{} {}/{} {} {} {}",
|
||||
r.code,
|
||||
r.used_count,
|
||||
r.max_uses,
|
||||
state,
|
||||
r.created_at.format("%Y-%m-%dT%H:%M:%SZ"),
|
||||
r.note.as_deref().unwrap_or("")
|
||||
);
|
||||
}
|
||||
}
|
||||
InviteCommand::Disable { code } => {
|
||||
if disable_code(&db, &code).await? {
|
||||
println!("disabled {code}");
|
||||
} else {
|
||||
eprintln!("no such invite code: {code}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// -- tests -------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn generated_codes_use_only_the_safe_alphabet() {
|
||||
let code = generate_code();
|
||||
// `mt-xxxxx-xxxxx`
|
||||
assert!(code.starts_with("mt-"), "code = {code}");
|
||||
let groups: Vec<&str> = code.split('-').collect();
|
||||
assert_eq!(groups.len(), GROUPS + 1, "code = {code}");
|
||||
assert_eq!(groups[0], CODE_PREFIX);
|
||||
for g in &groups[1..] {
|
||||
assert_eq!(g.len(), GROUP_LEN, "group {g} in {code}");
|
||||
for ch in g.chars() {
|
||||
assert!(
|
||||
CODE_ALPHABET.contains(&(ch as u8)),
|
||||
"character {ch:?} in {code} is outside the alphabet"
|
||||
);
|
||||
}
|
||||
}
|
||||
// The letters people mistype must never appear.
|
||||
for bad in ['i', 'l', 'o', 'u'] {
|
||||
assert!(!code[3..].contains(bad), "{code} contains {bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_codes_do_not_repeat() {
|
||||
// Not a randomness test — a 1000-draw collision would mean the
|
||||
// generator is returning a constant or reusing a seeded RNG,
|
||||
// which is the failure mode that actually happens when someone
|
||||
// swaps `OsRng` for `thread_rng` with a fixed seed in a test
|
||||
// helper.
|
||||
let mut seen = HashSet::new();
|
||||
for _ in 0..1000 {
|
||||
assert!(seen.insert(generate_code()), "duplicate code in 1000 draws");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_trims_and_lowercases() {
|
||||
assert_eq!(normalize(" MT-ABCDE-FGHJK \n").as_deref(), Some("mt-abcde-fghjk"));
|
||||
assert_eq!(normalize("mt-abcde-fghjk").as_deref(), Some("mt-abcde-fghjk"));
|
||||
// Absent and blank must be indistinguishable to the route.
|
||||
assert_eq!(normalize(""), None);
|
||||
assert_eq!(normalize(" "), None);
|
||||
assert_eq!(normalize("\t\n"), None);
|
||||
}
|
||||
|
||||
fn args(v: &[&str]) -> Vec<String> {
|
||||
v.iter().map(|s| s.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_create_with_defaults_and_flags() {
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["create"])).unwrap(),
|
||||
InviteCommand::Create {
|
||||
count: 1,
|
||||
uses: 1,
|
||||
note: None
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["create", "--count", "5", "--uses", "1"])).unwrap(),
|
||||
InviteCommand::Create {
|
||||
count: 5,
|
||||
uses: 1,
|
||||
note: None
|
||||
}
|
||||
);
|
||||
// Order must not matter, and --note takes the next argument
|
||||
// verbatim (spaces included).
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["create", "--note", "meetup 2026", "--uses", "3"])).unwrap(),
|
||||
InviteCommand::Create {
|
||||
count: 1,
|
||||
uses: 3,
|
||||
note: Some("meetup 2026".to_string())
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_malformed_create_flags() {
|
||||
// A typo'd flag must not be silently ignored — that would hand
|
||||
// out codes with the default limits.
|
||||
assert!(parse_invite_args(&args(&["create", "--use", "3"])).is_err());
|
||||
assert!(parse_invite_args(&args(&["create", "--count"])).is_err());
|
||||
assert!(parse_invite_args(&args(&["create", "--count", "x"])).is_err());
|
||||
assert!(parse_invite_args(&args(&["create", "--count", "0"])).is_err());
|
||||
// max_uses > 0 is a table constraint; catch it before Postgres does.
|
||||
assert!(parse_invite_args(&args(&["create", "--uses", "0"])).is_err());
|
||||
assert!(parse_invite_args(&args(&["create", "--uses", "-2"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_list_and_disable() {
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["list"])).unwrap(),
|
||||
InviteCommand::List { all: false }
|
||||
);
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["list", "--all"])).unwrap(),
|
||||
InviteCommand::List { all: true }
|
||||
);
|
||||
assert!(parse_invite_args(&args(&["list", "--everything"])).is_err());
|
||||
// `disable` normalises the code the same way redeem does, so an
|
||||
// operator pasting a shouted code still disables the right row.
|
||||
assert_eq!(
|
||||
parse_invite_args(&args(&["disable", " MT-ABCDE-FGHJK "])).unwrap(),
|
||||
InviteCommand::Disable {
|
||||
code: "mt-abcde-fghjk".to_string()
|
||||
}
|
||||
);
|
||||
assert!(parse_invite_args(&args(&["disable"])).is_err());
|
||||
assert!(parse_invite_args(&args(&["disable", "a", "b"])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_unknown_verb_and_empty_args() {
|
||||
assert!(parse_invite_args(&args(&[])).is_err());
|
||||
assert!(parse_invite_args(&args(&["destroy"])).is_err());
|
||||
}
|
||||
}
|
||||
@@ -39,9 +39,18 @@ pub fn issue_access_jwt(
|
||||
let now = chrono::Utc::now().timestamp();
|
||||
let exp = now + 3600;
|
||||
let claims = JwtClaims {
|
||||
iss: format!("did:web:{}", cfg.pds_public_url.trim_start_matches("http://").trim_start_matches("https://")),
|
||||
// Same derivation as `describeServer` and `/.well-known/did.json`
|
||||
// (`AppConfig::pds_did`), so a verifier can take `iss`, resolve
|
||||
// the did:web document and arrive at the key this token is
|
||||
// signed with. The previous inline version dropped the
|
||||
// percent-encoding of the port, producing an `iss` that no
|
||||
// did:web resolver could follow.
|
||||
iss: cfg.pds_did(),
|
||||
sub: did.to_string(),
|
||||
aud: "did:web:appview.maarcadetweet.local".into(),
|
||||
// The AppView this token is meant for. Derived from
|
||||
// `APPVIEW_PUBLIC_URL` rather than hardcoded, so the AppView can
|
||||
// check it against its own identity (`AppConfig::appview_did`).
|
||||
aud: cfg.appview_did(),
|
||||
iat: now,
|
||||
exp,
|
||||
jti: Some(uuid::Uuid::new_v4().to_string()),
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
mod appview_push;
|
||||
mod car;
|
||||
mod dag_cbor;
|
||||
mod firehose;
|
||||
mod invite;
|
||||
mod jwt_issuer;
|
||||
mod keys;
|
||||
mod password;
|
||||
@@ -15,6 +18,16 @@ use serde_json::json;
|
||||
use tracing::{info, warn};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
/// Usage line for the binary itself. The subcommands are operator
|
||||
/// tooling; the no-argument form is the server, which is what every
|
||||
/// deploy script and systemd unit invokes.
|
||||
const USAGE: &str = "\
|
||||
usage: pds-server [command]
|
||||
|
||||
(no command) run the PDS server
|
||||
invite … manage invite codes (see `pds-server invite help`)
|
||||
help show this message";
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Load `.env` from the working directory (and upwards) if present.
|
||||
@@ -23,11 +36,57 @@ async fn main() -> anyhow::Result<()> {
|
||||
// PDS_HOST`. Real environment variables always win over the file.
|
||||
let _ = dotenvy::dotenv();
|
||||
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
|
||||
.init();
|
||||
// Argument dispatch, by hand.
|
||||
//
|
||||
// The workspace carries no argument-parsing crate and this does not
|
||||
// justify adding one: exactly one subcommand exists, and the
|
||||
// overwhelmingly common invocation is the bare binary. Anything we
|
||||
// do not recognise is an error rather than being ignored — a
|
||||
// mistyped `pds-server invit create` that silently booted a server
|
||||
// would look like it worked and mint no codes.
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
match args.first().map(|s| s.as_str()) {
|
||||
None => {}
|
||||
Some("invite") => {
|
||||
// CLI output is meant to be read and pasted, so keep the
|
||||
// log stream quiet unless the operator asked for it. Without
|
||||
// this, `sqlx::migrate` chatters over the codes.
|
||||
init_tracing("warn");
|
||||
return invite::run_cli(&args[1..]).await;
|
||||
}
|
||||
Some("help") | Some("-h") | Some("--help") => {
|
||||
println!("{USAGE}");
|
||||
return Ok(());
|
||||
}
|
||||
Some(other) => {
|
||||
eprintln!("pds-server: unknown command: {other}\n\n{USAGE}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
init_tracing("info");
|
||||
|
||||
let cfg = at_shared::config::AppConfig::from_env()?;
|
||||
|
||||
// Announce the relaxed security posture before we bind a port.
|
||||
//
|
||||
// `PDS_INVITE_REQUIRED` is the only switch in `AppConfig` that
|
||||
// defaults to *open* (so the integration suites and dev instances
|
||||
// can keep creating throwaway accounts), which makes this warning
|
||||
// the only thing standing between "we made the PDS public" and
|
||||
// "anyone on the internet can mint repos on our disk". Mirrors
|
||||
// `appview`'s `auth::log_startup_posture`.
|
||||
if !cfg.pds_invite_required {
|
||||
warn!(
|
||||
"PDS_INVITE_REQUIRED is not true — com.atproto.server.createAccount accepts \
|
||||
ANY caller, and every accepted account allocates a repo, a server-held key \
|
||||
pair and firehose events. Fine on a private/dev instance; on a publicly \
|
||||
reachable PDS set PDS_INVITE_REQUIRED=true and hand out codes with \
|
||||
`pds-server invite create`."
|
||||
);
|
||||
} else {
|
||||
info!("PDS_INVITE_REQUIRED=true — createAccount requires a valid invite code");
|
||||
}
|
||||
let db = sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(32)
|
||||
.min_connections(2)
|
||||
@@ -69,10 +128,26 @@ async fn main() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Install the tracing subscriber, with `default_filter` as the level
|
||||
/// when `RUST_LOG` says nothing.
|
||||
///
|
||||
/// Factored out because the two entry points want different defaults:
|
||||
/// the server wants `info`, the `invite` subcommand wants `warn` so that
|
||||
/// migration chatter does not land in the middle of a list of codes the
|
||||
/// operator is about to copy.
|
||||
fn init_tracing(default_filter: &str) {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default_filter)),
|
||||
)
|
||||
.init();
|
||||
}
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.route("/", get(root))
|
||||
.route("/healthz", get(healthz))
|
||||
.route("/.well-known/did.json", get(did_document))
|
||||
.route(
|
||||
"/xrpc/com.atproto.server.describeServer",
|
||||
get(describe_server),
|
||||
@@ -145,6 +220,13 @@ pub fn router(state: AppState) -> Router {
|
||||
"/xrpc/com.atproto.sync.getBlob",
|
||||
get(routes::blob::get_blob),
|
||||
)
|
||||
// The firehose. A WebSocket upgrade arrives as a plain GET, so this
|
||||
// is a normal `get` route whose handler happens to return an
|
||||
// upgrade response.
|
||||
.route(
|
||||
"/xrpc/com.atproto.sync.subscribeRepos",
|
||||
get(routes::subscribe_repos::subscribe_repos),
|
||||
)
|
||||
.route(
|
||||
"/blob/:cid",
|
||||
get(routes::blob::get_blob_by_cid),
|
||||
@@ -163,15 +245,80 @@ async fn healthz() -> Json<serde_json::Value> {
|
||||
Json(json!({ "ok": true }))
|
||||
}
|
||||
|
||||
/// `GET /.well-known/did.json` — the PDS's own DID document.
|
||||
///
|
||||
/// This is how the AppView (and any other relying party) learns the
|
||||
/// P-256 public key that the access JWTs in
|
||||
/// `Authorization: Bearer …` are signed with. Without it the AppView
|
||||
/// could not verify a token at all, and the only alternative would be
|
||||
/// shipping `PDS_JWT_SECRET` to a second service — a private signing
|
||||
/// key crossing a service boundary, for a check that needs nothing but
|
||||
/// the public half.
|
||||
///
|
||||
/// Nothing in this response is secret. `publicKeyMultibase` is the
|
||||
/// uncompressed P-256 point derived from `PDS_JWT_SECRET` by
|
||||
/// [`jwt_issuer::server_p256_public_multibase`]; the secret itself
|
||||
/// never leaves this process.
|
||||
///
|
||||
/// The document id is [`AppConfig::pds_did`], i.e. it follows
|
||||
/// `PDS_PUBLIC_URL` — so a `did:web:` resolver that starts from the DID,
|
||||
/// rebuilds the URL and fetches this path lands back here rather than at
|
||||
/// some other host's document.
|
||||
async fn did_document(State(state): State<AppState>) -> Result<Json<serde_json::Value>, (axum::http::StatusCode, Json<serde_json::Value>)> {
|
||||
let did = state.cfg.pds_did();
|
||||
let public_multibase = jwt_issuer::server_p256_public_multibase(&state.cfg).map_err(|e| {
|
||||
// A malformed `PDS_JWT_SECRET` is the one way this fails, and
|
||||
// it is exactly the failure that also breaks every token this
|
||||
// server issues — surface it instead of publishing a document
|
||||
// with a missing key.
|
||||
(
|
||||
axum::http::StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({
|
||||
"error": "InternalServerError",
|
||||
"message": format!("server key unavailable: {e}"),
|
||||
})),
|
||||
)
|
||||
})?;
|
||||
Ok(Json(json!({
|
||||
"@context": [
|
||||
"https://www.w3.org/ns/did/v1",
|
||||
"https://w3id.org/security/multikey/v1",
|
||||
],
|
||||
"id": did,
|
||||
"verificationMethod": [{
|
||||
// `#atproto` is the fragment AT Proto uses for a repo's
|
||||
// signing key; we reuse it for the server key so a generic
|
||||
// did:web consumer finds it in the usual place.
|
||||
"id": format!("{did}#atproto"),
|
||||
"type": "Multikey",
|
||||
"controller": did,
|
||||
"publicKeyMultibase": public_multibase,
|
||||
}],
|
||||
"service": [{
|
||||
"id": "#atproto_pds",
|
||||
"type": "AtprotoPersonalDataServer",
|
||||
"serviceEndpoint": state.cfg.pds_public_url,
|
||||
}],
|
||||
})))
|
||||
}
|
||||
|
||||
async fn describe_server(State(state): State<AppState>) -> Json<DescribeServerResp> {
|
||||
Json(DescribeServerResp {
|
||||
did: "did:web:pds.maarcadetweet.local".into(),
|
||||
// Derived from `PDS_PUBLIC_URL`, never hardcoded — see
|
||||
// `AppConfig::pds_did`. The same value ids the document at
|
||||
// `/.well-known/did.json`.
|
||||
did: state.cfg.pds_did(),
|
||||
available_user_domains: vec![state
|
||||
.cfg
|
||||
.pds_handle_dns_zone
|
||||
.trim_start_matches('.')
|
||||
.to_string()],
|
||||
invite_code_required: false,
|
||||
// The real switch, not a hardcoded `false`. A client reads this
|
||||
// to decide whether to ask the user for a code *before*
|
||||
// collecting a handle and password — advertising `false` on a
|
||||
// server that then answers `400 InvalidInviteCode` sends the
|
||||
// user back to the start of a form they already filled in.
|
||||
invite_code_required: state.cfg.pds_invite_required,
|
||||
links: json!({
|
||||
"termsOfService": null,
|
||||
"privacyPolicy": null,
|
||||
|
||||
@@ -93,6 +93,55 @@ pub async fn create_account(
|
||||
|
||||
let mut tx = state.db.begin().await.map_err(|e| internal(e))?;
|
||||
|
||||
// Invite gate.
|
||||
//
|
||||
// Inside the transaction, and *first* inside it, for two reasons.
|
||||
//
|
||||
// Inside, because the code must be spent if and only if an account
|
||||
// was really created. Redeeming before `begin()` (or in a
|
||||
// transaction of its own) would burn a code every time the INSERT
|
||||
// below hit the `users.handle` unique index — a user who lost a
|
||||
// handle race would also lose their invite, with nothing to show
|
||||
// for it. Everything from here to `tx.commit()` rolls back together.
|
||||
//
|
||||
// First, because `invite::redeem` takes the code row's lock, and
|
||||
// holding it across the account INSERTs is what serialises two
|
||||
// registrations that present the same last remaining use. See
|
||||
// `invite::redeem` for how the conditional UPDATE turns that lock
|
||||
// into a correct decision rather than a stale one.
|
||||
//
|
||||
// Note this runs after the handle/password validation above, so a
|
||||
// malformed request is rejected without touching a code at all.
|
||||
if state.cfg.pds_invite_required {
|
||||
let supplied = req
|
||||
.invite_code
|
||||
.as_deref()
|
||||
.and_then(crate::invite::normalize);
|
||||
match supplied {
|
||||
None => return Err(invalid_invite_code()),
|
||||
Some(code) => {
|
||||
if let Err(e) =
|
||||
crate::invite::redeem(&mut tx, &code, &did, &req.handle).await
|
||||
{
|
||||
return match e {
|
||||
crate::invite::RedeemError::Invalid => {
|
||||
// Deliberately not logged with the code at
|
||||
// info level: a public endpoint that echoes
|
||||
// every guessed code into the log is a way
|
||||
// to fill the disk from outside.
|
||||
warn!(
|
||||
handle = %req.handle,
|
||||
"createAccount rejected: invite code invalid, disabled or spent"
|
||||
);
|
||||
Err(invalid_invite_code())
|
||||
}
|
||||
crate::invite::RedeemError::Db(db_err) => Err(internal(db_err)),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sqlx::query(
|
||||
r#"INSERT INTO users (did, handle, email, password_hash, signing_key, rotation_key)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)"#,
|
||||
@@ -306,6 +355,28 @@ pub async fn refresh_session(
|
||||
}))
|
||||
}
|
||||
|
||||
/// The one error a failed invite check produces.
|
||||
///
|
||||
/// Same `(StatusCode, Json<ErrorBody>)` shape every other route in this
|
||||
/// module returns, so a client parses it with the code it already has:
|
||||
/// `{"error": "InvalidInviteCode", "message": "..."}` under a `400`.
|
||||
///
|
||||
/// One message for every failure mode — missing, unknown, disabled,
|
||||
/// spent — on purpose. A distinct "that code exists but is used up"
|
||||
/// would let an unauthenticated caller walk the code space and learn
|
||||
/// which strings are real, which is most of the work of stealing one.
|
||||
/// The operator can tell the cases apart from `pds-server invite list`;
|
||||
/// the internet cannot.
|
||||
fn invalid_invite_code() -> (StatusCode, Json<crate::routes::types::ErrorBody>) {
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(crate::routes::types::ErrorBody::new(
|
||||
"InvalidInviteCode",
|
||||
Some("a valid invite code is required to create an account on this server".into()),
|
||||
)),
|
||||
)
|
||||
}
|
||||
|
||||
fn internal(e: impl std::fmt::Display) -> (StatusCode, Json<crate::routes::types::ErrorBody>) {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
|
||||
@@ -17,7 +17,10 @@
|
||||
//! removed from the MST, a new commit is signed, the AppView is
|
||||
//! told to drop the row, and we return the new commit CID + rev.
|
||||
|
||||
use crate::routes::helpers::{apply_repo_write, err, lookup_handle, to_sqlx_error, RepoWriteOutcome};
|
||||
use crate::firehose::RepoOp;
|
||||
use crate::routes::helpers::{
|
||||
apply_repo_write, err, lookup_handle, to_sqlx_error, RepoWriteOutcome, RepoWriteResult,
|
||||
};
|
||||
use at_repo::blockstore::Blockstore;
|
||||
use crate::routes::types::ErrorBody;
|
||||
use crate::state::AppState;
|
||||
@@ -189,11 +192,16 @@ fn build_like_record(req: &CreateLikeReq) -> Result<Value, (StatusCode, Json<Err
|
||||
/// Postgres transaction with `SELECT … FOR UPDATE` on the user's
|
||||
/// `repos` row. Concurrent writers for the same DID now serialise
|
||||
/// behind the row lock instead of clobbering each other.)
|
||||
///
|
||||
/// Returns the full [`RepoWriteResult`] rather than just the commit: the
|
||||
/// firehose event that went into the same transaction carries the sequence
|
||||
/// number, which the handlers log so an operator can line a write up against
|
||||
/// what a subscriber received.
|
||||
async fn apply_and_commit<F>(
|
||||
state: &AppState,
|
||||
did: &str,
|
||||
f: F,
|
||||
) -> Result<at_repo::commit::Commit, (StatusCode, Json<ErrorBody>)>
|
||||
) -> Result<RepoWriteResult, (StatusCode, Json<ErrorBody>)>
|
||||
where
|
||||
F: for<'b> FnOnce(
|
||||
&'b mut at_repo::repo::Repo<at_repo::blockstore::MemoryBlockstore>,
|
||||
@@ -201,7 +209,7 @@ where
|
||||
Box<dyn std::future::Future<Output = Result<RepoWriteOutcome, sqlx::Error>> + Send + 'b>,
|
||||
>,
|
||||
{
|
||||
apply_repo_write(state, did, f).await.map(|o| o.commit)
|
||||
apply_repo_write(state, did, f).await
|
||||
}
|
||||
|
||||
// -- handlers ---------------------------------------------------------------
|
||||
@@ -275,7 +283,7 @@ pub async fn create_like(
|
||||
let push_rkey = rkey.clone();
|
||||
let push_handle_str: Option<String> = lookup_handle(&state, &did).await;
|
||||
|
||||
let commit = apply_and_commit(&state, &did, move |repo| {
|
||||
let write = apply_and_commit(&state, &did, move |repo| {
|
||||
let value_cid = value_cid;
|
||||
let rkey = rkey;
|
||||
let record_buf = record_buf;
|
||||
@@ -297,16 +305,21 @@ pub async fn create_like(
|
||||
commit,
|
||||
head_cid_bytes,
|
||||
head_commit_bytes,
|
||||
// Always a create: the rkey is a freshly minted TID, so it
|
||||
// cannot collide with an existing entry.
|
||||
ops: vec![RepoOp::create(LIKE_COLLECTION, &rkey, value_cid)],
|
||||
})
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
let commit = write.commit;
|
||||
info!(
|
||||
collection = LIKE_COLLECTION,
|
||||
rkey = %push_rkey,
|
||||
cid = %value_cid,
|
||||
commit = %commit.cid,
|
||||
seq = write.event.seq,
|
||||
"like created"
|
||||
);
|
||||
|
||||
@@ -366,10 +379,21 @@ pub async fn delete_record(
|
||||
// `Repo::delete_record` is idempotent at the MST level (returns
|
||||
// an unchanged tree if the key isn't present), so we always
|
||||
// sign a new commit — the spec says 200 on a no-op delete.
|
||||
let commit = apply_and_commit(&state, &did, move |repo| {
|
||||
let write = apply_and_commit(&state, &did, move |repo| {
|
||||
let collection = collection;
|
||||
let rkey = rkey;
|
||||
Box::pin(async move {
|
||||
// Report the op only when there was something to remove.
|
||||
// `delete_record` is idempotent — deleting a missing rkey signs
|
||||
// an unchanged tree — and announcing a delete for a record that
|
||||
// never existed would make a consumer drop a row it may legitimately
|
||||
// hold under a different rkey, or (worse) log a phantom deletion
|
||||
// on every retry of a duplicate unlike.
|
||||
let existed = repo
|
||||
.get_record(&collection, &rkey)
|
||||
.await
|
||||
.map_err(to_sqlx_error)?
|
||||
.is_some();
|
||||
repo.delete_record(&collection, &rkey)
|
||||
.await
|
||||
.map_err(to_sqlx_error)?;
|
||||
@@ -380,15 +404,22 @@ pub async fn delete_record(
|
||||
commit,
|
||||
head_cid_bytes,
|
||||
head_commit_bytes,
|
||||
ops: if existed {
|
||||
vec![RepoOp::delete(&collection, &rkey)]
|
||||
} else {
|
||||
Vec::new()
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
let commit = write.commit;
|
||||
info!(
|
||||
collection = %push_collection,
|
||||
rkey = %push_rkey,
|
||||
commit = %commit.cid,
|
||||
seq = write.event.seq,
|
||||
"record deleted"
|
||||
);
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
//! writers for the same DID can't trample each other's MST updates
|
||||
//! (Phase 5b review C1).
|
||||
|
||||
use crate::firehose::{self, FirehoseEvent, RepoOp};
|
||||
use crate::routes::types::ErrorBody;
|
||||
use crate::state::AppState;
|
||||
use at_crypto::cid::cid_from_multihash_bytes;
|
||||
@@ -24,6 +25,7 @@ use cid::Cid;
|
||||
use k256::ecdsa::SigningKey;
|
||||
use k256::SecretKey;
|
||||
use sqlx::Postgres;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Load every block belonging to `did` from the `repo_blocks` table into a
|
||||
@@ -184,14 +186,31 @@ pub fn to_sqlx_error(e: anyhow::Error) -> sqlx::Error {
|
||||
// transaction commits or rolls back, so concurrent writers serialise
|
||||
// behind the holder rather than racing on the head_commit column.
|
||||
|
||||
/// Result of a successful repo write: the new signed commit, the CID
|
||||
/// pointing at the freshly-written head block, and the new revision
|
||||
/// string. Callers use the commit for AppView ingest pushes.
|
||||
/// What the closure handed to [`apply_repo_write`] returns: the new signed
|
||||
/// commit, the CID pointing at the freshly-written head block, and the record
|
||||
/// operations the closure performed.
|
||||
///
|
||||
/// `ops` is not derivable from the commit — the MST stores the resulting
|
||||
/// tree, not the edit that produced it, and it cannot tell a create from an
|
||||
/// update at all. Only the closure knows what it did, so it says so, and the
|
||||
/// firehose event is built from that. Every write path must fill this in
|
||||
/// truthfully: an empty `ops` produces a commit frame that tells the AppView
|
||||
/// "something changed, guess what".
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RepoWriteOutcome {
|
||||
pub commit: at_repo::commit::Commit,
|
||||
pub head_cid_bytes: Vec<u8>,
|
||||
pub head_commit_bytes: Vec<u8>,
|
||||
pub ops: Vec<RepoOp>,
|
||||
}
|
||||
|
||||
/// What [`apply_repo_write`] returns to the route handler: the commit (used
|
||||
/// for the response body and the AppView push) plus the firehose event that
|
||||
/// was appended in the same transaction and has already been broadcast.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RepoWriteResult {
|
||||
pub commit: at_repo::commit::Commit,
|
||||
pub event: FirehoseEvent,
|
||||
}
|
||||
|
||||
/// Apply a write to the user's repo under a row-level lock on the
|
||||
@@ -218,11 +237,27 @@ pub struct RepoWriteOutcome {
|
||||
/// write the row), but the transaction stays open until after the
|
||||
/// `UPDATE`. If the closure or `UPDATE` fails, the transaction rolls
|
||||
/// back and no head pointer or block row changes are visible.
|
||||
///
|
||||
/// ## The firehose event rides in the same transaction
|
||||
///
|
||||
/// Between step 7 and the COMMIT we append one row to `firehose_events`
|
||||
/// (see [`crate::firehose`]). It has to be the *same* transaction, not a
|
||||
/// follow-up write: if the event were appended afterwards, a crash in the
|
||||
/// window between the two would leave a repo whose head has moved but whose
|
||||
/// event log never mentions it — and since a consumer's cursor only ever
|
||||
/// moves forward, that commit would be invisible to every subscriber
|
||||
/// permanently. Sharing the transaction makes "the head moved" and "an event
|
||||
/// exists for it" one atomic fact. Conversely, a rollback discards both, so
|
||||
/// no subscriber ever sees an event for a commit that did not happen.
|
||||
///
|
||||
/// The broadcast to live subscribers happens *after* `COMMIT`, for the same
|
||||
/// reason in reverse: a subscriber must never receive an event that a
|
||||
/// rollback then erases.
|
||||
pub async fn apply_repo_write<F>(
|
||||
state: &AppState,
|
||||
did: &str,
|
||||
f: F,
|
||||
) -> Result<RepoWriteOutcome, (StatusCode, Json<ErrorBody>)>
|
||||
) -> Result<RepoWriteResult, (StatusCode, Json<ErrorBody>)>
|
||||
where
|
||||
F: for<'b> FnOnce(
|
||||
&'b mut Repo<MemoryBlockstore>,
|
||||
@@ -240,8 +275,12 @@ where
|
||||
|
||||
// 2. Take the row-level write lock. Postgres parks competing
|
||||
// transactions here until we COMMIT/ROLLBACK.
|
||||
let head_row: Option<(Vec<u8>, Vec<u8>, Option<Vec<u8>>)> = sqlx::query_as(
|
||||
"SELECT head_cid, head_commit, prev_commit
|
||||
// `rev` comes along because it is the *previous* commit's revision,
|
||||
// which the firehose frame publishes as `since` — a consumer uses it to
|
||||
// notice that it skipped an intermediate commit. It has to be read here,
|
||||
// under the lock, before the UPDATE below overwrites it.
|
||||
let head_row: Option<(Vec<u8>, Vec<u8>, Option<Vec<u8>>, String)> = sqlx::query_as(
|
||||
"SELECT head_cid, head_commit, prev_commit, rev
|
||||
FROM repos
|
||||
WHERE did = $1
|
||||
FOR UPDATE",
|
||||
@@ -257,8 +296,8 @@ where
|
||||
)
|
||||
})?;
|
||||
|
||||
let (head_cid_blob, head_commit_blob) = match head_row {
|
||||
Some(r) => (r.0, r.1),
|
||||
let (head_cid_blob, head_commit_blob, prev_rev) = match head_row {
|
||||
Some(r) => (r.0, r.1, r.3),
|
||||
None => {
|
||||
return Err((
|
||||
StatusCode::NOT_FOUND,
|
||||
@@ -354,6 +393,26 @@ where
|
||||
// to run additional queries under the row lock, we'd extend
|
||||
// this helper to also hand out a `&mut PgConnection` (which
|
||||
// doesn't have the lifetime headache of `&mut Transaction`).
|
||||
// Snapshot the CIDs the repo already had *before* the closure runs.
|
||||
// Diffing against this afterwards is what tells us which blocks are
|
||||
// new in this commit — the firehose CAR carries only those, because a
|
||||
// diff stream that re-sent the whole repo on every post would be
|
||||
// useless. The snapshot is taken here, after the head block re-seed
|
||||
// above, so the existing head commit does not look new.
|
||||
let blocks_before: HashSet<Cid> = blockstore
|
||||
.list()
|
||||
.await
|
||||
.map_err(|e| {
|
||||
err(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"InternalServerError",
|
||||
format!("blockstore list: {e:#}"),
|
||||
)
|
||||
})?
|
||||
.into_iter()
|
||||
.map(|(cid, _)| cid)
|
||||
.collect();
|
||||
|
||||
let outcome: RepoWriteOutcome = f(&mut repo).await.map_err(|e| {
|
||||
err(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -409,6 +468,51 @@ where
|
||||
)
|
||||
})?;
|
||||
|
||||
// 8. Append the firehose event. Same transaction as everything above —
|
||||
// see the "rides in the same transaction" note on this function.
|
||||
//
|
||||
// The CAR carries the commit block as its root plus every block that
|
||||
// was not in the repo when we started: the new MST nodes and the new
|
||||
// record value. Blocks that already existed are omitted; a consumer
|
||||
// that needs an ancestor asks `com.atproto.sync.getBlocks` for it.
|
||||
let new_blocks: Vec<(Cid, Vec<u8>)> = all_blocks
|
||||
.iter()
|
||||
.filter(|(cid, _)| !blocks_before.contains(*cid) && **cid != outcome.commit.cid)
|
||||
.map(|(cid, bytes)| (*cid, bytes.clone()))
|
||||
.collect();
|
||||
let blocks_car = firehose::build_blocks_car(
|
||||
outcome.commit.cid,
|
||||
&outcome.head_commit_bytes,
|
||||
&new_blocks,
|
||||
);
|
||||
|
||||
// A repo whose head was the all-zero sentinel had no previous commit, so
|
||||
// there is no previous revision to report — `since` is null rather than
|
||||
// the `"0"` placeholder `createAccount` seeds the row with.
|
||||
let since: Option<&str> = if is_zero_blob(&head_cid_blob) {
|
||||
None
|
||||
} else {
|
||||
Some(prev_rev.as_str())
|
||||
};
|
||||
|
||||
let event = firehose::insert_event_in_tx(
|
||||
&mut tx,
|
||||
did,
|
||||
&outcome.commit.rev,
|
||||
since,
|
||||
outcome.commit.cid,
|
||||
blocks_car,
|
||||
outcome.ops.clone(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
err(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
"InternalServerError",
|
||||
format!("firehose_events insert: {e}"),
|
||||
)
|
||||
})?;
|
||||
|
||||
tx.commit().await.map_err(|e| {
|
||||
err(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -417,7 +521,18 @@ where
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(outcome)
|
||||
// 9. Only now, with the commit durable, hand the event to live
|
||||
// subscribers. Publishing is non-blocking and cannot fail in a way
|
||||
// that matters: a slow subscriber is dealt with on its own side (see
|
||||
// the lag policy in `crate::firehose`), and with no subscribers at all
|
||||
// the send is a no-op. The row is on disk either way, so nothing is
|
||||
// lost if this reaches nobody.
|
||||
state.firehose.publish(event.clone());
|
||||
|
||||
Ok(RepoWriteResult {
|
||||
commit: outcome.commit,
|
||||
event,
|
||||
})
|
||||
}
|
||||
|
||||
/// Persist every block in `blocks` into `repo_blocks` using the open
|
||||
|
||||
@@ -5,5 +5,6 @@ pub mod helpers;
|
||||
pub mod identity;
|
||||
pub mod profile;
|
||||
pub mod repo;
|
||||
pub mod subscribe_repos;
|
||||
pub mod sync;
|
||||
pub mod types;
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
//! overwrite the corresponding fields. Best-effort push to the
|
||||
//! AppView follows so the `profiles` cache reflects the new avatar /
|
||||
//! display name / bio without waiting for the Jetstream replay.
|
||||
use crate::firehose::RepoOp;
|
||||
use crate::jwt_issuer;
|
||||
use crate::routes::helpers::{
|
||||
apply_repo_write, err, load_head_commit, load_signing_key, load_user_blockstore,
|
||||
@@ -98,6 +99,9 @@ pub async fn set_profile(
|
||||
|
||||
// Fetch the existing record, if any.
|
||||
let existing = read_profile_record(&state, &did).await?;
|
||||
// Remembered before `existing` is consumed by the merge — the firehose op
|
||||
// needs to know whether this is the first profile write for the account.
|
||||
let existing_present = existing.is_some();
|
||||
|
||||
// For any blob CIDs in the request, look up the real
|
||||
// `mime_type` / `size` from the `blobs` table — and verify
|
||||
@@ -152,6 +156,12 @@ pub async fn set_profile(
|
||||
};
|
||||
|
||||
let next_for_block = next.clone();
|
||||
// `existing` was read before the merge above: a profile record that was
|
||||
// already there makes this an `update` on the firehose, a first-ever
|
||||
// `setProfile` a `create`. The rkey is the fixed `self`, so this is the
|
||||
// one write path where updates are the common case rather than the
|
||||
// exception.
|
||||
let profile_existed = existing_present;
|
||||
let outcome = apply_repo_write(&state, &did, move |repo| {
|
||||
let value_cid = value_cid;
|
||||
let next_for_block = next_for_block;
|
||||
@@ -175,6 +185,12 @@ pub async fn set_profile(
|
||||
commit,
|
||||
head_cid_bytes,
|
||||
head_commit_bytes,
|
||||
ops: vec![RepoOp::put(
|
||||
"app.bsky.actor.profile",
|
||||
"self",
|
||||
value_cid,
|
||||
profile_existed,
|
||||
)],
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -183,6 +199,7 @@ pub async fn set_profile(
|
||||
info!(
|
||||
did = %did,
|
||||
cid = %outcome.commit.cid,
|
||||
seq = outcome.event.seq,
|
||||
"profile record created"
|
||||
);
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use crate::firehose::RepoOp;
|
||||
use crate::routes::helpers::{
|
||||
apply_repo_write, err, lookup_handle, to_sqlx_error, RepoWriteOutcome,
|
||||
};
|
||||
@@ -110,6 +111,17 @@ pub async fn create_record(
|
||||
let record_buf = record_buf;
|
||||
let collection = collection;
|
||||
Box::pin(async move {
|
||||
// Ask the MST whether the key is already there *before* writing.
|
||||
// The firehose distinguishes `create` from `update` and the tree
|
||||
// itself cannot: after `put_record` both look identical. A
|
||||
// caller-supplied `rkey` (rather than the generated TID) is the
|
||||
// case that actually hits this — an overwrite of an existing
|
||||
// record must not be announced as a create.
|
||||
let existed = repo
|
||||
.get_record(&collection, &rkey)
|
||||
.await
|
||||
.map_err(to_sqlx_error)?
|
||||
.is_some();
|
||||
// Repo assumes the value block is already in the
|
||||
// blockstore — that's the caller's responsibility.
|
||||
repo.blockstore
|
||||
@@ -127,23 +139,34 @@ pub async fn create_record(
|
||||
commit,
|
||||
head_cid_bytes,
|
||||
head_commit_bytes,
|
||||
ops: vec![RepoOp::put(&collection, &rkey, value_cid, existed)],
|
||||
})
|
||||
})
|
||||
})
|
||||
.await?;
|
||||
|
||||
let uri = format!("at://{did}/{push_coll}/{push_rkey}");
|
||||
let seq = outcome.event.seq;
|
||||
let commit = outcome.commit;
|
||||
info!(uri = %uri, cid = %value_cid, commit = %commit.cid, "record created");
|
||||
info!(
|
||||
uri = %uri,
|
||||
cid = %value_cid,
|
||||
commit = %commit.cid,
|
||||
seq,
|
||||
"record created"
|
||||
);
|
||||
|
||||
// 10. Best-effort push to the AppView's `/internal/ingest-commit`.
|
||||
// We send the full record value (not just the CID) because the
|
||||
// AppView's indexer reads `embed` and `reply` off it.
|
||||
//
|
||||
// **Spawned** (not awaited) so a transient AppView outage never
|
||||
// blocks the user's write response. If the push fails, the
|
||||
// global Jetstream feed will eventually replay the commit to
|
||||
// the AppView.
|
||||
// blocks the user's write response. Losing the push is no longer
|
||||
// terminal: the same commit was appended to `firehose_events` in
|
||||
// the write transaction above (see `seq` in the log line), so an
|
||||
// AppView that reconnects to `com.atproto.sync.subscribeRepos`
|
||||
// with its cursor picks it up. The push is now purely a latency
|
||||
// optimisation, not the only delivery path.
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = push_handle
|
||||
.push_create(
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
//! `GET /xrpc/com.atproto.sync.subscribeRepos` — the firehose WebSocket.
|
||||
//!
|
||||
//! The frame format, the deviation from the atproto spec, the lag policy and
|
||||
//! the retention story all live in the module header of [`crate::firehose`];
|
||||
//! this file is only the socket.
|
||||
//!
|
||||
//! ## The handshake, and why it is ordered the way it is
|
||||
//!
|
||||
//! ```text
|
||||
//! 1. subscribe to the live broadcast <-- BEFORE any DB read
|
||||
//! 2. read (min, max) from firehose_events
|
||||
//! 3. decide what the cursor means <-- firehose::plan_cursor
|
||||
//! 4. drain the DB replay, remembering the highest seq sent
|
||||
//! 5. forward live events with seq > that high-water mark
|
||||
//! ```
|
||||
//!
|
||||
//! Step 1 has to come first. If we read the database and *then* subscribed,
|
||||
//! an event committed in between would be in neither: too late for the replay
|
||||
//! query, too early for the receiver. Subscribing first turns that race into
|
||||
//! a duplicate instead of a gap — the event is both replayed from the table
|
||||
//! and sitting in the channel — and a duplicate is something we can filter,
|
||||
//! which is what the high-water mark in step 5 does.
|
||||
//!
|
||||
//! The filter is exact rather than approximate because the write path
|
||||
//! serialises `firehose_events` inserts under an advisory lock (see
|
||||
//! [`crate::firehose::insert_event_in_tx`]): if the replay query saw seq `N`,
|
||||
//! then every seq below `N` is already committed and was also seen. So
|
||||
//! "everything the replay covered" is precisely "seq <= N", and every event
|
||||
//! that arrives on the channel afterwards has seq > N. No gap, no duplicate,
|
||||
//! at the handover.
|
||||
//!
|
||||
//! Without a cursor there is no replay at all and the high-water mark stays
|
||||
//! at zero: a `broadcast::Receiver` only ever yields messages sent after it
|
||||
//! was created, so "live only" needs no filtering.
|
||||
|
||||
use crate::firehose::{
|
||||
self, CursorPlan, FirehoseEvent, MAX_LAG_RECOVERIES, REPLAY_PAGE_SIZE,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
use axum::extract::ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade};
|
||||
use axum::extract::{Query, State};
|
||||
use axum::response::Response;
|
||||
use serde::Deserialize;
|
||||
use tokio::sync::broadcast::error::RecvError;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use futures::{SinkExt, StreamExt};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct SubscribeQuery {
|
||||
/// The seq of the last event the client already has. Everything with a
|
||||
/// larger seq is replayed before the live stream starts. Absent means
|
||||
/// "live only".
|
||||
pub cursor: Option<i64>,
|
||||
}
|
||||
|
||||
/// The upgrade handler. Everything interesting happens in [`run`].
|
||||
pub async fn subscribe_repos(
|
||||
State(state): State<AppState>,
|
||||
Query(q): Query<SubscribeQuery>,
|
||||
ws: WebSocketUpgrade,
|
||||
) -> Response {
|
||||
ws.on_upgrade(move |socket| run(socket, state, q.cursor))
|
||||
}
|
||||
|
||||
/// Drive one subscriber for the life of its connection.
|
||||
async fn run(socket: WebSocket, state: AppState, cursor: Option<i64>) {
|
||||
// 1. Subscribe first — see the ordering note in the module header.
|
||||
let mut rx = state.firehose.subscribe();
|
||||
|
||||
let (mut sink, mut stream) = socket.split();
|
||||
|
||||
// A firehose subscriber sends nothing after the upgrade, but we still
|
||||
// have to read the socket: that is the only way a Close frame (or a
|
||||
// client that vanishes without sending data) is noticed while we are
|
||||
// parked waiting for an event that may not come for hours. The reader
|
||||
// task does nothing but detect the end of the connection and say so.
|
||||
let (dead_tx, mut dead_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
tokio::spawn(async move {
|
||||
while let Some(msg) = stream.next().await {
|
||||
match msg {
|
||||
Ok(Message::Close(_)) | Err(_) => break,
|
||||
// Ping/Pong are handled by axum itself; anything else a
|
||||
// client sends on this endpoint is meaningless and ignored
|
||||
// rather than treated as an error.
|
||||
Ok(_) => continue,
|
||||
}
|
||||
}
|
||||
let _ = dead_tx.send(());
|
||||
});
|
||||
|
||||
// 2./3. Work out what the cursor asks for.
|
||||
let bounds = match firehose::seq_bounds(&state.db).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "subscribeRepos: firehose_events bounds read failed");
|
||||
let _ = send_error(&mut sink, "InternalServerError", "event log unavailable").await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut replay_from = match firehose::plan_cursor(cursor, bounds) {
|
||||
CursorPlan::LiveOnly => None,
|
||||
CursorPlan::Replay { from } => Some(from),
|
||||
CursorPlan::OutdatedCursor { from, message } => {
|
||||
// Not fatal: the client keeps its connection and gets everything
|
||||
// we still have. It is told about the hole so it can decide
|
||||
// whether to backfill some other way.
|
||||
info!(cursor = ?cursor, %message, "subscribeRepos: outdated cursor");
|
||||
if sink
|
||||
.send(Message::Binary(firehose::encode_info_frame(
|
||||
"OutdatedCursor",
|
||||
&message,
|
||||
)))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
Some(from)
|
||||
}
|
||||
CursorPlan::FutureCursor { message } => {
|
||||
// Fatal. Continuing would leave the client waiting for events
|
||||
// that will be numbered below its cursor and therefore filtered
|
||||
// out forever — silence is the worst possible answer here.
|
||||
info!(cursor = ?cursor, %message, "subscribeRepos: future cursor");
|
||||
let _ = send_error(&mut sink, "FutureCursor", &message).await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
info!(
|
||||
cursor = ?cursor,
|
||||
replay_from = ?replay_from,
|
||||
subscribers = state.firehose.subscriber_count(),
|
||||
"subscribeRepos: client connected"
|
||||
);
|
||||
|
||||
// 4. Drain the replay. `high_water` ends up as the last seq the client
|
||||
// has been given, which is exactly the boundary the live filter needs.
|
||||
let mut high_water: i64 = 0;
|
||||
if let Some(from) = replay_from.take() {
|
||||
match replay(&mut sink, &state, from, &mut dead_rx).await {
|
||||
Ok(last) => high_water = last,
|
||||
// The socket died mid-replay, or the log became unreadable.
|
||||
// Either way there is nothing left to do for this connection.
|
||||
Err(()) => return,
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Live.
|
||||
let mut lag_recoveries: u32 = 0;
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = &mut dead_rx => {
|
||||
debug!("subscribeRepos: client closed");
|
||||
return;
|
||||
}
|
||||
recv = rx.recv() => match recv {
|
||||
Ok(event) => {
|
||||
if event.seq <= high_water {
|
||||
// Already delivered by the replay. This is the
|
||||
// duplicate the subscribe-first ordering trades the
|
||||
// gap for.
|
||||
continue;
|
||||
}
|
||||
if !send_event(&mut sink, &event).await {
|
||||
return;
|
||||
}
|
||||
high_water = event.seq;
|
||||
}
|
||||
Err(RecvError::Lagged(skipped)) => {
|
||||
lag_recoveries += 1;
|
||||
warn!(
|
||||
skipped,
|
||||
attempt = lag_recoveries,
|
||||
high_water,
|
||||
"subscribeRepos: subscriber lagged; falling back to the database replay"
|
||||
);
|
||||
if lag_recoveries > MAX_LAG_RECOVERIES {
|
||||
// See the lag policy in `crate::firehose`: we rescue a
|
||||
// slow client, repeatedly, but not forever.
|
||||
let _ = send_error(
|
||||
&mut sink,
|
||||
"ConsumerTooSlow",
|
||||
"consumer fell behind repeatedly; reconnect with a cursor",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
let message = format!(
|
||||
"consumer lagged by {skipped} events; resuming from seq {high_water} \
|
||||
via the durable log"
|
||||
);
|
||||
if sink
|
||||
.send(Message::Binary(firehose::encode_info_frame(
|
||||
"OutdatedCursor",
|
||||
&message,
|
||||
)))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Nothing is lost: every event is in `firehose_events`,
|
||||
// so re-reading from the high-water mark is the same
|
||||
// stream the channel dropped. The advisory-lock ordering
|
||||
// guarantees the new high-water mark is again an exact
|
||||
// boundary for the live filter.
|
||||
match replay(&mut sink, &state, high_water, &mut dead_rx).await {
|
||||
Ok(last) => high_water = last.max(high_water),
|
||||
Err(()) => return,
|
||||
}
|
||||
}
|
||||
Err(RecvError::Closed) => {
|
||||
// Only happens at process shutdown, when the AppState
|
||||
// (and with it the sender) is dropped.
|
||||
debug!("subscribeRepos: broadcast channel closed");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream every event with `seq > from` out of the database, page by page.
|
||||
///
|
||||
/// Returns the highest seq actually sent (or `from` when there was nothing to
|
||||
/// send), or `Err(())` when the connection or the database gave out — in
|
||||
/// which case the caller should drop the connection.
|
||||
///
|
||||
/// Paging matters: a client reconnecting with `cursor=0` after a long uptime
|
||||
/// would otherwise pull the whole table, CARs and all, into memory at once.
|
||||
/// It also gives the loop a natural place to notice a client that closed the
|
||||
/// socket halfway through a large backfill.
|
||||
async fn replay(
|
||||
sink: &mut futures::stream::SplitSink<WebSocket, Message>,
|
||||
state: &AppState,
|
||||
from: i64,
|
||||
dead_rx: &mut tokio::sync::oneshot::Receiver<()>,
|
||||
) -> Result<i64, ()> {
|
||||
let mut cursor = from;
|
||||
loop {
|
||||
if dead_rx.try_recv().is_ok() {
|
||||
return Err(());
|
||||
}
|
||||
let page = match firehose::load_events_after(&state.db, cursor, REPLAY_PAGE_SIZE).await {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
warn!(error = %format!("{e:#}"), "subscribeRepos: replay read failed");
|
||||
let _ = send_error(sink, "InternalServerError", "event log read failed").await;
|
||||
return Err(());
|
||||
}
|
||||
};
|
||||
if page.is_empty() {
|
||||
return Ok(cursor);
|
||||
}
|
||||
for event in &page {
|
||||
if !send_event(sink, event).await {
|
||||
return Err(());
|
||||
}
|
||||
cursor = event.seq;
|
||||
}
|
||||
// A short page means we reached the end of the log.
|
||||
if (page.len() as i64) < REPLAY_PAGE_SIZE {
|
||||
return Ok(cursor);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Send one `#commit` frame. Returns `false` when the socket is gone.
|
||||
async fn send_event(
|
||||
sink: &mut futures::stream::SplitSink<WebSocket, Message>,
|
||||
event: &FirehoseEvent,
|
||||
) -> bool {
|
||||
sink.send(Message::Binary(event.to_frame())).await.is_ok()
|
||||
}
|
||||
|
||||
/// Send a terminal error frame and close the socket.
|
||||
///
|
||||
/// The close is explicit (rather than just dropping the sink) so a client
|
||||
/// distinguishes "the server said no" from "the connection broke".
|
||||
async fn send_error(
|
||||
sink: &mut futures::stream::SplitSink<WebSocket, Message>,
|
||||
error: &str,
|
||||
message: &str,
|
||||
) -> bool {
|
||||
if sink
|
||||
.send(Message::Binary(firehose::encode_error_frame(error, message)))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
sink.send(Message::Close(Some(CloseFrame {
|
||||
code: axum::extract::ws::close_code::NORMAL,
|
||||
reason: error.to_string().into(),
|
||||
})))
|
||||
.await
|
||||
.is_ok()
|
||||
}
|
||||
@@ -6,6 +6,18 @@ pub struct CreateAccountReq {
|
||||
pub email: Option<String>,
|
||||
pub password: Option<String>,
|
||||
pub did: Option<String>,
|
||||
/// The invite code, when `PDS_INVITE_REQUIRED` is on.
|
||||
///
|
||||
/// The alias is not cosmetic. This struct — like every other type in
|
||||
/// this module — is snake_case on the wire, which is what our own
|
||||
/// clients send. The AT Protocol lexicon for
|
||||
/// `com.atproto.server.createAccount` spells the field `inviteCode`,
|
||||
/// so every off-the-shelf atproto client sends *that*, and without
|
||||
/// the alias serde would drop it into `None` silently — the account
|
||||
/// would be refused with "an invite code is required" while the user
|
||||
/// is looking at the code they just pasted. Accepting both spellings
|
||||
/// costs one attribute; debugging that report costs an afternoon.
|
||||
#[serde(alias = "inviteCode")]
|
||||
pub invite_code: Option<String>,
|
||||
pub recovery_key: Option<String>,
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::appview_push::AppViewPushClient;
|
||||
use crate::firehose::Firehose;
|
||||
use at_blob::S3BlobStore;
|
||||
use at_identity::plc::PlcClient;
|
||||
use at_lexicon::{Lex, LexRegistry};
|
||||
@@ -16,6 +17,14 @@ pub struct AppState {
|
||||
pub blockstore: Arc<MemoryBlockstore>,
|
||||
pub plc: PlcClient,
|
||||
pub appview: AppViewPushClient,
|
||||
/// Live fan-out for `com.atproto.sync.subscribeRepos`.
|
||||
///
|
||||
/// Lives on the shared state rather than in the route module because the
|
||||
/// *write* paths publish into it — `routes::helpers::apply_repo_write`
|
||||
/// hands every committed event over here — while the WebSocket handler
|
||||
/// only subscribes. Cloning `AppState` clones the sender, which is the
|
||||
/// intended way to reach it from a handler.
|
||||
pub firehose: Firehose,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
@@ -26,9 +35,9 @@ impl AppState {
|
||||
Lex::from_json(include_str!("../../../lexicons/app/twi/post.json")).unwrap(),
|
||||
);
|
||||
// AT-Protocol standard collections: only the records the user
|
||||
// might legitimately create server-side (feed.like + feed.repost).
|
||||
// The full atproto collection library is out of scope — for
|
||||
// anything else, callers pass `validate: false` in the
|
||||
// might legitimately create server-side (feed.like, feed.repost,
|
||||
// graph.follow). The full atproto collection library is out of
|
||||
// scope — for anything else, callers pass `validate: false` in the
|
||||
// createRecord body.
|
||||
lex.lexicons.insert(
|
||||
"app.bsky.feed.like".to_string(),
|
||||
@@ -38,6 +47,17 @@ impl AppState {
|
||||
"app.bsky.feed.repost".to_string(),
|
||||
Lex::from_json(include_str!("../../../lexicons/app/bsky/feed/repost.json")).unwrap(),
|
||||
);
|
||||
// Follow record. Its absence was a real outage: the desktop
|
||||
// client creates follows through `createRecord`, which validates
|
||||
// by default, so every follow came back
|
||||
// `unknown lexicon: app.bsky.graph.follow` — the button could
|
||||
// never have worked. `subject` is a bare DID string here, not a
|
||||
// strongRef like like/repost use, matching what the client sends
|
||||
// and what the AppView's `follow_subject_did` reads.
|
||||
lex.lexicons.insert(
|
||||
"app.bsky.graph.follow".to_string(),
|
||||
Lex::from_json(include_str!("../../../lexicons/app/bsky/graph/follow.json")).unwrap(),
|
||||
);
|
||||
// Profile record — avatar/banner/display name/description.
|
||||
// Validates the createRecord body when the Tauri client calls
|
||||
// its setProfile command. Other fields stay optional so a
|
||||
@@ -63,6 +83,7 @@ impl AppState {
|
||||
blockstore: Arc::new(MemoryBlockstore::new()),
|
||||
plc: PlcClient::new(plc_url),
|
||||
appview,
|
||||
firehose: Firehose::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -314,19 +314,32 @@ async fn upload_blob_rejects_oversized() {
|
||||
// 413 from axum's body extractor.
|
||||
let payload = vec![0u8; 2 * 1024 * 1024];
|
||||
|
||||
let resp = c
|
||||
// Two legitimate outcomes, and which one happens is a race the test
|
||||
// cannot win: the limit trips while the client is still writing the
|
||||
// 2 MiB body. If the rejection reaches the socket first, the client
|
||||
// reads `413`; if the server closes its side first, the client's
|
||||
// write fails with a connection reset and never gets to read a
|
||||
// status. Asserting only on `413` made this test fail roughly one run
|
||||
// in three. What actually matters — and what both outcomes prove — is
|
||||
// that the upload was refused rather than accepted.
|
||||
match c
|
||||
.post(format!("{}/xrpc/com.atproto.uploadBlob", PDS_URL))
|
||||
.bearer_auth(&jwt)
|
||||
.header("Content-Type", "image/png")
|
||||
.body(payload)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
resp.status().as_u16(),
|
||||
413,
|
||||
"oversized upload must return 413"
|
||||
);
|
||||
{
|
||||
Ok(resp) => assert_eq!(
|
||||
resp.status().as_u16(),
|
||||
413,
|
||||
"oversized upload must be refused with 413"
|
||||
),
|
||||
Err(e) => assert!(
|
||||
e.is_request(),
|
||||
"the only acceptable error is the server hanging up mid-body, got {e:?}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// `com.atproto.uploadBlob` rejects requests with no `Authorization`
|
||||
|
||||
@@ -0,0 +1,677 @@
|
||||
//! Integration tests for `com.atproto.sync.subscribeRepos`.
|
||||
//!
|
||||
//! Same contract as the other integration suites in this crate: they talk to a
|
||||
//! PDS listening on `127.0.0.1:2583` and **fail open** — if nothing answers
|
||||
//! `/healthz`, the test prints a note and returns green rather than failing a
|
||||
//! developer's `cargo test` on a machine with no server running. Start the
|
||||
//! server (`./target/debug/pds-server` with `.env` sourced) to actually
|
||||
//! exercise them.
|
||||
//!
|
||||
//! ## Why the frames are decoded by hand here
|
||||
//!
|
||||
//! `pds-server` is a binary, so a test crate cannot import its `dag_cbor`
|
||||
//! module — and that is a feature, not a limitation. These tests are the
|
||||
//! *consumer* side of the wire contract, and a consumer that reuses the
|
||||
//! producer's encoder proves nothing: it would happily agree with a frame
|
||||
//! that no other implementation can read. The decoder below is written from
|
||||
//! the spec (tag 42, identity prefix, length-first map keys) and knows
|
||||
//! nothing about how the server produced the bytes.
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::{SinkExt, StreamExt};
|
||||
use tokio_tungstenite::tungstenite::Message;
|
||||
|
||||
const PDS_URL: &str = "http://127.0.0.1:2583";
|
||||
const PDS_WS: &str = "ws://127.0.0.1:2583";
|
||||
|
||||
// -- harness ---------------------------------------------------------------
|
||||
|
||||
fn http() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn wait_for_pds() -> bool {
|
||||
let c = http();
|
||||
for _ in 0..20 {
|
||||
if let Ok(r) = c.get(format!("{PDS_URL}/healthz")).send().await {
|
||||
if r.status().is_success() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
async fn fresh_user(prefix: &str) -> (reqwest::Client, String, String) {
|
||||
let c = http();
|
||||
let handle = format!(
|
||||
"{}_{}.maarcadetweet.local",
|
||||
prefix,
|
||||
uuid::Uuid::new_v4().simple()
|
||||
);
|
||||
let acc: Value = c
|
||||
.post(format!("{PDS_URL}/xrpc/com.atproto.server.createAccount"))
|
||||
.json(&json!({"handle": handle, "password": "hunter2hunter2"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let did = acc["did"].as_str().expect("createAccount did").to_string();
|
||||
let jwt = acc["access_jwt"].as_str().expect("access_jwt").to_string();
|
||||
(c, did, jwt)
|
||||
}
|
||||
|
||||
async fn create_post(c: &reqwest::Client, did: &str, jwt: &str, text: &str) -> Value {
|
||||
c.post(format!("{PDS_URL}/xrpc/com.atproto.repo.createRecord"))
|
||||
.bearer_auth(jwt)
|
||||
.json(&json!({
|
||||
"repo": did,
|
||||
"collection": "app.twi.post",
|
||||
"record": { "text": text, "createdAt": "2026-09-10T12:00:00Z" },
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
type Socket = tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>;
|
||||
|
||||
async fn subscribe(cursor: Option<i64>) -> Socket {
|
||||
let url = match cursor {
|
||||
Some(c) => format!("{PDS_WS}/xrpc/com.atproto.sync.subscribeRepos?cursor={c}"),
|
||||
None => format!("{PDS_WS}/xrpc/com.atproto.sync.subscribeRepos"),
|
||||
};
|
||||
let (socket, _resp) = tokio_tungstenite::connect_async(&url)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("subscribeRepos connect to {url}: {e}"));
|
||||
socket
|
||||
}
|
||||
|
||||
/// Read the next **binary** message, or `None` on timeout.
|
||||
///
|
||||
/// Text messages would be a protocol violation on this endpoint and are
|
||||
/// asserted against rather than skipped.
|
||||
async fn next_frame(socket: &mut Socket) -> Option<Vec<u8>> {
|
||||
let deadline = Duration::from_secs(10);
|
||||
loop {
|
||||
match tokio::time::timeout(deadline, socket.next()).await {
|
||||
Err(_) => return None,
|
||||
Ok(None) => return None,
|
||||
Ok(Some(Ok(Message::Binary(b)))) => return Some(b),
|
||||
Ok(Some(Ok(Message::Ping(_)))) | Ok(Some(Ok(Message::Pong(_)))) => continue,
|
||||
Ok(Some(Ok(Message::Close(_)))) => return None,
|
||||
Ok(Some(Ok(other))) => panic!("subscribeRepos sent a non-binary frame: {other:?}"),
|
||||
Ok(Some(Err(e))) => panic!("subscribeRepos socket error: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Give the server a moment to finish `on_upgrade` and actually subscribe to
|
||||
/// the broadcast channel before we trigger a write.
|
||||
///
|
||||
/// The TCP handshake completing does not mean the handler has run. Without
|
||||
/// this the test would occasionally write before the subscription exists and
|
||||
/// then wait for a frame that was published to nobody. (The *cursor* replay
|
||||
/// path is the real fix for that race in production; the live-only test is
|
||||
/// deliberately testing the raceable path, so it waits.)
|
||||
async fn settle() {
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
|
||||
// -- an independent DAG-CBOR reader ----------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
enum Cbor {
|
||||
Null,
|
||||
Bool(bool),
|
||||
Int(i64),
|
||||
Bytes(Vec<u8>),
|
||||
Text(String),
|
||||
Array(Vec<Cbor>),
|
||||
Map(BTreeMap<String, Cbor>),
|
||||
/// tag(42) + bytes(0x00 || cid) — the binary CID is kept raw and
|
||||
/// re-encoded to a `bafy…` string on demand, so the test never depends on
|
||||
/// the server's own CID formatting.
|
||||
Link(Vec<u8>),
|
||||
}
|
||||
|
||||
impl Cbor {
|
||||
fn get(&self, key: &str) -> &Cbor {
|
||||
match self {
|
||||
Cbor::Map(m) => m
|
||||
.get(key)
|
||||
.unwrap_or_else(|| panic!("missing key `{key}` in {self:?}")),
|
||||
other => panic!("not a map: {other:?}"),
|
||||
}
|
||||
}
|
||||
fn opt(&self, key: &str) -> Option<&Cbor> {
|
||||
match self {
|
||||
Cbor::Map(m) => m.get(key),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
fn int(&self) -> i64 {
|
||||
match self {
|
||||
Cbor::Int(i) => *i,
|
||||
other => panic!("not an int: {other:?}"),
|
||||
}
|
||||
}
|
||||
fn text(&self) -> &str {
|
||||
match self {
|
||||
Cbor::Text(s) => s,
|
||||
other => panic!("not text: {other:?}"),
|
||||
}
|
||||
}
|
||||
fn bool(&self) -> bool {
|
||||
match self {
|
||||
Cbor::Bool(b) => *b,
|
||||
other => panic!("not a bool: {other:?}"),
|
||||
}
|
||||
}
|
||||
fn bytes(&self) -> &[u8] {
|
||||
match self {
|
||||
Cbor::Bytes(b) => b,
|
||||
other => panic!("not bytes: {other:?}"),
|
||||
}
|
||||
}
|
||||
fn array(&self) -> &[Cbor] {
|
||||
match self {
|
||||
Cbor::Array(a) => a,
|
||||
other => panic!("not an array: {other:?}"),
|
||||
}
|
||||
}
|
||||
/// The link's CID rendered as a base32 `bafy…` string, for comparison
|
||||
/// against what the XRPC JSON responses return.
|
||||
fn link_cid(&self) -> String {
|
||||
match self {
|
||||
Cbor::Link(raw) => cid::Cid::read_bytes(&raw[..])
|
||||
.expect("tag-42 payload must be a valid binary CID")
|
||||
.to_string(),
|
||||
other => panic!("not a link: {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_head(b: &[u8], p: usize) -> (u8, u64, usize) {
|
||||
let first = b[p];
|
||||
let major = first >> 5;
|
||||
let low = first & 0x1f;
|
||||
let (arg, extra) = match low {
|
||||
0..=23 => (low as u64, 0usize),
|
||||
24 => (b[p + 1] as u64, 1),
|
||||
25 => (u16::from_be_bytes([b[p + 1], b[p + 2]]) as u64, 2),
|
||||
26 => (
|
||||
u32::from_be_bytes([b[p + 1], b[p + 2], b[p + 3], b[p + 4]]) as u64,
|
||||
4,
|
||||
),
|
||||
27 => {
|
||||
let mut n = 0u64;
|
||||
for i in 0..8 {
|
||||
n = (n << 8) | b[p + 1 + i] as u64;
|
||||
}
|
||||
(n, 8)
|
||||
}
|
||||
other => panic!("indefinite or reserved CBOR head 0x{other:02x} — illegal in DAG-CBOR"),
|
||||
};
|
||||
(major, arg, p + 1 + extra)
|
||||
}
|
||||
|
||||
fn decode_at(b: &[u8], p: usize) -> (Cbor, usize) {
|
||||
let (major, arg, mut p) = read_head(b, p);
|
||||
match major {
|
||||
0 => (Cbor::Int(arg as i64), p),
|
||||
1 => (Cbor::Int(-(arg as i64) - 1), p),
|
||||
2 => {
|
||||
let end = p + arg as usize;
|
||||
(Cbor::Bytes(b[p..end].to_vec()), end)
|
||||
}
|
||||
3 => {
|
||||
let end = p + arg as usize;
|
||||
(
|
||||
Cbor::Text(std::str::from_utf8(&b[p..end]).unwrap().to_string()),
|
||||
end,
|
||||
)
|
||||
}
|
||||
4 => {
|
||||
let mut items = Vec::new();
|
||||
for _ in 0..arg {
|
||||
let (v, next) = decode_at(b, p);
|
||||
items.push(v);
|
||||
p = next;
|
||||
}
|
||||
(Cbor::Array(items), p)
|
||||
}
|
||||
5 => {
|
||||
let mut m = BTreeMap::new();
|
||||
let mut prev_key: Option<String> = None;
|
||||
for _ in 0..arg {
|
||||
let (k, next) = decode_at(b, p);
|
||||
p = next;
|
||||
let key = k.text().to_string();
|
||||
// Canonical DAG-CBOR order: shorter keys first, then
|
||||
// bytewise. Asserted here because a consumer written against
|
||||
// a strict codec (cborg's `dag-cbor` decoder, for one) will
|
||||
// reject a frame whose keys are out of order.
|
||||
if let Some(prev) = &prev_key {
|
||||
let ordered = (prev.len(), prev.as_bytes()) < (key.len(), key.as_bytes());
|
||||
assert!(ordered, "map keys out of canonical order: {prev:?} then {key:?}");
|
||||
}
|
||||
prev_key = Some(key.clone());
|
||||
let (v, next) = decode_at(b, p);
|
||||
p = next;
|
||||
m.insert(key, v);
|
||||
}
|
||||
(Cbor::Map(m), p)
|
||||
}
|
||||
6 => {
|
||||
assert_eq!(arg, 42, "DAG-CBOR permits only tag 42");
|
||||
let (inner, next) = decode_at(b, p);
|
||||
let raw = match inner {
|
||||
Cbor::Bytes(v) => v,
|
||||
other => panic!("tag 42 must wrap bytes, got {other:?}"),
|
||||
};
|
||||
assert_eq!(
|
||||
raw.first(),
|
||||
Some(&0x00),
|
||||
"a binary CID link must start with the 0x00 multibase identity prefix"
|
||||
);
|
||||
(Cbor::Link(raw[1..].to_vec()), next)
|
||||
}
|
||||
7 => match arg {
|
||||
20 => (Cbor::Bool(false), p),
|
||||
21 => (Cbor::Bool(true), p),
|
||||
22 => (Cbor::Null, p),
|
||||
other => panic!("unsupported CBOR simple value {other}"),
|
||||
},
|
||||
other => panic!("unsupported CBOR major type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Split one binary frame into its header and body values, and assert that
|
||||
/// the two together consume the whole message — a frame with trailing bytes
|
||||
/// would silently desynchronise a streaming consumer.
|
||||
fn parse_frame(bytes: &[u8]) -> (Cbor, Cbor) {
|
||||
let (header, after_header) = decode_at(bytes, 0);
|
||||
let (body, end) = decode_at(bytes, after_header);
|
||||
assert_eq!(end, bytes.len(), "frame must be exactly two CBOR values");
|
||||
(header, body)
|
||||
}
|
||||
|
||||
/// Minimal CAR v1 reader: returns the root CIDs and the block CIDs, both as
|
||||
/// `bafy…` strings.
|
||||
fn parse_car(bytes: &[u8]) -> (Vec<String>, Vec<String>) {
|
||||
fn varint(b: &[u8], p: &mut usize) -> u64 {
|
||||
let (mut v, mut shift) = (0u64, 0u32);
|
||||
loop {
|
||||
let byte = b[*p];
|
||||
*p += 1;
|
||||
v |= ((byte & 0x7f) as u64) << shift;
|
||||
if byte & 0x80 == 0 {
|
||||
return v;
|
||||
}
|
||||
shift += 7;
|
||||
}
|
||||
}
|
||||
let mut p = 0usize;
|
||||
let header_len = varint(bytes, &mut p) as usize;
|
||||
let header = &bytes[p..p + header_len];
|
||||
p += header_len;
|
||||
|
||||
// The CAR header is a CBOR map; walk it with the same head reader. Note
|
||||
// that this server's CAR header tags its roots *without* the 0x00
|
||||
// identity prefix (a documented deviation in `car.rs`), so the roots are
|
||||
// read as plain tagged byte strings rather than through `decode_at`.
|
||||
let mut hp = 0usize;
|
||||
let (major, n, next) = read_head(header, hp);
|
||||
assert_eq!(major, 5, "CAR header must be a map");
|
||||
hp = next;
|
||||
let mut roots = Vec::new();
|
||||
for _ in 0..n {
|
||||
let (m, len, next) = read_head(header, hp);
|
||||
assert_eq!(m, 3);
|
||||
hp = next;
|
||||
let key = std::str::from_utf8(&header[hp..hp + len as usize]).unwrap().to_string();
|
||||
hp += len as usize;
|
||||
if key == "version" {
|
||||
let (m, v, next) = read_head(header, hp);
|
||||
assert_eq!(m, 0);
|
||||
assert_eq!(v, 1, "CAR must be v1");
|
||||
hp = next;
|
||||
} else if key == "roots" {
|
||||
let (m, count, next) = read_head(header, hp);
|
||||
assert_eq!(m, 4);
|
||||
hp = next;
|
||||
for _ in 0..count {
|
||||
let (m, tag, next) = read_head(header, hp);
|
||||
assert_eq!((m, tag), (6, 42), "root must be a tag-42 link");
|
||||
hp = next;
|
||||
let (m, len, next) = read_head(header, hp);
|
||||
assert_eq!(m, 2);
|
||||
hp = next;
|
||||
// A DAG-CBOR link wraps `0x00 || <binary CID>`; the 0x00 is
|
||||
// the multibase identity prefix and is not part of the CID.
|
||||
// Assert on it rather than skipping it silently — this
|
||||
// reader stands in for a foreign consumer, and dropping the
|
||||
// check would let the header regress unnoticed.
|
||||
let raw = &header[hp..hp + len as usize];
|
||||
assert_eq!(
|
||||
raw.first(),
|
||||
Some(&0x00),
|
||||
"CAR root link must carry the multibase identity prefix"
|
||||
);
|
||||
roots.push(cid::Cid::read_bytes(&raw[1..]).unwrap().to_string());
|
||||
hp += len as usize;
|
||||
}
|
||||
} else {
|
||||
panic!("unexpected CAR header key {key}");
|
||||
}
|
||||
}
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
while p < bytes.len() {
|
||||
let section_len = varint(bytes, &mut p) as usize;
|
||||
let section = &bytes[p..p + section_len];
|
||||
let cid = cid::Cid::read_bytes(section).unwrap();
|
||||
blocks.push(cid.to_string());
|
||||
p += section_len;
|
||||
}
|
||||
(roots, blocks)
|
||||
}
|
||||
|
||||
// -- tests -----------------------------------------------------------------
|
||||
|
||||
/// A live subscriber receives a `#commit` frame for a record created after it
|
||||
/// connected, and every field of that frame says what it should.
|
||||
#[tokio::test]
|
||||
async fn live_subscriber_receives_a_commit_frame() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let (c, did, jwt) = fresh_user("fhlive").await;
|
||||
|
||||
let mut socket = subscribe(None).await;
|
||||
settle().await;
|
||||
|
||||
let created = create_post(&c, &did, &jwt, "hello firehose").await;
|
||||
let record_cid = created["cid"].as_str().expect("createRecord cid").to_string();
|
||||
let commit_cid = created["commit"]["cid"]
|
||||
.as_str()
|
||||
.expect("createRecord commit.cid")
|
||||
.to_string();
|
||||
let commit_rev = created["commit"]["rev"].as_str().unwrap().to_string();
|
||||
let uri = created["uri"].as_str().unwrap().to_string();
|
||||
let rkey = uri.rsplit('/').next().unwrap().to_string();
|
||||
|
||||
// The account was created moments ago and has never written before, so
|
||||
// the first frame we see for it is this post's. Other accounts may be
|
||||
// writing concurrently, so filter by DID rather than taking frame 1.
|
||||
let (header, body) = loop {
|
||||
let bytes = next_frame(&mut socket)
|
||||
.await
|
||||
.expect("expected a #commit frame within the timeout");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) == Some("#commit")
|
||||
&& body.get("repo").text() == did
|
||||
{
|
||||
break (header, body);
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(header.get("op").int(), 1, "regular frames carry op = 1");
|
||||
assert_eq!(header.get("t").text(), "#commit");
|
||||
|
||||
assert!(body.get("seq").int() > 0, "seq must be a real cursor value");
|
||||
assert_eq!(body.get("repo").text(), did);
|
||||
assert_eq!(body.get("rev").text(), commit_rev);
|
||||
assert_eq!(body.get("commit").link_cid(), commit_cid);
|
||||
assert!(!body.get("rebase").bool());
|
||||
assert!(!body.get("tooBig").bool());
|
||||
assert!(body.get("blobs").array().is_empty());
|
||||
// First-ever commit on a brand new repo — there is no predecessor.
|
||||
assert_eq!(*body.get("since"), Cbor::Null, "first commit has since = null");
|
||||
// RFC 3339 with a Z offset.
|
||||
let time = body.get("time").text();
|
||||
assert!(
|
||||
chrono_like_rfc3339(time),
|
||||
"time must be RFC 3339, got {time}"
|
||||
);
|
||||
|
||||
// ops
|
||||
let ops = body.get("ops").array();
|
||||
assert_eq!(ops.len(), 1, "one record write means one op");
|
||||
assert_eq!(ops[0].get("action").text(), "create");
|
||||
assert_eq!(ops[0].get("path").text(), format!("app.twi.post/{rkey}"));
|
||||
assert_eq!(ops[0].get("cid").link_cid(), record_cid);
|
||||
|
||||
// blocks: a CAR rooted at the commit, containing the commit block and the
|
||||
// new record block.
|
||||
let (roots, blocks) = parse_car(body.get("blocks").bytes());
|
||||
assert_eq!(roots, vec![commit_cid.clone()], "CAR root is the commit");
|
||||
assert!(
|
||||
blocks.contains(&commit_cid),
|
||||
"CAR must carry the commit block itself; got {blocks:?}"
|
||||
);
|
||||
assert!(
|
||||
blocks.contains(&record_cid),
|
||||
"CAR must carry the new record block; got {blocks:?}"
|
||||
);
|
||||
|
||||
let _ = socket.send(Message::Close(None)).await;
|
||||
}
|
||||
|
||||
/// The same event, fetched again from the durable log with a cursor, is
|
||||
/// byte-identical to the live frame.
|
||||
///
|
||||
/// Byte-identity is the strong form of the claim and the one that matters: a
|
||||
/// consumer that deduplicates by hashing frames, or that verifies a signature
|
||||
/// over them, must not see two different representations of one event.
|
||||
#[tokio::test]
|
||||
async fn cursor_replay_reproduces_the_live_frame_exactly() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let (c, did, jwt) = fresh_user("fhreplay").await;
|
||||
|
||||
let mut live = subscribe(None).await;
|
||||
settle().await;
|
||||
create_post(&c, &did, &jwt, "replay me").await;
|
||||
|
||||
let live_bytes = loop {
|
||||
let bytes = next_frame(&mut live)
|
||||
.await
|
||||
.expect("expected a live #commit frame");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) == Some("#commit")
|
||||
&& body.get("repo").text() == did
|
||||
{
|
||||
break bytes;
|
||||
}
|
||||
};
|
||||
let (_h, live_body) = parse_frame(&live_bytes);
|
||||
let seq = live_body.get("seq").int();
|
||||
let _ = live.send(Message::Close(None)).await;
|
||||
|
||||
// Reconnect asking for everything after the event *before* ours, so the
|
||||
// replay's first matching frame is the one we just saw.
|
||||
let mut replayed = subscribe(Some(seq - 1)).await;
|
||||
let replay_bytes = loop {
|
||||
let bytes = next_frame(&mut replayed)
|
||||
.await
|
||||
.expect("expected the event to come back from the replay");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) == Some("#commit")
|
||||
&& body.get("seq").int() == seq
|
||||
{
|
||||
break bytes;
|
||||
}
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
replay_bytes, live_bytes,
|
||||
"a replayed frame must be byte-identical to the live one"
|
||||
);
|
||||
let _ = replayed.send(Message::Close(None)).await;
|
||||
}
|
||||
|
||||
/// Handing over from replay to live loses nothing and duplicates nothing.
|
||||
///
|
||||
/// Connect with a cursor at the current head (so the replay is empty), then
|
||||
/// write twice: both events must arrive, in order, exactly once each.
|
||||
#[tokio::test]
|
||||
async fn replay_to_live_handover_has_no_gap_and_no_duplicate() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let (c, did, jwt) = fresh_user("fhhandover").await;
|
||||
|
||||
// Establish where the log currently ends by writing one event and reading
|
||||
// its seq off the live stream.
|
||||
let mut probe = subscribe(None).await;
|
||||
settle().await;
|
||||
create_post(&c, &did, &jwt, "probe").await;
|
||||
let head_seq = loop {
|
||||
let bytes = next_frame(&mut probe).await.expect("probe frame");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) == Some("#commit")
|
||||
&& body.get("repo").text() == did
|
||||
{
|
||||
break body.get("seq").int();
|
||||
}
|
||||
};
|
||||
let _ = probe.send(Message::Close(None)).await;
|
||||
|
||||
// Now reconnect at that exact cursor: nothing to replay, straight to live.
|
||||
let mut socket = subscribe(Some(head_seq)).await;
|
||||
settle().await;
|
||||
|
||||
create_post(&c, &did, &jwt, "after handover one").await;
|
||||
create_post(&c, &did, &jwt, "after handover two").await;
|
||||
|
||||
let mut seen: Vec<i64> = Vec::new();
|
||||
while seen.len() < 2 {
|
||||
let bytes = next_frame(&mut socket)
|
||||
.await
|
||||
.expect("expected both post-handover frames");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) != Some("#commit") {
|
||||
continue;
|
||||
}
|
||||
let seq = body.get("seq").int();
|
||||
assert!(
|
||||
seq > head_seq,
|
||||
"the cursor said we already had seq {head_seq}; got {seq} again"
|
||||
);
|
||||
if body.get("repo").text() == did {
|
||||
assert!(!seen.contains(&seq), "event {seq} delivered twice");
|
||||
seen.push(seq);
|
||||
}
|
||||
}
|
||||
assert_eq!(seen.len(), 2);
|
||||
assert!(seen[0] < seen[1], "events must arrive in seq order: {seen:?}");
|
||||
let _ = socket.send(Message::Close(None)).await;
|
||||
}
|
||||
|
||||
/// A cursor past the end of the log is a terminal error frame, not silence.
|
||||
#[tokio::test]
|
||||
async fn future_cursor_gets_an_error_frame() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let mut socket = subscribe(Some(i64::MAX / 2)).await;
|
||||
let bytes = next_frame(&mut socket)
|
||||
.await
|
||||
.expect("expected an error frame for a future cursor");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
assert_eq!(header.get("op").int(), -1, "error frames carry op = -1");
|
||||
assert!(
|
||||
header.opt("t").is_none(),
|
||||
"an error header has no `t`, only `op`"
|
||||
);
|
||||
assert_eq!(body.get("error").text(), "FutureCursor");
|
||||
assert!(
|
||||
!body.get("message").text().is_empty(),
|
||||
"the error should say what went wrong"
|
||||
);
|
||||
}
|
||||
|
||||
/// A record deleted through `deleteRecord` produces a `delete` op with a null
|
||||
/// CID — the one op shape that is not a link.
|
||||
#[tokio::test]
|
||||
async fn delete_produces_a_delete_op_with_a_null_cid() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let (c, did, jwt) = fresh_user("fhdelete").await;
|
||||
let created = create_post(&c, &did, &jwt, "to be deleted").await;
|
||||
let rkey = created["uri"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.rsplit('/')
|
||||
.next()
|
||||
.unwrap()
|
||||
.to_string();
|
||||
|
||||
let mut socket = subscribe(None).await;
|
||||
settle().await;
|
||||
|
||||
let resp = c
|
||||
.post(format!("{PDS_URL}/xrpc/com.atproto.repo.deleteRecord"))
|
||||
.bearer_auth(&jwt)
|
||||
.json(&json!({
|
||||
"repo": did,
|
||||
"collection": "app.twi.post",
|
||||
"rkey": rkey,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(resp.status().is_success(), "deleteRecord: {:?}", resp.status());
|
||||
|
||||
let (_header, body) = loop {
|
||||
let bytes = next_frame(&mut socket).await.expect("expected a delete frame");
|
||||
let (header, body) = parse_frame(&bytes);
|
||||
if header.opt("t").map(|t| t.text()) == Some("#commit")
|
||||
&& body.get("repo").text() == did
|
||||
{
|
||||
break (header, body);
|
||||
}
|
||||
};
|
||||
|
||||
let ops = body.get("ops").array();
|
||||
assert_eq!(ops.len(), 1);
|
||||
assert_eq!(ops[0].get("action").text(), "delete");
|
||||
assert_eq!(ops[0].get("path").text(), format!("app.twi.post/{rkey}"));
|
||||
assert_eq!(*ops[0].get("cid"), Cbor::Null, "a delete has no resulting CID");
|
||||
|
||||
// The second commit on this repo, so `since` is the previous revision.
|
||||
assert!(
|
||||
matches!(body.get("since"), Cbor::Text(_)),
|
||||
"a follow-up commit must name its predecessor's rev, got {:?}",
|
||||
body.get("since")
|
||||
);
|
||||
let _ = socket.send(Message::Close(None)).await;
|
||||
}
|
||||
|
||||
/// Shape check for the frame's `time`: RFC 3339, UTC, with a `Z` suffix.
|
||||
fn chrono_like_rfc3339(s: &str) -> bool {
|
||||
chrono::DateTime::parse_from_rfc3339(s).is_ok() && s.ends_with('Z')
|
||||
}
|
||||
@@ -0,0 +1,793 @@
|
||||
//! Invite-code enforcement on `com.atproto.server.createAccount`.
|
||||
//!
|
||||
//! # Why this file starts its own PDS
|
||||
//!
|
||||
//! Every other integration suite in this crate talks to whatever PDS the
|
||||
//! developer already has running on `:2583` and skips itself when there
|
||||
//! isn't one. That works because those tests only need *a* PDS. These
|
||||
//! need a PDS with `PDS_INVITE_REQUIRED=true`, and the ambient one is
|
||||
//! (correctly) started with the default `false` — otherwise every other
|
||||
//! suite, which creates throwaway accounts with no code, would fail.
|
||||
//!
|
||||
//! Asking the developer to restart their PDS with a different flag
|
||||
//! before this file passes would mean the flag's behaviour is only ever
|
||||
//! tested by hand. So each test here spawns its own `pds-server` on a
|
||||
//! free port with the flag set the way that test needs it, and kills it
|
||||
//! on the way out ([`Pds`]'s `Drop`). `env!("CARGO_BIN_EXE_pds-server")`
|
||||
//! is cargo's own path to the binary it just built for this test run, so
|
||||
//! the process under test is always the current code.
|
||||
//!
|
||||
//! The suite still fails open, in the same spirit as its neighbours: if
|
||||
//! the child never becomes healthy — no Postgres, no `.env`, no
|
||||
//! `DATABASE_URL_PDS` — the tests print why and return green rather than
|
||||
//! failing a workstation that simply isn't running the stack.
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use std::process::{Child, Command};
|
||||
use std::time::Duration;
|
||||
|
||||
/// A `pds-server` child process bound to its own port, killed when the
|
||||
/// test that started it goes out of scope.
|
||||
///
|
||||
/// The `Drop` impl is the reason this is a struct at all: a test that
|
||||
/// panics mid-way must not leave a server holding a port and a pool of
|
||||
/// Postgres connections for the rest of the run.
|
||||
struct Pds {
|
||||
child: Child,
|
||||
port: u16,
|
||||
http: reqwest::Client,
|
||||
}
|
||||
|
||||
impl Drop for Pds {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl Pds {
|
||||
fn url(&self, path: &str) -> String {
|
||||
format!("http://127.0.0.1:{}{}", self.port, path)
|
||||
}
|
||||
|
||||
async fn create_account(&self, body: Value) -> (u16, Value) {
|
||||
let resp = self
|
||||
.http
|
||||
.post(self.url("/xrpc/com.atproto.server.createAccount"))
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.expect("createAccount request");
|
||||
let status = resp.status().as_u16();
|
||||
let body: Value = resp.json().await.unwrap_or(Value::Null);
|
||||
(status, body)
|
||||
}
|
||||
|
||||
async fn describe(&self) -> Value {
|
||||
self.http
|
||||
.get(self.url("/xrpc/com.atproto.server.describeServer"))
|
||||
.send()
|
||||
.await
|
||||
.expect("describeServer")
|
||||
.json()
|
||||
.await
|
||||
.expect("describeServer json")
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask the OS for a port nobody is using, then let go of it.
|
||||
///
|
||||
/// There is a window between the drop and the child's `bind` in which
|
||||
/// something else could take the port; on a test machine that window is
|
||||
/// theoretical, and the alternative (a fixed port) would make two
|
||||
/// concurrently running tests in this file collide *reliably* instead of
|
||||
/// never.
|
||||
fn free_port() -> Option<u16> {
|
||||
let l = std::net::TcpListener::bind("127.0.0.1:0").ok()?;
|
||||
let p = l.local_addr().ok()?.port();
|
||||
drop(l);
|
||||
Some(p)
|
||||
}
|
||||
|
||||
/// Start a `pds-server` with `PDS_INVITE_REQUIRED` set to `required`.
|
||||
///
|
||||
/// Returns `None` when the stack this needs isn't available, which the
|
||||
/// callers turn into a skip. The child inherits the ambient environment
|
||||
/// (so `DATABASE_URL_PDS` and friends come from `.env` exactly as they
|
||||
/// do for the real server — `dotenvy` does not override real variables,
|
||||
/// so our overrides below win).
|
||||
async fn start_pds(required: bool) -> Option<Pds> {
|
||||
let port = free_port()?;
|
||||
let child = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.env("PDS_HOST", "127.0.0.1")
|
||||
.env("PDS_PORT", port.to_string())
|
||||
.env("PDS_PUBLIC_URL", format!("http://127.0.0.1:{port}"))
|
||||
.env("PDS_INVITE_REQUIRED", if required { "true" } else { "false" })
|
||||
// Point the PLC submit at a closed port. `create_account`
|
||||
// tolerates a failed submit by design (the DID is computed
|
||||
// locally), and a connection refused on loopback fails in
|
||||
// microseconds — whereas the real directory would add a network
|
||||
// round-trip to every account this file creates, and might
|
||||
// actually publish throwaway test DIDs.
|
||||
.env("PLC_DIRECTORY_URL", "http://127.0.0.1:1")
|
||||
.env("RUST_LOG", "warn")
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.spawn()
|
||||
.ok()?;
|
||||
|
||||
let http = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.build()
|
||||
.ok()?;
|
||||
let mut pds = Pds { child, port, http };
|
||||
|
||||
for _ in 0..80 {
|
||||
// If the child already exited (bad/missing env, no Postgres),
|
||||
// stop waiting — there is nothing to become healthy.
|
||||
if let Ok(Some(_)) = pds.child.try_wait() {
|
||||
return None;
|
||||
}
|
||||
if let Ok(r) = pds.http.get(pds.url("/healthz")).send().await {
|
||||
if r.status().is_success() {
|
||||
return Some(pds);
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
async fn db_pool() -> Option<sqlx::PgPool> {
|
||||
let url = std::env::var("DATABASE_URL_PDS")
|
||||
.unwrap_or_else(|_| "postgres://pds:pds@127.0.0.1:5434/pds".to_string());
|
||||
sqlx::postgres::PgPoolOptions::new()
|
||||
.max_connections(4)
|
||||
.acquire_timeout(Duration::from_secs(3))
|
||||
.connect(&url)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Put a code straight into the table in whatever state the test needs.
|
||||
///
|
||||
/// Tests seed through SQL rather than through `pds-server invite create`
|
||||
/// because the states that matter here — already spent, disabled — are
|
||||
/// not states the CLI can mint directly, and because a test that had to
|
||||
/// shell out to a second binary to arrange its fixture would be testing
|
||||
/// two things at once.
|
||||
async fn seed_code(db: &sqlx::PgPool, max_uses: i32, used: i32, disabled: bool) -> String {
|
||||
let code = format!("mt-test-{}", uuid::Uuid::new_v4().simple());
|
||||
sqlx::query(
|
||||
"INSERT INTO invite_codes (code, max_uses, used_count, disabled) VALUES ($1, $2, $3, $4)",
|
||||
)
|
||||
.bind(&code)
|
||||
.bind(max_uses)
|
||||
.bind(used)
|
||||
.bind(disabled)
|
||||
.execute(db)
|
||||
.await
|
||||
.expect("seed invite code");
|
||||
code
|
||||
}
|
||||
|
||||
async fn used_count(db: &sqlx::PgPool, code: &str) -> i32 {
|
||||
sqlx::query_scalar::<_, i32>("SELECT used_count FROM invite_codes WHERE code = $1")
|
||||
.bind(code)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
.expect("read used_count")
|
||||
}
|
||||
|
||||
async fn use_rows(db: &sqlx::PgPool, code: &str) -> Vec<(String, String)> {
|
||||
sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT did, handle FROM invite_code_uses WHERE code = $1 ORDER BY used_at",
|
||||
)
|
||||
.bind(code)
|
||||
.fetch_all(db)
|
||||
.await
|
||||
.expect("read invite_code_uses")
|
||||
}
|
||||
|
||||
/// A unique throwaway handle.
|
||||
///
|
||||
/// `createAccount` caps handles at 64 characters, and
|
||||
/// `<prefix>_<32 hex>.maarcadetweet.local` overshoots that for anything
|
||||
/// but the shortest prefix — a limit that shows up as a confusing
|
||||
/// `InvalidHandle` in a test that is about invite codes. Half the UUID
|
||||
/// is 64 bits of uniqueness, which is plenty for a test fixture and
|
||||
/// leaves room for a readable prefix.
|
||||
fn handle(prefix: &str) -> String {
|
||||
let uniq = uuid::Uuid::new_v4().simple().to_string();
|
||||
format!("{}_{}.maarcadetweet.local", prefix, &uniq[..16])
|
||||
}
|
||||
|
||||
/// Every invite rejection must look the same to a client: `400` with the
|
||||
/// module's usual `{error, message}` body and the name
|
||||
/// `InvalidInviteCode`.
|
||||
fn assert_invalid_invite(status: u16, body: &Value, what: &str) {
|
||||
assert_eq!(status, 400, "{what}: expected 400, body = {body}");
|
||||
assert_eq!(
|
||||
body["error"], "InvalidInviteCode",
|
||||
"{what}: wrong error name, body = {body}"
|
||||
);
|
||||
assert!(
|
||||
body["message"].is_string(),
|
||||
"{what}: error body must carry a message, body = {body}"
|
||||
);
|
||||
// No DID may have been minted on a rejected request.
|
||||
assert!(
|
||||
body["did"].is_null(),
|
||||
"{what}: rejected request returned a did, body = {body}"
|
||||
);
|
||||
}
|
||||
|
||||
// -- enforcement ------------------------------------------------------------
|
||||
|
||||
/// The happy path, and the property that makes a single-use code
|
||||
/// single-use: after the account exists, the same code is dead.
|
||||
///
|
||||
/// Also checks the audit trail, which is the reason
|
||||
/// `invite_code_uses` exists at all — "which account did this code
|
||||
/// create" has to be answerable after the fact.
|
||||
#[tokio::test]
|
||||
async fn valid_code_admits_one_account_then_is_spent() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
// The switch must be advertised, not just enforced — a client reads
|
||||
// this before it asks the user for anything.
|
||||
assert_eq!(
|
||||
pds.describe().await["invite_code_required"],
|
||||
json!(true),
|
||||
"describeServer must report the actual PDS_INVITE_REQUIRED value"
|
||||
);
|
||||
|
||||
let code = seed_code(&db, 1, 0, false).await;
|
||||
let h = handle("inv_ok");
|
||||
let (status, body) = pds
|
||||
.create_account(json!({
|
||||
"handle": h,
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(status, 200, "valid code must create an account: {body}");
|
||||
let did = body["did"].as_str().expect("did").to_string();
|
||||
|
||||
assert_eq!(used_count(&db, &code).await, 1, "code must be counted as used");
|
||||
let uses = use_rows(&db, &code).await;
|
||||
assert_eq!(uses.len(), 1);
|
||||
assert_eq!(uses[0].0, did, "audit row must name the account it created");
|
||||
assert_eq!(uses[0].1, h, "audit row must snapshot the handle");
|
||||
|
||||
// Second attempt on the now-spent code.
|
||||
let (status2, body2) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_second"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(status2, &body2, "spent code");
|
||||
assert_eq!(
|
||||
used_count(&db, &code).await,
|
||||
1,
|
||||
"a rejected attempt must not move the counter"
|
||||
);
|
||||
}
|
||||
|
||||
/// Every way a code can fail, and the missing-code case, all land on the
|
||||
/// same `400 InvalidInviteCode`.
|
||||
#[tokio::test]
|
||||
async fn unknown_disabled_spent_and_missing_codes_are_rejected() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
// Unknown.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_unknown"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": "mt-zzzzz-zzzzz",
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "unknown code");
|
||||
|
||||
// Disabled, with uses left — proves `disabled` is checked and not
|
||||
// just the counter.
|
||||
let disabled = seed_code(&db, 5, 0, true).await;
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_disabled"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": disabled,
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "disabled code");
|
||||
assert_eq!(used_count(&db, &disabled).await, 0);
|
||||
|
||||
// Already at its limit.
|
||||
let spent = seed_code(&db, 2, 2, false).await;
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_spent"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": spent,
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "exhausted code");
|
||||
|
||||
// No field at all.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_none"),
|
||||
"password": "hunter2hunter2",
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "missing code");
|
||||
|
||||
// Present but blank / whitespace — must be indistinguishable from
|
||||
// absent, not an attempt to look up the empty string.
|
||||
for blank in ["", " "] {
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_blank"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": blank,
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "blank code");
|
||||
}
|
||||
}
|
||||
|
||||
/// The camelCase spelling from the atproto lexicon, and the multi-use
|
||||
/// case the schema exists for.
|
||||
#[tokio::test]
|
||||
async fn camel_case_spelling_works_and_multi_use_codes_stop_at_the_limit() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
// `inviteCode` is what an off-the-shelf atproto client sends.
|
||||
let code = seed_code(&db, 3, 0, false).await;
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_camel"),
|
||||
"password": "hunter2hunter2",
|
||||
"inviteCode": code,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "inviteCode spelling must be accepted: {b}");
|
||||
|
||||
// Case and stray whitespace are normalised, so a code shouted or
|
||||
// pasted out of a chat window still works.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_case"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": format!(" {} ", code.to_uppercase()),
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "normalised code must be accepted: {b}");
|
||||
|
||||
// Third and last use.
|
||||
let (s, _) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_third"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200);
|
||||
|
||||
// Fourth is one too many.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_fourth"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_invalid_invite(s, &b, "one past max_uses");
|
||||
|
||||
assert_eq!(used_count(&db, &code).await, 3);
|
||||
assert_eq!(use_rows(&db, &code).await.len(), 3);
|
||||
}
|
||||
|
||||
/// A failed account creation must not consume the code.
|
||||
///
|
||||
/// The cheapest way to make the account creation fail *after* the
|
||||
/// redemption has already run is a handle that is already taken: the
|
||||
/// redeem happens first inside the transaction, the `users` insert then
|
||||
/// trips the unique index, and the whole transaction rolls back. If the
|
||||
/// redemption had been done outside the transaction (or committed
|
||||
/// separately) the user would have lost their code to someone else's
|
||||
/// handle.
|
||||
#[tokio::test]
|
||||
async fn a_failed_registration_does_not_burn_the_code() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let taken = handle("inv_taken");
|
||||
let first = seed_code(&db, 1, 0, false).await;
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": taken, "password": "hunter2hunter2", "invite_code": first,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "{b}");
|
||||
|
||||
// Now a *different* code, used on a handle that cannot be created.
|
||||
let code = seed_code(&db, 1, 0, false).await;
|
||||
let (s, _b) = pds
|
||||
.create_account(json!({
|
||||
"handle": taken, "password": "hunter2hunter2", "invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 409, "duplicate handle is still a 409");
|
||||
assert_eq!(
|
||||
used_count(&db, &code).await,
|
||||
0,
|
||||
"the code must survive a registration that rolled back"
|
||||
);
|
||||
assert!(use_rows(&db, &code).await.is_empty());
|
||||
|
||||
// And it still works afterwards.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_retry"), "password": "hunter2hunter2", "invite_code": code,
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "unburned code must still be redeemable: {b}");
|
||||
}
|
||||
|
||||
// -- the race ---------------------------------------------------------------
|
||||
|
||||
/// Two (here: eight) registrations arriving at the same instant on the
|
||||
/// last remaining use of a code. Exactly one may get in.
|
||||
///
|
||||
/// This is the test the whole design is built around. A
|
||||
/// `SELECT`-then-`UPDATE` implementation passes every other test in this
|
||||
/// file and fails this one: all eight requests read `used_count = 0`,
|
||||
/// all eight decide they are allowed, and the server hands out eight
|
||||
/// accounts for a one-use code while the row afterwards claims a single
|
||||
/// redemption. The fix is that `invite::redeem` never reads before it
|
||||
/// writes — the `WHERE used_count < max_uses` is part of the `UPDATE`,
|
||||
/// so Postgres re-evaluates it against the committed row after the
|
||||
/// row lock is released and the losers match zero rows.
|
||||
///
|
||||
/// Every request uses a distinct handle, so nothing but the invite code
|
||||
/// can be what serialises them.
|
||||
#[tokio::test]
|
||||
async fn concurrent_registrations_cannot_share_one_use() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
const N: usize = 8;
|
||||
let code = seed_code(&db, 1, 0, false).await;
|
||||
|
||||
let mut tasks = Vec::with_capacity(N);
|
||||
for i in 0..N {
|
||||
let http = pds.http.clone();
|
||||
let url = pds.url("/xrpc/com.atproto.server.createAccount");
|
||||
let code = code.clone();
|
||||
let h = handle(&format!("inv_race{i}"));
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let resp = http
|
||||
.post(url)
|
||||
.json(&json!({
|
||||
"handle": h,
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("concurrent createAccount");
|
||||
let status = resp.status().as_u16();
|
||||
let body: Value = resp.json().await.unwrap_or(Value::Null);
|
||||
(status, body)
|
||||
}));
|
||||
}
|
||||
|
||||
let mut ok = Vec::new();
|
||||
let mut rejected = 0usize;
|
||||
for t in tasks {
|
||||
let (status, body) = t.await.unwrap();
|
||||
match status {
|
||||
200 => ok.push(body),
|
||||
400 => {
|
||||
assert_eq!(body["error"], "InvalidInviteCode", "body = {body}");
|
||||
rejected += 1;
|
||||
}
|
||||
other => panic!("unexpected status {other}: {body}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
ok.len(),
|
||||
1,
|
||||
"a one-use code let {} concurrent registrations through — the redeem is racy",
|
||||
ok.len()
|
||||
);
|
||||
assert_eq!(rejected, N - 1);
|
||||
assert_eq!(used_count(&db, &code).await, 1);
|
||||
let uses = use_rows(&db, &code).await;
|
||||
assert_eq!(uses.len(), 1, "counter and audit rows disagree: {uses:?}");
|
||||
assert_eq!(uses[0].0, ok[0]["did"].as_str().unwrap());
|
||||
}
|
||||
|
||||
/// The same race with room for more than one winner: a three-use code
|
||||
/// hit by eight simultaneous registrations must admit exactly three.
|
||||
///
|
||||
/// Worth having next to the one-use case because an implementation can
|
||||
/// be "safe" by accident for a single use (e.g. by serialising every
|
||||
/// registration globally) and still lose count when several are
|
||||
/// genuinely allowed to proceed.
|
||||
#[tokio::test]
|
||||
async fn concurrent_registrations_respect_a_multi_use_limit() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
const N: usize = 8;
|
||||
const USES: i32 = 3;
|
||||
let code = seed_code(&db, USES, 0, false).await;
|
||||
|
||||
let mut tasks = Vec::with_capacity(N);
|
||||
for i in 0..N {
|
||||
let http = pds.http.clone();
|
||||
let url = pds.url("/xrpc/com.atproto.server.createAccount");
|
||||
let code = code.clone();
|
||||
let h = handle(&format!("inv_mrace{i}"));
|
||||
tasks.push(tokio::spawn(async move {
|
||||
let resp = http
|
||||
.post(url)
|
||||
.json(&json!({
|
||||
"handle": h,
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": code,
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.expect("concurrent createAccount");
|
||||
let status = resp.status().as_u16();
|
||||
let body: Value = resp.json().await.unwrap_or(Value::Null);
|
||||
(status, body)
|
||||
}));
|
||||
}
|
||||
|
||||
let mut ok = 0usize;
|
||||
for t in tasks {
|
||||
let (status, body) = t.await.unwrap();
|
||||
match status {
|
||||
200 => ok += 1,
|
||||
400 => assert_eq!(body["error"], "InvalidInviteCode", "body = {body}"),
|
||||
other => panic!("unexpected status {other}: {body}"),
|
||||
}
|
||||
}
|
||||
assert_eq!(ok, USES as usize, "a {USES}-use code admitted {ok} accounts");
|
||||
assert_eq!(used_count(&db, &code).await, USES);
|
||||
assert_eq!(use_rows(&db, &code).await.len(), USES as usize);
|
||||
}
|
||||
|
||||
// -- the switch off ---------------------------------------------------------
|
||||
|
||||
/// With `PDS_INVITE_REQUIRED=false` — the default, and what every other
|
||||
/// test suite in this workspace relies on — nothing about `createAccount`
|
||||
/// changes.
|
||||
///
|
||||
/// This is the regression test for the whole feature's blast radius: the
|
||||
/// switch is off by default precisely so that the existing suites keep
|
||||
/// creating accounts with no code, and if that ever stopped being true
|
||||
/// the failure would show up as dozens of unrelated tests breaking. It
|
||||
/// shows up here instead.
|
||||
#[tokio::test]
|
||||
async fn switch_off_leaves_create_account_untouched() {
|
||||
let Some(_db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
let Some(pds) = start_pds(false).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=false, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
pds.describe().await["invite_code_required"],
|
||||
json!(false),
|
||||
"describeServer must report the actual PDS_INVITE_REQUIRED value"
|
||||
);
|
||||
|
||||
// No code at all: the historical behaviour.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_off"),
|
||||
"password": "hunter2hunter2",
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "no-code registration must still work: {b}");
|
||||
assert!(b["did"].as_str().unwrap().starts_with("did:"));
|
||||
assert!(b["access_jwt"].is_string());
|
||||
|
||||
// A code that does not exist is simply ignored rather than becoming
|
||||
// a new way to fail — a client that was talking to an invite-only
|
||||
// PDS yesterday must not break when the operator opens the server up.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_off_bogus"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": "mt-does-notexist",
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "an ignored code must not fail the request: {b}");
|
||||
|
||||
// The other validations are untouched.
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_off_short"),
|
||||
"password": "short",
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 400);
|
||||
assert_eq!(b["error"], "InvalidPassword");
|
||||
}
|
||||
|
||||
// -- the CLI ----------------------------------------------------------------
|
||||
|
||||
/// `pds-server invite create` / `list` / `disable`, run as the operator
|
||||
/// would run them, against the real database.
|
||||
///
|
||||
/// The point is not that the SQL works (the tests above cover that) but
|
||||
/// that the *binary* exposes it: that `invite` short-circuits before the
|
||||
/// server starts, that `create` prints bare codes one per line so they
|
||||
/// can be pasted, and that a code it minted is actually redeemable.
|
||||
#[tokio::test]
|
||||
async fn invite_cli_mints_listable_redeemable_codes() {
|
||||
let Some(db) = db_pool().await else {
|
||||
eprintln!("no pds database, skipping");
|
||||
return;
|
||||
};
|
||||
|
||||
let out = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invite", "create", "--count", "3", "--uses", "2", "--note", "cli test"])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run invite create");
|
||||
if !out.status.success() {
|
||||
eprintln!(
|
||||
"invite create failed (no env/db?), skipping: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&out.stdout);
|
||||
let codes: Vec<&str> = stdout.lines().filter(|l| !l.trim().is_empty()).collect();
|
||||
assert_eq!(codes.len(), 3, "one code per line, nothing else: {stdout:?}");
|
||||
for c in &codes {
|
||||
// Bare, paste-ready: no labels, no quotes, no indentation.
|
||||
assert_eq!(*c, c.trim(), "code line has surrounding whitespace: {c:?}");
|
||||
assert!(c.starts_with("mt-"), "unexpected code shape: {c}");
|
||||
assert_eq!(
|
||||
sqlx::query_scalar::<_, i32>("SELECT max_uses FROM invite_codes WHERE code = $1")
|
||||
.bind(c)
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.expect("minted code must be in the table"),
|
||||
2,
|
||||
"--uses must reach the row"
|
||||
);
|
||||
}
|
||||
// All three distinct — a generator that returned a constant would
|
||||
// otherwise only show up as a primary-key error.
|
||||
let unique: std::collections::HashSet<&&str> = codes.iter().collect();
|
||||
assert_eq!(unique.len(), 3);
|
||||
|
||||
// `list` must show what `redeem` would accept.
|
||||
let listed = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invite", "list"])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run invite list");
|
||||
assert!(listed.status.success());
|
||||
let listed = String::from_utf8_lossy(&listed.stdout);
|
||||
for c in &codes {
|
||||
assert!(listed.contains(*c), "invite list omitted {c}");
|
||||
}
|
||||
|
||||
// `disable` takes a code out without deleting it.
|
||||
let disabled = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invite", "disable", codes[0]])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run invite disable");
|
||||
assert!(disabled.status.success());
|
||||
assert!(
|
||||
sqlx::query_scalar::<_, bool>("SELECT disabled FROM invite_codes WHERE code = $1")
|
||||
.bind(codes[0])
|
||||
.fetch_one(&db)
|
||||
.await
|
||||
.unwrap()
|
||||
);
|
||||
let listed = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invite", "list"])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run invite list");
|
||||
let listed = String::from_utf8_lossy(&listed.stdout);
|
||||
assert!(
|
||||
!listed.contains(codes[0]),
|
||||
"a disabled code must not show in the default listing"
|
||||
);
|
||||
|
||||
// And a minted code really lets an account through.
|
||||
let Some(pds) = start_pds(true).await else {
|
||||
eprintln!("could not start a pds with PDS_INVITE_REQUIRED=true, skipping redeem check");
|
||||
return;
|
||||
};
|
||||
let (s, b) = pds
|
||||
.create_account(json!({
|
||||
"handle": handle("inv_cli"),
|
||||
"password": "hunter2hunter2",
|
||||
"invite_code": codes[1],
|
||||
}))
|
||||
.await;
|
||||
assert_eq!(s, 200, "CLI-minted code must be redeemable: {b}");
|
||||
}
|
||||
|
||||
/// An unknown subcommand must not silently boot a server, and `help`
|
||||
/// must not need a database.
|
||||
#[tokio::test]
|
||||
async fn unknown_subcommand_fails_instead_of_starting_a_server() {
|
||||
let out = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invit"])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run bad subcommand");
|
||||
assert!(!out.status.success(), "a typo'd subcommand must not exit 0");
|
||||
let stderr = String::from_utf8_lossy(&out.stderr);
|
||||
assert!(stderr.contains("unknown command"), "stderr = {stderr}");
|
||||
|
||||
let out = Command::new(env!("CARGO_BIN_EXE_pds-server"))
|
||||
.args(["invite", "help"])
|
||||
.env("RUST_LOG", "warn")
|
||||
.output()
|
||||
.expect("run invite help");
|
||||
assert!(out.status.success());
|
||||
assert!(String::from_utf8_lossy(&out.stdout).contains("pds-server invite"));
|
||||
}
|
||||
@@ -38,9 +38,109 @@ async fn describe_server() {
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(r["did"].is_string());
|
||||
// The DID is derived from `PDS_PUBLIC_URL`, not hardcoded — so we
|
||||
// assert the *shape* (any deployment must produce a did:web) and
|
||||
// leave the exact value to `at_shared::config`'s unit tests.
|
||||
let did = r["did"].as_str().expect("describeServer must return a did");
|
||||
assert!(did.starts_with("did:web:"), "did = {did}");
|
||||
assert!(r["available_user_domains"].is_array());
|
||||
assert_eq!(r["invite_code_required"], json!(false));
|
||||
// `invite_code_required` used to be a hardcoded `false` here. It is
|
||||
// now whatever `PDS_INVITE_REQUIRED` says, so this suite — which
|
||||
// talks to whatever PDS the developer happens to be running — can
|
||||
// only assert the type. That the value actually tracks the switch is
|
||||
// pinned in `invite_integration.rs`, which starts a PDS with the
|
||||
// flag set both ways and checks both answers.
|
||||
assert!(
|
||||
r["invite_code_required"].is_boolean(),
|
||||
"invite_code_required = {}",
|
||||
r["invite_code_required"]
|
||||
);
|
||||
// If this test process shares the server's environment (the
|
||||
// documented way to run the suite is
|
||||
// `set -a; . ./.env; set +a; cargo test`), hold it to the exact
|
||||
// value too.
|
||||
if let Ok(raw) = std::env::var("PDS_INVITE_REQUIRED") {
|
||||
let expected = matches!(
|
||||
raw.trim().to_ascii_lowercase().as_str(),
|
||||
"1" | "true" | "yes" | "on"
|
||||
);
|
||||
assert_eq!(
|
||||
r["invite_code_required"],
|
||||
json!(expected),
|
||||
"describeServer disagrees with PDS_INVITE_REQUIRED={raw}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /.well-known/did.json` — the document the AppView fetches to
|
||||
/// learn the key our access tokens are signed with.
|
||||
///
|
||||
/// Two properties matter beyond "it returns JSON": the document's `id`
|
||||
/// must be the same DID `describeServer` advertises (otherwise a client
|
||||
/// that trusts one and resolves the other ends up at a different
|
||||
/// identity), and it must carry a usable `publicKeyMultibase`.
|
||||
#[tokio::test]
|
||||
async fn did_document_publishes_the_server_key() {
|
||||
if !wait_for_pds().await {
|
||||
eprintln!("pds not running, skipping");
|
||||
return;
|
||||
}
|
||||
let c = client().await;
|
||||
let doc: Value = c
|
||||
.get(format!("{}/.well-known/did.json", PDS_URL))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let id = doc["id"].as_str().expect("did document needs an id");
|
||||
assert!(id.starts_with("did:web:"), "id = {id}");
|
||||
|
||||
let described: Value = c
|
||||
.get(format!("{}/xrpc/com.atproto.server.describeServer", PDS_URL))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
described["did"].as_str().unwrap(),
|
||||
id,
|
||||
"describeServer and the did document must name the same identity"
|
||||
);
|
||||
|
||||
let vm = &doc["verificationMethod"][0];
|
||||
assert_eq!(vm["type"], json!("Multikey"));
|
||||
assert_eq!(vm["controller"], json!(id));
|
||||
assert_eq!(vm["id"], json!(format!("{id}#atproto")));
|
||||
let key = vm["publicKeyMultibase"]
|
||||
.as_str()
|
||||
.expect("verificationMethod needs publicKeyMultibase");
|
||||
// base58-btc multibase — the `z` prefix the AppView's decoder wants.
|
||||
assert!(key.starts_with('z'), "key = {key}");
|
||||
|
||||
// And it really is the key our tokens verify against: mint a
|
||||
// session and check the access JWT against the published key.
|
||||
let handle = format!("didjson_{}.maarcadetweet.local", uuid::Uuid::new_v4().simple());
|
||||
let acc: Value = c
|
||||
.post(format!("{}/xrpc/com.atproto.server.createAccount", PDS_URL))
|
||||
.json(&json!({"handle": handle, "password": "hunter2hunter2"}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
let jwt = acc["access_jwt"].as_str().expect("access_jwt");
|
||||
let claims = at_crypto::jwt::verify_jwt(jwt, key)
|
||||
.expect("access token must verify against the published key");
|
||||
assert_eq!(claims.sub, acc["did"].as_str().unwrap());
|
||||
assert_eq!(claims.scope.as_deref(), Some("com.atproto.access"));
|
||||
// `iss` is the same did:web the document identifies.
|
||||
assert_eq!(claims.iss, id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -360,7 +460,17 @@ fn parse_car(bytes: &[u8]) -> ParsedCar {
|
||||
assert_eq!(maj, 2, "root CID must be a byte string");
|
||||
p += c;
|
||||
let cid_bytes = &bytes[p..p + ln as usize];
|
||||
let cid_hex: String = cid_bytes
|
||||
// A DAG-CBOR link wraps `0x00 || <binary CID>`. The 0x00 is
|
||||
// the multibase identity prefix, not part of the CID, so it
|
||||
// comes off before the hex comparison against a real CID's
|
||||
// bytes. Asserted rather than skipped: this helper is the
|
||||
// only place the header's wire shape is checked.
|
||||
assert_eq!(
|
||||
cid_bytes.first(),
|
||||
Some(&0x00),
|
||||
"root link must carry the multibase identity prefix"
|
||||
);
|
||||
let cid_hex: String = cid_bytes[1..]
|
||||
.iter()
|
||||
.map(|b| format!("{:02x}", b))
|
||||
.collect();
|
||||
@@ -981,42 +1091,86 @@ async fn sync_list_repos_keyset_pagination() {
|
||||
assert!(resp["uri"].is_string(), "createRecord: {:?}", resp);
|
||||
created_dids.push(did);
|
||||
}
|
||||
let min_did = created_dids.iter().min().unwrap().clone();
|
||||
let start_cursor = did_cursor_lt(&min_did);
|
||||
// Two separate properties, deliberately not tested by one long walk
|
||||
// from the top of the table: `repos` grows without bound on a
|
||||
// long-lived instance (a few thousand rows here), the seeded DIDs are
|
||||
// random `did:plc:bafy…` hashes scattered across that range, and a
|
||||
// full scan at two rows per page ran into its own iteration cap —
|
||||
// failing for table size rather than for anything about pagination.
|
||||
|
||||
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
let mut cursor: Option<String> = Some(start_cursor);
|
||||
let mut pages = 0;
|
||||
loop {
|
||||
pages += 1;
|
||||
assert!(pages < 2000, "pagination did not terminate");
|
||||
// 1. Every seeded DID is reachable: anchor the cursor immediately
|
||||
// before it and it must be on the first page.
|
||||
for did in &created_dids {
|
||||
let url = format!(
|
||||
"{}/xrpc/com.atproto.sync.listRepos?limit=2&cursor={}",
|
||||
PDS_URL,
|
||||
urlencode(cursor.as_deref().unwrap_or(""))
|
||||
urlencode(&did_cursor_just_before(did))
|
||||
);
|
||||
let resp = client().await.get(&url).send().await.unwrap();
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let repos = body["repos"].as_array().expect("repos array");
|
||||
assert!(
|
||||
repos.iter().any(|r| r["did"].as_str() == Some(did.as_str())),
|
||||
"DID not on the page starting immediately before it: {did}"
|
||||
);
|
||||
}
|
||||
|
||||
// 2. The keyset itself: walking forward from the lowest seeded DID
|
||||
// yields strictly increasing DIDs, never a duplicate, and the
|
||||
// cursor the server hands back is always the last DID of the page.
|
||||
// A bounded number of pages is enough — these are properties of
|
||||
// every step, not of the whole table.
|
||||
let min_did = created_dids.iter().min().unwrap().clone();
|
||||
let mut cursor = did_cursor_just_before(&min_did);
|
||||
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
|
||||
let mut last: Option<String> = None;
|
||||
for _ in 0..25 {
|
||||
let url = format!(
|
||||
"{}/xrpc/com.atproto.sync.listRepos?limit=2&cursor={}",
|
||||
PDS_URL,
|
||||
urlencode(&cursor)
|
||||
);
|
||||
let resp = client().await.get(&url).send().await.unwrap();
|
||||
assert_eq!(resp.status().as_u16(), 200);
|
||||
let body: Value = resp.json().await.unwrap();
|
||||
let repos = body["repos"].as_array().expect("repos array");
|
||||
if repos.is_empty() {
|
||||
break;
|
||||
}
|
||||
for r in repos {
|
||||
let did = r["did"].as_str().unwrap().to_string();
|
||||
assert!(
|
||||
seen.insert(did.clone()),
|
||||
"duplicate DID across pages: {did}"
|
||||
);
|
||||
}
|
||||
if created_dids.iter().all(|d| seen.contains(d)) {
|
||||
break;
|
||||
if let Some(prev) = &last {
|
||||
assert!(
|
||||
&did > prev,
|
||||
"listRepos must be strictly ascending by DID: {prev} then {did}"
|
||||
);
|
||||
}
|
||||
last = Some(did);
|
||||
}
|
||||
match body["cursor"].as_str() {
|
||||
Some(c) => cursor = Some(c.to_string()),
|
||||
None => panic!(
|
||||
"pagination exhausted before all created DIDs were seen; missing {:?}",
|
||||
created_dids.iter().filter(|d| !seen.contains(*d)).collect::<Vec<_>>()
|
||||
),
|
||||
Some(c) => {
|
||||
assert_eq!(
|
||||
Some(c),
|
||||
last.as_deref(),
|
||||
"cursor must be the last DID of the page just served"
|
||||
);
|
||||
cursor = c.to_string();
|
||||
}
|
||||
// Fewer rows than the limit: the end of the table, and the
|
||||
// server correctly stops handing out a cursor.
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
seen.len() >= 2,
|
||||
"expected the walk to cover at least two pages, saw {}",
|
||||
seen.len()
|
||||
);
|
||||
}
|
||||
|
||||
fn urlencode(s: &str) -> String {
|
||||
|
||||
@@ -1,8 +1,30 @@
|
||||
//! Thin HTTP client the Tauri commands use to talk to the AppView.
|
||||
//!
|
||||
//! All four methods return parsed JSON or a stringified error that the
|
||||
//! Every method returns parsed JSON or a stringified error that the
|
||||
//! Tauri command layer surfaces to the Svelte frontend as the
|
||||
//! `Result::Err` payload.
|
||||
//!
|
||||
//! # Authentication
|
||||
//!
|
||||
//! The AppView's *viewer-scoped* endpoints require the account's access
|
||||
//! JWT in an `Authorization: Bearer` header, and additionally check that
|
||||
//! the token's `sub` equals the `did` query parameter:
|
||||
//!
|
||||
//! * `GET /api/timeline/home`
|
||||
//! * `GET /api/notifications`
|
||||
//! * `GET /api/notifications/count`
|
||||
//! * `POST /api/notifications/seen`
|
||||
//!
|
||||
//! Those four methods therefore take an `access_jwt` argument (last, the
|
||||
//! same position `pds_client.rs` uses). Everything else —
|
||||
//! `/api/profile*`, `/api/search`, `/api/post/{uri}`, `/api/thread`,
|
||||
//! `/api/followers`, `/api/following` — stays public and deliberately
|
||||
//! sends no token, so the read-only views keep working while logged out.
|
||||
//!
|
||||
//! On an auth failure the AppView answers `401` with
|
||||
//! `{"error":"AuthMissing"|"TokenInvalid","message":…}` or `403` with
|
||||
//! `{"error":"Forbidden",…}`. See [`status_error`] for why the response
|
||||
//! body must survive into the error message.
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use reqwest::Client;
|
||||
@@ -58,9 +80,10 @@ pub struct SearchResponse {
|
||||
///
|
||||
/// `like_count` and `repost_count` are included when the server
|
||||
/// resolves a real post; they're `None` for the "not in index"
|
||||
/// sentinel response (where `post` is null). The AppView has no
|
||||
/// auth yet, so we don't get `viewer_liked` / `viewer_reposted`
|
||||
/// from the server.
|
||||
/// sentinel response (where `post` is null). `/api/post/{uri}` is a
|
||||
/// public endpoint that takes no token, so there is no viewer to
|
||||
/// resolve against and we don't get `viewer_liked` /
|
||||
/// `viewer_reposted`; [`Self::fetch_thread`] with a `viewer_did` does.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ThreadResponse {
|
||||
pub post: Option<PostDto>,
|
||||
@@ -192,16 +215,22 @@ impl AppViewClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/timeline/home?did=&limit=&cursor=`
|
||||
/// `GET /api/timeline/home?did=&limit=&cursor=` — **authenticated**.
|
||||
///
|
||||
/// `access_jwt` goes out as `Authorization: Bearer`; the AppView
|
||||
/// rejects the call with 401 without it and with 403 when the
|
||||
/// token's `sub` doesn't match `did`.
|
||||
pub async fn fetch_timeline(
|
||||
&self,
|
||||
did: &str,
|
||||
cursor: Option<&str>,
|
||||
limit: u32,
|
||||
access_jwt: &str,
|
||||
) -> Result<TimelineResponse> {
|
||||
let mut req = self
|
||||
.client
|
||||
.get(format!("{}/api/timeline/home", self.base_url))
|
||||
.bearer_auth(access_jwt)
|
||||
.query(&[("did", did), ("limit", &limit.to_string())]);
|
||||
if let Some(c) = cursor {
|
||||
req = req.query(&[("cursor", c)]);
|
||||
@@ -211,13 +240,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send timeline request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: timeline home returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("timeline home", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<TimelineResponse>()
|
||||
@@ -244,13 +267,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send profile request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: profile returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("profile", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<ProfileResponse>()
|
||||
@@ -268,13 +285,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send profile-by-did request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: profile-by-did returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("profile-by-did", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<ProfileResponse>()
|
||||
@@ -292,13 +303,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send search request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: search returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("search", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<SearchResponse>()
|
||||
@@ -323,13 +328,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send post request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: post returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("post", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<ThreadResponse>()
|
||||
@@ -363,13 +362,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send thread request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: thread returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("thread", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<ThreadFullResponse>()
|
||||
@@ -377,17 +370,20 @@ impl AppViewClient {
|
||||
.context("appview: thread JSON parse")
|
||||
}
|
||||
|
||||
/// `GET /api/notifications?did=&limit=&cursor=` — newest first,
|
||||
/// same opaque-cursor pagination contract as the timeline.
|
||||
/// `GET /api/notifications?did=&limit=&cursor=` — **authenticated**;
|
||||
/// newest first, same opaque-cursor pagination contract as the
|
||||
/// timeline.
|
||||
pub async fn fetch_notifications(
|
||||
&self,
|
||||
did: &str,
|
||||
cursor: Option<&str>,
|
||||
limit: u32,
|
||||
access_jwt: &str,
|
||||
) -> Result<NotificationsResponse> {
|
||||
let mut req = self
|
||||
.client
|
||||
.get(format!("{}/api/notifications", self.base_url))
|
||||
.bearer_auth(access_jwt)
|
||||
.query(&[("did", did), ("limit", &limit.to_string())]);
|
||||
if let Some(c) = cursor {
|
||||
req = req.query(&[("cursor", c)]);
|
||||
@@ -397,13 +393,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.context("appview: failed to send notifications request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: notifications returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("notifications", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<NotificationsResponse>()
|
||||
@@ -411,25 +401,24 @@ impl AppViewClient {
|
||||
.context("appview: notifications JSON parse")
|
||||
}
|
||||
|
||||
/// `GET /api/notifications/count?did=` — unread count for the
|
||||
/// NavRail badge. Cheap enough to poll (partial index on the
|
||||
/// server side).
|
||||
pub async fn notification_count(&self, did: &str) -> Result<NotificationCountResponse> {
|
||||
/// `GET /api/notifications/count?did=` — **authenticated**; unread
|
||||
/// count for the NavRail badge. Cheap enough to poll (partial index
|
||||
/// on the server side).
|
||||
pub async fn notification_count(
|
||||
&self,
|
||||
did: &str,
|
||||
access_jwt: &str,
|
||||
) -> Result<NotificationCountResponse> {
|
||||
let resp = self
|
||||
.client
|
||||
.get(format!("{}/api/notifications/count", self.base_url))
|
||||
.bearer_auth(access_jwt)
|
||||
.query(&[("did", did)])
|
||||
.send()
|
||||
.await
|
||||
.context("appview: failed to send notification-count request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: notification count returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("notification count", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<NotificationCountResponse>()
|
||||
@@ -437,9 +426,10 @@ impl AppViewClient {
|
||||
.context("appview: notification count JSON parse")
|
||||
}
|
||||
|
||||
/// `POST /api/notifications/seen` — mark everything indexed at or
|
||||
/// before `seen_at` as read. Passing `None` marks *all* currently
|
||||
/// unread rows. Idempotent; a second call reports `updated: 0`.
|
||||
/// `POST /api/notifications/seen` — **authenticated**; mark
|
||||
/// everything indexed at or before `seen_at` as read. Passing `None`
|
||||
/// marks *all* currently unread rows. Idempotent; a second call
|
||||
/// reports `updated: 0`.
|
||||
///
|
||||
/// The server accepts both `seenAt` and `seen_at`; we send the
|
||||
/// camelCase spelling because that's what the wire contract
|
||||
@@ -448,6 +438,7 @@ impl AppViewClient {
|
||||
&self,
|
||||
did: &str,
|
||||
seen_at: Option<&str>,
|
||||
access_jwt: &str,
|
||||
) -> Result<NotificationSeenResponse> {
|
||||
let mut body = serde_json::json!({ "did": did });
|
||||
if let Some(ts) = seen_at {
|
||||
@@ -456,18 +447,13 @@ impl AppViewClient {
|
||||
let resp = self
|
||||
.client
|
||||
.post(format!("{}/api/notifications/seen", self.base_url))
|
||||
.bearer_auth(access_jwt)
|
||||
.json(&body)
|
||||
.send()
|
||||
.await
|
||||
.context("appview: failed to send notifications-seen request")?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"appview: notifications seen returned {}: {}",
|
||||
status,
|
||||
body
|
||||
));
|
||||
return Err(status_error("notifications seen", resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<NotificationSeenResponse>()
|
||||
@@ -517,9 +503,7 @@ impl AppViewClient {
|
||||
.await
|
||||
.with_context(|| format!("appview: failed to send {path} request"))?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
return Err(anyhow!("appview: {} returned {}: {}", path, status, body));
|
||||
return Err(status_error(path, resp).await);
|
||||
}
|
||||
resp
|
||||
.json::<ActorListResponse>()
|
||||
@@ -528,6 +512,33 @@ impl AppViewClient {
|
||||
}
|
||||
}
|
||||
|
||||
/// Turn a non-2xx AppView response into an `anyhow::Error` whose
|
||||
/// message carries the server's response body **verbatim**.
|
||||
///
|
||||
/// Keeping the body is load-bearing, not cosmetic. The AppView answers
|
||||
/// an expired or malformed access token with
|
||||
/// `401 {"error":"TokenInvalid","message":…}`, and the frontend's
|
||||
/// `safeInvoke` (`src/lib/api/client.ts`) decides whether to refresh the
|
||||
/// session and retry by sniffing the *stringified* Rust error for the
|
||||
/// literal substring `"TokenInvalid"` (or `"ExpiredSignature"`). The
|
||||
/// chain is therefore:
|
||||
///
|
||||
/// ```text
|
||||
/// AppView 401 body ──► status_error() ──► anyhow msg
|
||||
/// ──► lib.rs `.map_err(|e| e.to_string())` ──► Tauri IPC reject
|
||||
/// ──► safeInvoke's isTokenInvalid() ──► auth_refresh + retry once
|
||||
/// ```
|
||||
///
|
||||
/// Every link is a plain string, so swallowing the body here (e.g.
|
||||
/// formatting only the status code) silently breaks token renewal —
|
||||
/// the user's timeline just stops updating an hour after login. The
|
||||
/// unit tests below pin the substring so that can't regress.
|
||||
async fn status_error(label: &str, resp: reqwest::Response) -> anyhow::Error {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
anyhow!("appview: {} returned {}: {}", label, status, body)
|
||||
}
|
||||
|
||||
/// Percent-encode every byte of `s` for use as a URL path segment.
|
||||
/// `axum`'s path extractor will decode it back. We use this rather
|
||||
/// than `url::Url::parse(...).path_segments()` because AT-Protocol
|
||||
@@ -553,6 +564,9 @@ fn percent_encode_path(s: &str) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[test]
|
||||
fn percent_encode_path_at_uri() {
|
||||
@@ -563,4 +577,241 @@ mod tests {
|
||||
"at%3A%2F%2Fdid%3Aplc%3Aabc%2Fapp.twi.post%2F3k2"
|
||||
);
|
||||
}
|
||||
|
||||
// -- mock AppView -------------------------------------------------
|
||||
//
|
||||
// A ~40-line HTTP/1.1 server on an ephemeral port, rather than a
|
||||
// mocking crate, so the test adds no dependency to a workspace that
|
||||
// currently has none for this. It answers every request with a
|
||||
// canned status + body and records each request head so the tests
|
||||
// can assert on the `Authorization` header we did (or deliberately
|
||||
// did not) send.
|
||||
|
||||
/// The raw request heads (request line + header block) the mock saw,
|
||||
/// in arrival order.
|
||||
type Recorded = Arc<Mutex<Vec<String>>>;
|
||||
|
||||
/// Byte offset of the `\r\n\r\n` that ends the header block.
|
||||
fn headers_end(buf: &[u8]) -> Option<usize> {
|
||||
buf.windows(4).position(|w| w == b"\r\n\r\n")
|
||||
}
|
||||
|
||||
/// Announced body length from a request head, if any. Needed so the
|
||||
/// POST test drains the JSON body before the mock closes the socket
|
||||
/// — closing mid-write would surface to `reqwest` as a connection
|
||||
/// error instead of the 401 we're trying to assert on.
|
||||
fn content_length(head: &str) -> usize {
|
||||
head.lines()
|
||||
.find_map(|l| {
|
||||
let (k, v) = l.split_once(':')?;
|
||||
k.trim()
|
||||
.eq_ignore_ascii_case("content-length")
|
||||
.then(|| v.trim().parse::<usize>().ok())?
|
||||
})
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Serve `n` connections, answering each with `status`/`reason` and
|
||||
/// `body`. Returns the base URL to point an [`AppViewClient`] at,
|
||||
/// plus the recording handle.
|
||||
async fn spawn_mock(
|
||||
status: u16,
|
||||
reason: &'static str,
|
||||
body: &'static str,
|
||||
n: usize,
|
||||
) -> (String, Recorded) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let recorded: Recorded = Arc::new(Mutex::new(Vec::new()));
|
||||
let rec = recorded.clone();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..n {
|
||||
let Ok((mut sock, _)) = listener.accept().await else {
|
||||
return;
|
||||
};
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
let mut chunk = [0u8; 2048];
|
||||
loop {
|
||||
let read = match sock.read(&mut chunk).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(k) => k,
|
||||
};
|
||||
buf.extend_from_slice(&chunk[..read]);
|
||||
if let Some(p) = headers_end(&buf) {
|
||||
let head = String::from_utf8_lossy(&buf[..p]).into_owned();
|
||||
if buf.len() - (p + 4) >= content_length(&head) {
|
||||
rec.lock().unwrap().push(head);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// `connection: close` keeps every request on a fresh
|
||||
// socket, so the recorded order matches the call order.
|
||||
let resp = format!(
|
||||
"HTTP/1.1 {status} {reason}\r\n\
|
||||
content-type: application/json\r\n\
|
||||
content-length: {}\r\n\
|
||||
connection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
let _ = sock.write_all(resp.as_bytes()).await;
|
||||
let _ = sock.shutdown().await;
|
||||
}
|
||||
});
|
||||
(format!("http://{addr}"), recorded)
|
||||
}
|
||||
|
||||
/// Value of the `Authorization` header in a recorded request head,
|
||||
/// or `None` when the request carried none.
|
||||
fn auth_header(head: &str) -> Option<String> {
|
||||
head.lines().find_map(|l| {
|
||||
let (k, v) = l.split_once(':')?;
|
||||
k.trim()
|
||||
.eq_ignore_ascii_case("authorization")
|
||||
.then(|| v.trim().to_string())
|
||||
})
|
||||
}
|
||||
|
||||
// -- the contract the TS retry chain depends on -------------------
|
||||
|
||||
/// The AppView's 401 body for an expired access token.
|
||||
const TOKEN_INVALID_BODY: &str =
|
||||
r#"{"error":"TokenInvalid","message":"ExpiredSignature"}"#;
|
||||
|
||||
/// **This is the test the token-renewal path hangs off.**
|
||||
///
|
||||
/// `src/lib/api/client.ts`'s `safeInvoke` refreshes the session and
|
||||
/// retries exactly once when `isTokenInvalid(e)` matches — and that
|
||||
/// predicate is a substring search for `"TokenInvalid"` /
|
||||
/// `"ExpiredSignature"` over the *stringified* error that came up
|
||||
/// from Rust. The AppView only ever states the code in its JSON
|
||||
/// body, so if [`status_error`] were to drop the body (or truncate
|
||||
/// it to the status code) the retry would never fire and the desktop
|
||||
/// client would sit on a dead timeline until the user restarts it.
|
||||
///
|
||||
/// So: for each of the four authenticated endpoints, assert the code
|
||||
/// survives verbatim all the way into `Error::to_string()` — which
|
||||
/// is exactly what `lib.rs` hands the Tauri IPC layer via
|
||||
/// `.map_err(|e| e.to_string())`.
|
||||
#[tokio::test]
|
||||
async fn token_invalid_code_survives_into_the_error_string() {
|
||||
let (base, _rec) = spawn_mock(401, "Unauthorized", TOKEN_INVALID_BODY, 4).await;
|
||||
let c = AppViewClient::new(base);
|
||||
|
||||
let errs = vec![
|
||||
c.fetch_timeline("did:plc:me", None, 30, "stale")
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
c.fetch_notifications("did:plc:me", None, 30, "stale")
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
c.notification_count("did:plc:me", "stale")
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
c.mark_notifications_seen("did:plc:me", None, "stale")
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
];
|
||||
|
||||
for e in &errs {
|
||||
// The literal the TS `isTokenInvalid()` greps for. Both
|
||||
// spellings it accepts are in this body.
|
||||
assert!(
|
||||
e.contains("TokenInvalid"),
|
||||
"error must carry the AppView's code verbatim, got: {e}"
|
||||
);
|
||||
assert!(
|
||||
e.contains("ExpiredSignature"),
|
||||
"error must carry the AppView's message verbatim, got: {e}"
|
||||
);
|
||||
// The status is useful context, but it is NOT what the retry
|
||||
// keys off — asserting it here documents that both travel.
|
||||
assert!(e.contains("401"), "status should travel too, got: {e}");
|
||||
}
|
||||
|
||||
// Each endpoint still labels itself, so a log line says which
|
||||
// call failed.
|
||||
assert!(errs[0].contains("timeline home"));
|
||||
assert!(errs[1].contains("notifications"));
|
||||
assert!(errs[2].contains("notification count"));
|
||||
assert!(errs[3].contains("notifications seen"));
|
||||
}
|
||||
|
||||
/// The 403 the AppView returns when the token is valid but its
|
||||
/// `sub` doesn't match the `did` query parameter. Deliberately
|
||||
/// *not* something `isTokenInvalid` matches: refreshing wouldn't
|
||||
/// help, so the retry must not fire — but the code still has to
|
||||
/// reach the UI so the message is actionable.
|
||||
#[tokio::test]
|
||||
async fn forbidden_body_survives_and_does_not_look_refreshable() {
|
||||
let (base, _rec) = spawn_mock(
|
||||
403,
|
||||
"Forbidden",
|
||||
r#"{"error":"Forbidden","message":"did does not match token subject"}"#,
|
||||
1,
|
||||
)
|
||||
.await;
|
||||
let c = AppViewClient::new(base);
|
||||
|
||||
let e = c
|
||||
.fetch_timeline("did:plc:someone-else", None, 30, "good-jwt")
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
|
||||
assert!(e.contains("Forbidden"), "got: {e}");
|
||||
assert!(e.contains("403"), "got: {e}");
|
||||
assert!(!e.contains("TokenInvalid"));
|
||||
assert!(!e.contains("ExpiredSignature"));
|
||||
}
|
||||
|
||||
/// The four viewer-scoped endpoints must actually put the JWT on
|
||||
/// the wire — an empty or missing header is a 401 from the server.
|
||||
#[tokio::test]
|
||||
async fn authenticated_endpoints_send_the_bearer_header() {
|
||||
let (base, rec) = spawn_mock(500, "Internal Server Error", "{}", 4).await;
|
||||
let c = AppViewClient::new(base);
|
||||
|
||||
let _ = c.fetch_timeline("did:plc:me", None, 30, "jwt-abc").await;
|
||||
let _ = c.fetch_notifications("did:plc:me", None, 30, "jwt-abc").await;
|
||||
let _ = c.notification_count("did:plc:me", "jwt-abc").await;
|
||||
let _ = c
|
||||
.mark_notifications_seen("did:plc:me", Some("2026-09-09T10:00:00Z"), "jwt-abc")
|
||||
.await;
|
||||
|
||||
let heads = rec.lock().unwrap().clone();
|
||||
assert_eq!(heads.len(), 4, "every call should have reached the server");
|
||||
for h in &heads {
|
||||
assert_eq!(
|
||||
auth_header(h).as_deref(),
|
||||
Some("Bearer jwt-abc"),
|
||||
"missing/!= bearer token in:\n{h}"
|
||||
);
|
||||
}
|
||||
// The POST still carries its JSON body alongside the header.
|
||||
assert!(heads[3].starts_with("POST /api/notifications/seen"));
|
||||
}
|
||||
|
||||
/// The public half of the API must keep working while logged out,
|
||||
/// so it must not grow an `Authorization` header by accident.
|
||||
#[tokio::test]
|
||||
async fn public_endpoints_send_no_authorization_header() {
|
||||
let (base, rec) = spawn_mock(200, "OK", r#"{"posts":[],"q":"hi"}"#, 1).await;
|
||||
let c = AppViewClient::new(base);
|
||||
let r = c.fetch_search("hi", 30).await.unwrap();
|
||||
assert_eq!(r.q, "hi");
|
||||
assert!(auth_header(&rec.lock().unwrap()[0]).is_none());
|
||||
|
||||
let (base2, rec2) = spawn_mock(200, "OK", r#"{"profiles":[],"cursor":null}"#, 2).await;
|
||||
let c2 = AppViewClient::new(base2);
|
||||
c2.fetch_followers("did:plc:me", None, 30).await.unwrap();
|
||||
c2.fetch_following("did:plc:me", None, 30).await.unwrap();
|
||||
for h in rec2.lock().unwrap().iter() {
|
||||
assert!(auth_header(h).is_none(), "unexpected auth header in:\n{h}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,15 +34,32 @@ async fn pds_describe(state: tauri::State<'_, AppState>) -> Result<serde_json::V
|
||||
state.pds.describe_server().await.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// `auth_register(handle, password, inviteCode?)` — create an account
|
||||
/// on the configured PDS and store the resulting session.
|
||||
///
|
||||
/// `invite_code` arrives from the frontend as `inviteCode` (Tauri maps
|
||||
/// camelCase JS argument keys onto snake_case Rust parameters, the same
|
||||
/// way `mark_notifications_seen` receives `seenAt`). It is `Option`
|
||||
/// because the invite gate is a *server* setting
|
||||
/// (`PDS_INVITE_REQUIRED`): the public instance at
|
||||
/// `https://tweet.maarcade.com` demands a code, a locally run dev PDS
|
||||
/// usually does not, and the client has no business deciding which.
|
||||
/// When no code is given the field is dropped from the request body
|
||||
/// rather than sent empty — see [`pds_client::CreateAccountReq`].
|
||||
///
|
||||
/// A refused code surfaces as the PDS's `400
|
||||
/// {"error":"InvalidInviteCode", …}` inside the stringified error, and
|
||||
/// `errorMessage()` in `client.ts` turns that into German copy.
|
||||
#[tauri::command]
|
||||
async fn auth_register(
|
||||
state: tauri::State<'_, AppState>,
|
||||
handle: String,
|
||||
password: String,
|
||||
invite_code: Option<String>,
|
||||
) -> Result<AccountSession, String> {
|
||||
let sess = state
|
||||
.pds
|
||||
.create_account(&handle, &password)
|
||||
.create_account(&handle, &password, invite_code.as_deref())
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let s = AccountSession {
|
||||
@@ -250,7 +267,12 @@ async fn repost_post(
|
||||
});
|
||||
let resp = state
|
||||
.pds
|
||||
.create_record_with(&sess.did, "app.bsky.feed.repost", record, false, &sess.access_jwt)
|
||||
// `true`: `app.bsky.feed.repost` is in the PDS's lexicon registry
|
||||
// and this record passes it (verified against a live PDS). The
|
||||
// `false` that stood here was inert — the flag was dropped before
|
||||
// the request — so validating is what has actually been happening
|
||||
// all along; saying so keeps the behaviour and drops the fiction.
|
||||
.create_record_with(&sess.did, "app.bsky.feed.repost", record, true, &sess.access_jwt)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(serde_json::json!({
|
||||
@@ -354,6 +376,28 @@ async fn unfollow_user(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Access JWT for the AppView's viewer-scoped endpoints.
|
||||
///
|
||||
/// The four authenticated AppView calls (`timeline_home`,
|
||||
/// `fetch_notifications`, `notification_count`,
|
||||
/// `mark_notifications_seen`) all need the same thing: the stored
|
||||
/// session's access JWT, or a message the UI can render when there
|
||||
/// isn't one. Factored out so no call site can accidentally send an
|
||||
/// empty `Authorization: Bearer` header — which the AppView would
|
||||
/// answer with a 401 `TokenInvalid`, and the frontend would then burn a
|
||||
/// pointless refresh round trip on before failing anyway.
|
||||
///
|
||||
/// The message deliberately contains neither `TokenInvalid` nor
|
||||
/// `ExpiredSignature`: `safeInvoke`'s `isTokenInvalid()` greps for those
|
||||
/// substrings, and a logged-out client has nothing to refresh *with*.
|
||||
fn require_access_jwt(state: &AppState, what: &str) -> Result<String, String> {
|
||||
state
|
||||
.store
|
||||
.load()
|
||||
.map(|s| s.access_jwt)
|
||||
.ok_or_else(|| format!("not logged in: {what} requires a signed-in session"))
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
async fn timeline_home(
|
||||
state: tauri::State<'_, AppState>,
|
||||
@@ -362,9 +406,10 @@ async fn timeline_home(
|
||||
limit: Option<u32>,
|
||||
) -> Result<appview_client::TimelineResponse, String> {
|
||||
let lim = limit.unwrap_or(30).clamp(1, 100);
|
||||
let jwt = require_access_jwt(&state, "the home timeline")?;
|
||||
state
|
||||
.appview
|
||||
.fetch_timeline(&did, cursor.as_deref(), lim)
|
||||
.fetch_timeline(&did, cursor.as_deref(), lim, &jwt)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -460,9 +505,10 @@ async fn fetch_notifications(
|
||||
limit: Option<u32>,
|
||||
) -> Result<appview_client::NotificationsResponse, String> {
|
||||
let lim = limit.unwrap_or(30).clamp(1, 100);
|
||||
let jwt = require_access_jwt(&state, "notifications")?;
|
||||
state
|
||||
.appview
|
||||
.fetch_notifications(&did, cursor.as_deref(), lim)
|
||||
.fetch_notifications(&did, cursor.as_deref(), lim, &jwt)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -473,9 +519,10 @@ async fn notification_count(
|
||||
state: tauri::State<'_, AppState>,
|
||||
did: String,
|
||||
) -> Result<appview_client::NotificationCountResponse, String> {
|
||||
let jwt = require_access_jwt(&state, "the unread-notification count")?;
|
||||
state
|
||||
.appview
|
||||
.notification_count(&did)
|
||||
.notification_count(&did, &jwt)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -491,9 +538,10 @@ async fn mark_notifications_seen(
|
||||
did: String,
|
||||
seen_at: Option<String>,
|
||||
) -> Result<appview_client::NotificationSeenResponse, String> {
|
||||
let jwt = require_access_jwt(&state, "marking notifications seen")?;
|
||||
state
|
||||
.appview
|
||||
.mark_notifications_seen(&did, seen_at.as_deref())
|
||||
.mark_notifications_seen(&did, seen_at.as_deref(), &jwt)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -737,16 +785,74 @@ async fn show_notification(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Default PDS base URL — the public instance.
|
||||
///
|
||||
/// This is what a *shipped* build talks to. It used to be
|
||||
/// `http://127.0.0.1:2583`, which meant a packaged `.app` handed to
|
||||
/// anyone but the developer pointed at a server that does not exist on
|
||||
/// their machine: every call failed with a connection error and the
|
||||
/// login screen could not even render `describeServer`. A default is
|
||||
/// the configuration of the people who never set one, so it has to be
|
||||
/// the production deployment.
|
||||
///
|
||||
/// No trailing slash: [`PdsHttpClient`] builds its endpoints as
|
||||
/// `{base}/xrpc/com.atproto.…`, so the base must end at the host.
|
||||
const DEFAULT_PDS_URL: &str = "https://tweet.maarcade.com";
|
||||
|
||||
/// Default AppView base URL. Same host as the PDS — the reverse proxy
|
||||
/// in front of `tweet.maarcade.com` routes by path prefix: `/xrpc/…`
|
||||
/// to the PDS, `/api/…` to the AppView. [`AppViewClient`] appends
|
||||
/// `/api/…` to this base (see the `format!("{}/api/…", self.base_url)`
|
||||
/// calls in `appview_client.rs`), so the two clients can and must
|
||||
/// share the one origin.
|
||||
const DEFAULT_APPVIEW_URL: &str = "https://tweet.maarcade.com";
|
||||
|
||||
/// Resolve one base URL from its environment variable, falling back to
|
||||
/// the compiled-in default.
|
||||
///
|
||||
/// **The environment always wins.** Development runs against a local
|
||||
/// stack — `MAARCADETWEET_PDS_URL=http://127.0.0.1:2583` and
|
||||
/// `MAARCADETWEET_APPVIEW_URL=http://127.0.0.1:2584`, which is what
|
||||
/// `scripts/` and the dev docker-compose set up — and pointing the
|
||||
/// desktop client at it must stay a matter of exporting two variables,
|
||||
/// never of rebuilding. Only an *unset* variable takes the production
|
||||
/// default.
|
||||
///
|
||||
/// A variable set to whitespace (or the empty string) counts as unset:
|
||||
/// an empty base URL would silently produce request URLs like
|
||||
/// `/xrpc/…` with no host, and `reqwest` would reject them as a
|
||||
/// relative-URL error far away from the actual mistake. Trailing
|
||||
/// slashes are trimmed because both clients append an absolute path to
|
||||
/// this string, and `https://host//api/x` is not the same route to
|
||||
/// every proxy.
|
||||
///
|
||||
/// Takes the already-performed lookup rather than the variable name so
|
||||
/// it stays a pure function — testable without mutating the process
|
||||
/// environment, and without a Tauri runtime.
|
||||
fn base_url_or_default(from_env: Result<String, std::env::VarError>, default: &str) -> String {
|
||||
let configured = from_env.ok();
|
||||
let trimmed = configured
|
||||
.as_deref()
|
||||
.map(|v| v.trim().trim_end_matches('/'))
|
||||
.filter(|v| !v.is_empty());
|
||||
trimmed.unwrap_or(default).to_string()
|
||||
}
|
||||
|
||||
#[cfg_attr(mobile, tauri::mobile_entry_point)]
|
||||
pub fn run() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "info".into()))
|
||||
.init();
|
||||
|
||||
let pds_url = std::env::var("MAARCADETWEET_PDS_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:2583".to_string());
|
||||
let appview_url = std::env::var("MAARCADETWEET_APPVIEW_URL")
|
||||
.unwrap_or_else(|_| "http://127.0.0.1:2584".to_string());
|
||||
let pds_url = base_url_or_default(
|
||||
std::env::var("MAARCADETWEET_PDS_URL"),
|
||||
DEFAULT_PDS_URL,
|
||||
);
|
||||
let appview_url = base_url_or_default(
|
||||
std::env::var("MAARCADETWEET_APPVIEW_URL"),
|
||||
DEFAULT_APPVIEW_URL,
|
||||
);
|
||||
tracing::info!(%pds_url, %appview_url, "resolved backend base URLs");
|
||||
|
||||
let state = AppState {
|
||||
pds: PdsHttpClient::new(pds_url.clone()),
|
||||
@@ -1025,3 +1131,75 @@ async fn profile_set(
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::env::VarError;
|
||||
|
||||
/// The whole point of the change: a build with nothing configured
|
||||
/// must talk to the public instance, not to a loopback port that
|
||||
/// only exists on a developer's laptop. Pinned as a literal so a
|
||||
/// well-meant "let's default back to localhost for dev" has to
|
||||
/// argue with a red test first.
|
||||
#[test]
|
||||
fn unset_env_falls_back_to_the_public_instance() {
|
||||
assert_eq!(
|
||||
base_url_or_default(Err(VarError::NotPresent), DEFAULT_PDS_URL),
|
||||
"https://tweet.maarcade.com"
|
||||
);
|
||||
assert_eq!(
|
||||
base_url_or_default(Err(VarError::NotPresent), DEFAULT_APPVIEW_URL),
|
||||
"https://tweet.maarcade.com"
|
||||
);
|
||||
// Both services live behind the same origin — the proxy splits
|
||||
// them by path prefix (`/xrpc/` vs `/api/`), which the clients
|
||||
// append themselves.
|
||||
assert_eq!(DEFAULT_PDS_URL, DEFAULT_APPVIEW_URL);
|
||||
assert!(!DEFAULT_PDS_URL.ends_with('/'));
|
||||
}
|
||||
|
||||
/// Development against a local stack has to keep working by
|
||||
/// exporting a variable, so a set value always beats the default.
|
||||
#[test]
|
||||
fn env_var_overrides_the_default() {
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok("http://127.0.0.1:2583".into()), DEFAULT_PDS_URL),
|
||||
"http://127.0.0.1:2583"
|
||||
);
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok("http://127.0.0.1:2584".into()), DEFAULT_APPVIEW_URL),
|
||||
"http://127.0.0.1:2584"
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty or whitespace-only variable is a misconfiguration, not
|
||||
/// a request for an empty base URL: `reqwest` would answer the
|
||||
/// resulting host-less URL with a relative-URL error nowhere near
|
||||
/// the cause. Trailing slashes go because both clients append an
|
||||
/// absolute path (`{base}/xrpc/…`, `{base}/api/…`).
|
||||
#[test]
|
||||
fn blank_env_is_ignored_and_trailing_slashes_are_trimmed() {
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok("".into()), DEFAULT_PDS_URL),
|
||||
DEFAULT_PDS_URL
|
||||
);
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok(" ".into()), DEFAULT_PDS_URL),
|
||||
DEFAULT_PDS_URL
|
||||
);
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok("http://127.0.0.1:2584/".into()), DEFAULT_APPVIEW_URL),
|
||||
"http://127.0.0.1:2584"
|
||||
);
|
||||
assert_eq!(
|
||||
base_url_or_default(Ok(" https://tweet.maarcade.com// ".into()), DEFAULT_PDS_URL),
|
||||
"https://tweet.maarcade.com"
|
||||
);
|
||||
// A non-UTF-8 variable is as unusable as an unset one.
|
||||
assert_eq!(
|
||||
base_url_or_default(Err(VarError::NotUnicode("\u{fffd}".into())), DEFAULT_PDS_URL),
|
||||
DEFAULT_PDS_URL
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,6 +26,27 @@ pub struct CreateAccountReq {
|
||||
pub handle: String,
|
||||
pub email: Option<String>,
|
||||
pub password: String,
|
||||
/// Invite code, required by the PDS whenever it runs with
|
||||
/// `PDS_INVITE_REQUIRED=true` (the public instance at
|
||||
/// `https://tweet.maarcade.com` does). Rejected codes come back as
|
||||
/// `400 {"error":"InvalidInviteCode", …}`.
|
||||
///
|
||||
/// **Wire name.** The server's `CreateAccountReq`
|
||||
/// (`crates/pds-server/src/routes/types.rs`) is snake_case with an
|
||||
/// `#[serde(alias = "inviteCode")]` for off-the-shelf atproto
|
||||
/// clients. We are not one of those: every body this client sends
|
||||
/// is snake_case (`refresh_jwt` in `refresh_session`, `handle` /
|
||||
/// `password` right here), so `invite_code` is the field name that
|
||||
/// matches the rest of the file. The alias exists for other people.
|
||||
///
|
||||
/// **Skipped when `None`.** Same reasoning as `validate` below —
|
||||
/// a PDS with the invite gate *off* must keep accepting our
|
||||
/// registrations, and sending `"invite_code": null` (let alone
|
||||
/// `""`) would be claiming the user supplied something. Omitting
|
||||
/// the key leaves the server's `Option` at `None`, which is exactly
|
||||
/// "the user gave no code".
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub invite_code: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
@@ -41,6 +62,10 @@ pub struct CreateRecordReq {
|
||||
pub repo: String,
|
||||
pub collection: String,
|
||||
pub record: serde_json::Value,
|
||||
/// Omitted rather than sent as `null` when the caller has no
|
||||
/// opinion — the PDS's own default (`true`) then applies.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub validate: Option<bool>,
|
||||
}
|
||||
|
||||
/// Strong reference as defined by
|
||||
@@ -126,15 +151,31 @@ impl PdsHttpClient {
|
||||
Ok(r)
|
||||
}
|
||||
|
||||
/// `com.atproto.server.createAccount`.
|
||||
///
|
||||
/// `invite_code` is `None` when the user left the field empty; the
|
||||
/// key is then left out of the body entirely (see
|
||||
/// [`CreateAccountReq::invite_code`]) so a PDS running without the
|
||||
/// invite gate still registers the account. A code that only
|
||||
/// contains whitespace is treated as absent for the same reason —
|
||||
/// the server's `invite::normalize` trims before looking it up, so
|
||||
/// `" "` could never match a real code anyway, and passing it on
|
||||
/// would only turn "you forgot the field" into "your code is
|
||||
/// wrong".
|
||||
pub async fn create_account(
|
||||
&self,
|
||||
handle: &str,
|
||||
password: &str,
|
||||
invite_code: Option<&str>,
|
||||
) -> Result<AccountSession> {
|
||||
let body = CreateAccountReq {
|
||||
handle: handle.to_string(),
|
||||
email: None,
|
||||
password: password.to_string(),
|
||||
invite_code: invite_code
|
||||
.map(str::trim)
|
||||
.filter(|c| !c.is_empty())
|
||||
.map(str::to_string),
|
||||
};
|
||||
let r = self
|
||||
.client
|
||||
@@ -184,12 +225,19 @@ impl PdsHttpClient {
|
||||
Ok(r.json().await?)
|
||||
}
|
||||
|
||||
/// `validate` is forwarded to the PDS, which defaults it to `true`.
|
||||
///
|
||||
/// It used to be `_validate` — accepted and silently dropped, so a
|
||||
/// caller asking for `false` still got server-side validation. That
|
||||
/// made no difference in practice (every collection the client writes
|
||||
/// is in the PDS's lexicon registry and passes), but a parameter that
|
||||
/// does nothing is a trap for the next caller who relies on it.
|
||||
pub async fn create_record_with(
|
||||
&self,
|
||||
repo: &str,
|
||||
collection: &str,
|
||||
record: serde_json::Value,
|
||||
_validate: bool,
|
||||
validate: bool,
|
||||
jwt: &str,
|
||||
) -> Result<CreateRecordResp> {
|
||||
let r = self
|
||||
@@ -200,6 +248,7 @@ impl PdsHttpClient {
|
||||
repo: repo.to_string(),
|
||||
collection: collection.to_string(),
|
||||
record,
|
||||
validate: Some(validate),
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
@@ -6,6 +6,8 @@
|
||||
fetchTimeline,
|
||||
fetchSearch,
|
||||
fetchPost,
|
||||
errorMessage,
|
||||
isAuthFailure,
|
||||
notificationCount,
|
||||
openExternalUrl,
|
||||
showError,
|
||||
@@ -377,6 +379,16 @@
|
||||
/// Pull the unread count for the NavRail badge. Swallows errors:
|
||||
/// the badge is ambient information, and a transient AppView hiccup
|
||||
/// shouldn't produce a toast every 5 seconds.
|
||||
///
|
||||
/// One class of error is *not* swallowed-and-retried, though. Since
|
||||
/// the AppView started requiring the access JWT on
|
||||
/// `/api/notifications/count`, a rejected token surfaces here — and
|
||||
/// by the time it does, `safeInvoke` has already spent its one
|
||||
/// automatic refresh. Retrying on a 5s timer would then be a request
|
||||
/// loop against a server that keeps answering 401/403 for as long as
|
||||
/// the app is open. So an auth failure stops the poll outright; the
|
||||
/// next successful login restarts it via the `session.subscribe`
|
||||
/// handler in `onMount`.
|
||||
async function refreshUnreadCount() {
|
||||
if (!currentUser) return;
|
||||
// While the notifications view is open the user is by definition
|
||||
@@ -386,8 +398,13 @@
|
||||
if (view === "notifications") return;
|
||||
try {
|
||||
unreadCount = await notificationCount(currentUser.did);
|
||||
} catch {
|
||||
/* ignore — keep the last known count */
|
||||
} catch (e) {
|
||||
if (isAuthFailure(e)) {
|
||||
console.warn("notification poll stopped: session rejected", e);
|
||||
stopPoll();
|
||||
return;
|
||||
}
|
||||
/* otherwise ignore — keep the last known count */
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,7 +437,7 @@
|
||||
if (fresh.length > 0) userPosts = [...fresh, ...userPosts];
|
||||
}
|
||||
} catch (e) {
|
||||
timelineError = String(e);
|
||||
timelineError = errorMessage(e);
|
||||
// Keep whatever we had on a transient failure.
|
||||
} finally {
|
||||
timelineLoading = false;
|
||||
@@ -440,7 +457,7 @@
|
||||
}
|
||||
timelineCursor = r.cursor;
|
||||
} catch (e) {
|
||||
timelineError = String(e);
|
||||
timelineError = errorMessage(e);
|
||||
} finally {
|
||||
timelineLoading = false;
|
||||
}
|
||||
@@ -528,17 +545,23 @@
|
||||
// Used in the Settings view to show which backends the client is
|
||||
// talking to. Kept as plain helpers so they can be swapped for a
|
||||
// `pds_describe`/`appview_describe` Tauri command later.
|
||||
// The fallbacks mirror `DEFAULT_PDS_URL` / `DEFAULT_APPVIEW_URL` in
|
||||
// `src-tauri/src/lib.rs`: both services sit behind the one public
|
||||
// origin, split by path prefix (`/xrpc/` → PDS, `/api/` → AppView).
|
||||
// If those constants ever move, move these with them — a Settings
|
||||
// pane that names the wrong backend is worse than one that names
|
||||
// none.
|
||||
function pdsBase(): string {
|
||||
if (typeof import.meta !== "undefined" && (import.meta as any).env?.VITE_PDS_URL) {
|
||||
return (import.meta as any).env.VITE_PDS_URL as string;
|
||||
}
|
||||
return "http://127.0.0.1:2583";
|
||||
return "https://tweet.maarcade.com";
|
||||
}
|
||||
function appviewBase(): string {
|
||||
if (typeof import.meta !== "undefined" && (import.meta as any).env?.VITE_APPVIEW_URL) {
|
||||
return (import.meta as any).env.VITE_APPVIEW_URL as string;
|
||||
}
|
||||
return "http://127.0.0.1:2584";
|
||||
return "https://tweet.maarcade.com";
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
// The AppView auth contract, from the client's side.
|
||||
//
|
||||
// Same setup as `notifications.test.ts`: `@tauri-apps/api/core` is
|
||||
// mocked so no Tauri shell is needed, and every assertion is about the
|
||||
// exact sequence of commands we hand the Rust IPC layer.
|
||||
//
|
||||
// What's pinned here:
|
||||
// * the **token-renewal chain** — a `TokenInvalid` coming out of the
|
||||
// AppView (not the PDS) triggers exactly one `auth_refresh` + one
|
||||
// retry, for each of the four now-authenticated endpoints;
|
||||
// * that the chain fires for a **bare string** rejection, which is
|
||||
// what `invoke` actually rejects with for our `Result<T, String>`
|
||||
// commands — the shape the old `typeof e !== "object"` guard
|
||||
// silently skipped;
|
||||
// * that it fires **once**, never in a loop, and not at all when the
|
||||
// refresh itself fails or when the error isn't refreshable;
|
||||
// * that the **public** endpoints still work with no session at all
|
||||
// and never reach for a refresh.
|
||||
//
|
||||
// The error strings below are verbatim what the Rust side produces:
|
||||
// `appview_client.rs`'s `status_error()` formats
|
||||
// `"appview: {label} returned {status}: {body}"`, and `lib.rs`
|
||||
// stringifies that into the command's `Err(String)`. The Rust test
|
||||
// `token_invalid_code_survives_into_the_error_string` pins the other
|
||||
// half of the same contract.
|
||||
//
|
||||
// Run with:
|
||||
// npx vitest run src/lib/api/appview-auth.test.ts
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const invokeMock = vi.fn();
|
||||
|
||||
vi.mock("@tauri-apps/api/core", () => ({
|
||||
invoke: (...args: unknown[]) => invokeMock(...args),
|
||||
isTauri: () => true,
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
invokeMock.mockReset();
|
||||
});
|
||||
|
||||
/// Exactly what a Tauri command rejects with once the AppView has
|
||||
/// refused an expired access token: a bare string, because our
|
||||
/// commands are `Result<T, String>` and `invoke` rejects with the
|
||||
/// deserialised payload — not an `Error`.
|
||||
function appviewTokenInvalid(label: string): string {
|
||||
return (
|
||||
`appview: ${label} returned 401 Unauthorized: ` +
|
||||
`{"error":"TokenInvalid","message":"ExpiredSignature"}`
|
||||
);
|
||||
}
|
||||
|
||||
const FRESH_SESSION = {
|
||||
did: "did:plc:me",
|
||||
handle: "me.test",
|
||||
access_jwt: "fresh-access",
|
||||
refresh_jwt: "fresh-refresh",
|
||||
};
|
||||
|
||||
/// The four endpoints that grew an auth guard, each with the command
|
||||
/// name the Rust side registers, the AppView's label in the error
|
||||
/// string, a caller, and the payload the retry should resolve with.
|
||||
const AUTHED = [
|
||||
{
|
||||
name: "timeline_home",
|
||||
label: "timeline home",
|
||||
payload: { posts: [], cursor: null },
|
||||
call: async () => {
|
||||
const { fetchTimeline } = await import("./client");
|
||||
return fetchTimeline("did:plc:me");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "fetch_notifications",
|
||||
label: "notifications",
|
||||
payload: { notifications: [], cursor: null },
|
||||
call: async () => {
|
||||
const { fetchNotifications } = await import("./client");
|
||||
return fetchNotifications("did:plc:me");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "notification_count",
|
||||
label: "notification count",
|
||||
payload: { count: 3 },
|
||||
call: async () => {
|
||||
const { notificationCount } = await import("./client");
|
||||
return notificationCount("did:plc:me");
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mark_notifications_seen",
|
||||
label: "notifications seen",
|
||||
payload: { ok: true, updated: 2 },
|
||||
call: async () => {
|
||||
const { markNotificationsSeen } = await import("./client");
|
||||
return markNotificationsSeen("did:plc:me", "2026-09-09T10:00:00Z");
|
||||
},
|
||||
},
|
||||
] as const;
|
||||
|
||||
describe("AppView token renewal", () => {
|
||||
for (const ep of AUTHED) {
|
||||
it(`${ep.name}: a TokenInvalid from the AppView refreshes and retries once`, async () => {
|
||||
invokeMock
|
||||
// 1. the call, rejected by the AppView's auth guard
|
||||
.mockRejectedValueOnce(appviewTokenInvalid(ep.label))
|
||||
// 2. auth_refresh mints a new access JWT from the refresh JWT
|
||||
.mockResolvedValueOnce(FRESH_SESSION)
|
||||
// 3. the same call again, now with the fresh token
|
||||
.mockResolvedValueOnce(ep.payload);
|
||||
|
||||
await expect(ep.call()).resolves.toBeDefined();
|
||||
|
||||
expect(invokeMock.mock.calls.map((c) => c[0])).toEqual([
|
||||
ep.name,
|
||||
"auth_refresh",
|
||||
ep.name,
|
||||
]);
|
||||
// The retry must repeat the *same* argument bag — a dropped
|
||||
// cursor or limit here would silently change what the user sees.
|
||||
expect(invokeMock.mock.calls[0][1]).toEqual(invokeMock.mock.calls[2][1]);
|
||||
});
|
||||
}
|
||||
|
||||
it("returns the retry's payload, not the failed first attempt", async () => {
|
||||
const { notificationCount } = await import("./client");
|
||||
invokeMock
|
||||
.mockRejectedValueOnce(appviewTokenInvalid("notification count"))
|
||||
.mockResolvedValueOnce(FRESH_SESSION)
|
||||
.mockResolvedValueOnce({ count: 7 });
|
||||
|
||||
await expect(notificationCount("did:plc:me")).resolves.toBe(7);
|
||||
});
|
||||
|
||||
it("fires for a bare-string rejection — the shape Tauri actually uses", async () => {
|
||||
// Regression guard. `invoke` rejects with the deserialised
|
||||
// `Err(String)` payload, i.e. a primitive string. A guard that
|
||||
// bails on anything that isn't an object never sees the code and
|
||||
// the retry silently never runs — the user's timeline just dies an
|
||||
// hour after login with no error anyone would connect to auth.
|
||||
const { fetchTimeline } = await import("./client");
|
||||
invokeMock
|
||||
.mockRejectedValueOnce(appviewTokenInvalid("timeline home"))
|
||||
.mockResolvedValueOnce(FRESH_SESSION)
|
||||
.mockResolvedValueOnce({ posts: [], cursor: null });
|
||||
|
||||
await expect(fetchTimeline("did:plc:me")).resolves.toEqual({
|
||||
posts: [],
|
||||
cursor: null,
|
||||
});
|
||||
expect(invokeMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("also fires when the error arrives as an Error object", async () => {
|
||||
const { fetchTimeline } = await import("./client");
|
||||
invokeMock
|
||||
.mockRejectedValueOnce(new Error(appviewTokenInvalid("timeline home")))
|
||||
.mockResolvedValueOnce(FRESH_SESSION)
|
||||
.mockResolvedValueOnce({ posts: [], cursor: null });
|
||||
|
||||
await expect(fetchTimeline("did:plc:me")).resolves.toBeDefined();
|
||||
expect(invokeMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("retries exactly once — a still-failing retry is not refreshed again", async () => {
|
||||
const { fetchNotifications } = await import("./client");
|
||||
invokeMock
|
||||
.mockRejectedValueOnce(appviewTokenInvalid("notifications"))
|
||||
.mockResolvedValueOnce(FRESH_SESSION)
|
||||
.mockRejectedValueOnce(appviewTokenInvalid("notifications"));
|
||||
|
||||
await expect(fetchNotifications("did:plc:me")).rejects.toThrow(
|
||||
/TokenInvalid/,
|
||||
);
|
||||
// Three calls, not five: no second refresh, no third attempt.
|
||||
expect(invokeMock).toHaveBeenCalledTimes(3);
|
||||
expect(invokeMock.mock.calls.filter((c) => c[0] === "auth_refresh")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("propagates the original error when the refresh itself fails", async () => {
|
||||
// The refresh JWT is good for 90 days, but it does eventually
|
||||
// expire (or get revoked). At that point there's nothing left to
|
||||
// do but surface the failure — retrying with the same dead token
|
||||
// would just be a second 401.
|
||||
const { notificationCount } = await import("./client");
|
||||
invokeMock
|
||||
.mockRejectedValueOnce(appviewTokenInvalid("notification count"))
|
||||
.mockRejectedValueOnce("refresh token expired");
|
||||
|
||||
await expect(notificationCount("did:plc:me")).rejects.toThrow(
|
||||
/TokenInvalid/,
|
||||
);
|
||||
expect(invokeMock.mock.calls.map((c) => c[0])).toEqual([
|
||||
"notification_count",
|
||||
"auth_refresh",
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not refresh on a 403 Forbidden — a new token wouldn't help", async () => {
|
||||
// The AppView returns this when the token is perfectly valid but
|
||||
// its `sub` doesn't match the `did` query parameter. Refreshing
|
||||
// mints another token for the same subject, so a retry is pure
|
||||
// waste.
|
||||
const { fetchTimeline } = await import("./client");
|
||||
invokeMock.mockRejectedValueOnce(
|
||||
'appview: timeline home returned 403 Forbidden: ' +
|
||||
'{"error":"Forbidden","message":"did does not match token subject"}',
|
||||
);
|
||||
|
||||
await expect(fetchTimeline("did:plc:someone-else")).rejects.toThrow(
|
||||
/Forbidden/,
|
||||
);
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not refresh when the shell says there is no session at all", async () => {
|
||||
// `require_access_jwt` in lib.rs. Nothing to refresh *from*, so the
|
||||
// message deliberately carries neither `TokenInvalid` nor
|
||||
// `ExpiredSignature`.
|
||||
const { fetchNotifications } = await import("./client");
|
||||
invokeMock.mockRejectedValueOnce(
|
||||
"not logged in: notifications requires a signed-in session",
|
||||
);
|
||||
|
||||
await expect(fetchNotifications("did:plc:me")).rejects.toThrow(
|
||||
/not logged in/,
|
||||
);
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("treats a 503 AuthUnavailable as transient, not as an auth failure", async () => {
|
||||
// The AppView answers 503 `AuthUnavailable` when it cannot reach
|
||||
// the PDS to fetch the verification key — it fails closed rather
|
||||
// than guessing. Our token is fine; the *server* is temporarily
|
||||
// unable to check it. So: no refresh (nothing is wrong with the
|
||||
// token), and `isAuthFailure` must stay false so the background
|
||||
// poll keeps trying instead of shutting itself down over an outage
|
||||
// that will resolve on its own.
|
||||
const { notificationCount, isAuthFailure } = await import("./client");
|
||||
const err =
|
||||
'appview: notification count returned 503 Service Unavailable: ' +
|
||||
'{"error":"AuthUnavailable","message":"could not fetch PDS key"}';
|
||||
|
||||
invokeMock.mockRejectedValueOnce(err);
|
||||
await expect(notificationCount("did:plc:me")).rejects.toThrow(
|
||||
/AuthUnavailable/,
|
||||
);
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
expect(isAuthFailure(err)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not refresh on a transient server error", async () => {
|
||||
const { notificationCount } = await import("./client");
|
||||
invokeMock.mockRejectedValueOnce(
|
||||
"appview: notification count returned 500 Internal Server Error: db down",
|
||||
);
|
||||
|
||||
await expect(notificationCount("did:plc:me")).rejects.toThrow(/500/);
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("an auth_* command never triggers a refresh (no login loop)", async () => {
|
||||
const { session } = await import("./client");
|
||||
invokeMock.mockRejectedValueOnce("TokenInvalid");
|
||||
|
||||
await expect(session.login("me.test", "pw")).rejects.toBeDefined();
|
||||
expect(invokeMock.mock.calls.map((c) => c[0])).toEqual(["auth_login"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("public AppView endpoints", () => {
|
||||
// These stay unauthenticated server-side, so they must keep working
|
||||
// with no session in the store: one invoke, no bearer token to fetch,
|
||||
// no refresh.
|
||||
const PUBLIC = [
|
||||
{
|
||||
name: "search",
|
||||
payload: { posts: [], q: "hi" },
|
||||
call: async () => (await import("./client")).fetchSearch("hi"),
|
||||
},
|
||||
{
|
||||
name: "profile_get",
|
||||
payload: {
|
||||
did: "did:plc:a",
|
||||
handle: "a.test",
|
||||
posts: [],
|
||||
followers: 0,
|
||||
following: 0,
|
||||
post_count: 0,
|
||||
},
|
||||
call: async () => (await import("./client")).fetchProfile("a.test"),
|
||||
},
|
||||
{
|
||||
name: "profile_get_by_did",
|
||||
payload: {
|
||||
did: "did:plc:a",
|
||||
handle: "a.test",
|
||||
posts: [],
|
||||
followers: 0,
|
||||
following: 0,
|
||||
post_count: 0,
|
||||
},
|
||||
call: async () => (await import("./client")).fetchProfileByDid("did:plc:a"),
|
||||
},
|
||||
{
|
||||
name: "post_get",
|
||||
payload: { post: null, thread: { parent: null, root: null } },
|
||||
call: async () =>
|
||||
(await import("./client")).fetchPost("at://did:plc:a/app.twi.post/1"),
|
||||
},
|
||||
{
|
||||
name: "fetch_thread",
|
||||
payload: { post: null, parents: [], root: null, replies: [] },
|
||||
call: async () =>
|
||||
(await import("./client")).fetchThread("at://did:plc:a/app.twi.post/1"),
|
||||
},
|
||||
{
|
||||
name: "fetch_followers",
|
||||
payload: { profiles: [], cursor: null },
|
||||
call: async () => (await import("./client")).fetchFollowers("did:plc:a"),
|
||||
},
|
||||
{
|
||||
name: "fetch_following",
|
||||
payload: { profiles: [], cursor: null },
|
||||
call: async () => (await import("./client")).fetchFollowing("did:plc:a"),
|
||||
},
|
||||
] as const;
|
||||
|
||||
for (const ep of PUBLIC) {
|
||||
it(`${ep.name} resolves without a session and without refreshing`, async () => {
|
||||
invokeMock.mockResolvedValueOnce(ep.payload);
|
||||
|
||||
await expect(ep.call()).resolves.toBeDefined();
|
||||
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
expect(invokeMock.mock.calls[0][0]).toBe(ep.name);
|
||||
expect(
|
||||
invokeMock.mock.calls.some((c) => c[0] === "auth_refresh"),
|
||||
).toBe(false);
|
||||
});
|
||||
}
|
||||
|
||||
it("a public call's own failure surfaces untouched", async () => {
|
||||
const { fetchSearch } = await import("./client");
|
||||
invokeMock.mockRejectedValueOnce(
|
||||
"appview: search returned 400 Bad Request: q is required",
|
||||
);
|
||||
await expect(fetchSearch("")).rejects.toThrow(/q is required/);
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isAuthFailure / errorMessage", () => {
|
||||
it("recognises every shape the AppView's auth guard can answer with", async () => {
|
||||
const { isAuthFailure } = await import("./client");
|
||||
for (const msg of [
|
||||
'appview: timeline home returned 401 Unauthorized: {"error":"AuthMissing","message":"no bearer"}',
|
||||
'appview: notifications returned 401 Unauthorized: {"error":"TokenInvalid","message":"ExpiredSignature"}',
|
||||
'appview: notification count returned 403 Forbidden: {"error":"Forbidden"}',
|
||||
"not logged in: the home timeline requires a signed-in session",
|
||||
]) {
|
||||
expect(isAuthFailure(msg)).toBe(true);
|
||||
expect(isAuthFailure(new Error(msg))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not mistake a server or network failure for an auth failure", async () => {
|
||||
const { isAuthFailure } = await import("./client");
|
||||
expect(
|
||||
isAuthFailure("appview: notifications returned 500: db down"),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isAuthFailure("appview: failed to send timeline request"),
|
||||
).toBe(false);
|
||||
expect(isAuthFailure(null)).toBe(false);
|
||||
expect(isAuthFailure(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("swaps the raw 401 wire string for copy the user can act on", async () => {
|
||||
const { errorMessage } = await import("./client");
|
||||
const raw =
|
||||
'appview: notifications returned 401 Unauthorized: {"error":"TokenInvalid","message":"ExpiredSignature"}';
|
||||
expect(errorMessage(raw)).toBe(
|
||||
"Sitzung abgelaufen oder abgelehnt — bitte neu anmelden.",
|
||||
);
|
||||
// Anything else is shown verbatim: there's nothing better to say
|
||||
// about a 500 than what the server said.
|
||||
expect(errorMessage("appview: search returned 500: db down")).toContain(
|
||||
"500",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -59,14 +59,22 @@ export async function getAppviewUrl(): Promise<string> {
|
||||
* webview it falls through to a normal `invoke` call.
|
||||
*
|
||||
* **Auto-refresh on 401**: the access JWT expires after 1 hour. When
|
||||
* the PDS rejects our token with `TokenInvalid` (the rusty
|
||||
* `routes::auth` handlers return `{"error":"TokenInvalid",...}`),
|
||||
* we ask the Rust shell for a fresh access JWT via the
|
||||
* `auth_refresh` Tauri command. The Rust side reads the stored
|
||||
* refresh JWT (valid for 90 days) and rotates both. We retry
|
||||
* exactly once on the same `cmd` + `args`. The `auth_*` commands
|
||||
* themselves are skipped so a failing login doesn't trigger an
|
||||
* infinite refresh loop.
|
||||
* the PDS *or the AppView* rejects our token with `TokenInvalid`
|
||||
* (both return `{"error":"TokenInvalid",...}` — the PDS from its
|
||||
* `routes::auth` handlers, the AppView from the guard on
|
||||
* `/api/timeline/home`, `/api/notifications`,
|
||||
* `/api/notifications/count` and `/api/notifications/seen`), we ask
|
||||
* the Rust shell for a fresh access JWT via the `auth_refresh` Tauri
|
||||
* command. The Rust side reads the stored refresh JWT (valid for 90
|
||||
* days) and rotates both. We retry exactly once on the same `cmd` +
|
||||
* `args`. The `auth_*` commands themselves are skipped so a failing
|
||||
* login doesn't trigger an infinite refresh loop.
|
||||
*
|
||||
* The whole chain is string-matching, end to end: the AppView states
|
||||
* the code only in its JSON body, `appview_client.rs`'s
|
||||
* `status_error()` formats that body into the `anyhow` message, and
|
||||
* `lib.rs` stringifies it into the command's `Err(String)`. See the
|
||||
* Rust-side test `token_invalid_code_survives_into_the_error_string`.
|
||||
*/
|
||||
async function safeInvoke<T>(cmd: string, args?: Record<string, unknown>): Promise<T> {
|
||||
if (!isTauri()) {
|
||||
@@ -82,17 +90,101 @@ async function safeInvoke<T>(cmd: string, args?: Record<string, unknown>): Promi
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalise whatever a rejected `invoke` handed us into a string.
|
||||
///
|
||||
/// This is not defensive padding — it is the difference between the
|
||||
/// retry chain working and not. Our Tauri commands are
|
||||
/// `Result<T, String>`, and `@tauri-apps/api`'s `invoke` rejects with
|
||||
/// the *deserialised* error payload, i.e. a bare JS **string**, not an
|
||||
/// `Error`. Anything that only reads `e.message` therefore sees
|
||||
/// nothing at all on the exact path that matters. Errors thrown
|
||||
/// locally (the browser-preview guard above, and the `Error` instances
|
||||
/// the tests use) still arrive as objects, so both shapes are handled.
|
||||
function errorText(e: unknown): string {
|
||||
if (typeof e === "string") return e;
|
||||
if (typeof e === "object" && e !== null) {
|
||||
const m = (e as { message?: unknown }).message;
|
||||
if (typeof m === "string") return m;
|
||||
}
|
||||
return String(e ?? "");
|
||||
}
|
||||
|
||||
/// Sniff out a `TokenInvalid` response from the Rust error string.
|
||||
/// Returns true when the error message looks like an expired/
|
||||
/// invalid JWT (the PDS uses a stable `"TokenInvalid"` code in its
|
||||
/// JSON error body, which `@tauri-apps/api/core` surfaces verbatim).
|
||||
/// Returns true when the error looks like an expired/invalid JWT —
|
||||
/// both the PDS and the AppView use a stable `"TokenInvalid"` code in
|
||||
/// their JSON error body, which travels verbatim through the Rust
|
||||
/// error message and out over the Tauri IPC boundary.
|
||||
function isTokenInvalid(e: unknown): boolean {
|
||||
if (typeof e !== "object" || e === null) return false;
|
||||
const msg = (e as { message?: string }).message ?? String(e);
|
||||
const msg = errorText(e);
|
||||
if (!msg) return false;
|
||||
return msg.includes("TokenInvalid") || msg.includes("ExpiredSignature");
|
||||
}
|
||||
|
||||
/// True when an error means "this call will not succeed until the user
|
||||
/// signs in again" — as opposed to a transient network/server hiccup.
|
||||
///
|
||||
/// Covers everything the AppView's auth guard can answer with
|
||||
/// (`AuthMissing` / `TokenInvalid` on 401, `Forbidden` on 403) plus the
|
||||
/// Rust shell's own "no session stored" message from
|
||||
/// `require_access_jwt`. Callers that poll in the background use this
|
||||
/// to *stop* polling: by the time one of these surfaces, `safeInvoke`
|
||||
/// has already spent its one refresh attempt, so retrying on a timer
|
||||
/// would just be a request loop against a server that keeps saying no.
|
||||
export function isAuthFailure(e: unknown): boolean {
|
||||
const msg = errorText(e);
|
||||
if (!msg) return false;
|
||||
return (
|
||||
msg.includes("AuthMissing") ||
|
||||
msg.includes("TokenInvalid") ||
|
||||
msg.includes("ExpiredSignature") ||
|
||||
msg.includes("Forbidden") ||
|
||||
msg.includes("not logged in")
|
||||
);
|
||||
}
|
||||
|
||||
/// True when the PDS refused a registration because of the invite
|
||||
/// code. The public instance runs with `PDS_INVITE_REQUIRED=true` and
|
||||
/// answers `400 {"error":"InvalidInviteCode","message":…}` — the same
|
||||
/// code for a missing, misspelled, disabled and already-spent code, on
|
||||
/// purpose: the server does not tell an unauthenticated caller which
|
||||
/// of those it was, since that would make invite codes enumerable.
|
||||
///
|
||||
/// Matched on the string for the same reason as [`isTokenInvalid`]:
|
||||
/// `pds_client.rs` bails with `createAccount failed: {status} {body}`
|
||||
/// and `lib.rs` stringifies that into the command's `Err(String)`, so
|
||||
/// the code travels verbatim across the IPC boundary.
|
||||
function isInvalidInviteCode(e: unknown): boolean {
|
||||
return errorText(e).includes("InvalidInviteCode");
|
||||
}
|
||||
|
||||
/// User-facing copy for a failed call, in the app's German UI voice.
|
||||
///
|
||||
/// An auth failure gets a sentence naming the actual remedy. The raw
|
||||
/// string a view would otherwise render —
|
||||
/// `appview: timeline home returned 401 Unauthorized:
|
||||
/// {"error":"TokenInvalid","message":"ExpiredSignature"}` — is precise
|
||||
/// and completely unactionable for the person reading it. Everything
|
||||
/// else falls through verbatim: a network error or a 500 is worth
|
||||
/// showing as-is, since there is nothing better to say about it.
|
||||
export function errorMessage(e: unknown): string {
|
||||
if (isAuthFailure(e)) {
|
||||
return "Sitzung abgelaufen oder abgelehnt — bitte neu anmelden.";
|
||||
}
|
||||
if (isInvalidInviteCode(e)) {
|
||||
// Deliberately covers "no code given" too: the raw body a user
|
||||
// would otherwise read is `createAccount failed: 400 Bad Request
|
||||
// {"error":"InvalidInviteCode","message":"a valid invite code is
|
||||
// required to create an account on this server"}`. Since the
|
||||
// server refuses to say *which* way the code was wrong, the copy
|
||||
// names both plausible fixes rather than guessing one.
|
||||
return (
|
||||
"Einladungscode ungültig oder bereits verbraucht — " +
|
||||
"bitte prüfen oder einen neuen Code anfordern."
|
||||
);
|
||||
}
|
||||
return String(e);
|
||||
}
|
||||
|
||||
export type Session = {
|
||||
did: string;
|
||||
handle: string;
|
||||
@@ -143,11 +235,38 @@ function createSessionStore() {
|
||||
set(s);
|
||||
return s;
|
||||
},
|
||||
async register(handle: string, password: string) {
|
||||
/// Create an account on the configured PDS.
|
||||
///
|
||||
/// `inviteCode` is optional because the invite gate lives on the
|
||||
/// *server* (`PDS_INVITE_REQUIRED`): the public instance at
|
||||
/// `https://tweet.maarcade.com` requires a code, a local dev PDS
|
||||
/// normally does not. The client therefore never refuses a
|
||||
/// registration for a missing code on its own — it would break
|
||||
/// development against localhost — it just forwards what the user
|
||||
/// typed and lets the PDS decide.
|
||||
///
|
||||
/// An empty (or whitespace-only) field is *omitted*, not sent as
|
||||
/// `""`. Those are two different statements: "I gave no code" vs.
|
||||
/// "my code is the empty string". The first is legitimate against
|
||||
/// an open server; the second is never true and would only turn
|
||||
/// into a confusing `InvalidInviteCode` on a server that has the
|
||||
/// gate switched off. Dropping the key leaves the Rust
|
||||
/// `Option<String>` at `None`, and `create_account` then leaves
|
||||
/// the field out of the JSON body entirely.
|
||||
async register(handle: string, password: string, inviteCode?: string) {
|
||||
if (!isTauri()) {
|
||||
throw new Error("register requires the Tauri desktop runtime");
|
||||
}
|
||||
const s = await safeInvoke<Session>("auth_register", { handle, password });
|
||||
const code = inviteCode?.trim();
|
||||
const s = await safeInvoke<Session>("auth_register", {
|
||||
handle,
|
||||
password,
|
||||
// Explicit `null` rather than a dropped key, matching how
|
||||
// `createPost` passes its optional `embed` / `reply`: it
|
||||
// deserialises into the Rust `Option<String>` as `None`
|
||||
// without depending on how `invoke` treats `undefined`.
|
||||
inviteCode: code ? code : null,
|
||||
});
|
||||
set(s);
|
||||
return s;
|
||||
},
|
||||
@@ -253,8 +372,10 @@ export type SearchResponse = {
|
||||
///
|
||||
/// `like_count` and `repost_count` are present when the post was
|
||||
/// found; they're `undefined` (or absent) for the "not in index"
|
||||
/// sentinel response (where `post` is null). AppView has no auth
|
||||
/// yet, so `viewer_liked` / `viewer_reposted` aren't returned.
|
||||
/// sentinel response (where `post` is null). `/api/post/{uri}` is one
|
||||
/// of the AppView's public endpoints — it takes no token and so has no
|
||||
/// viewer to resolve against, hence no `viewer_liked` /
|
||||
/// `viewer_reposted`. Use [`fetchThread`] with a `viewerDid` for those.
|
||||
export type ThreadResponse = {
|
||||
post: Post | null;
|
||||
thread: {
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
// Unit tests for the invite-code half of the registration path.
|
||||
//
|
||||
// Same setup as `notifications.test.ts`: `@tauri-apps/api/core` is
|
||||
// mocked so no Tauri shell is needed, and every assertion is about the
|
||||
// exact command name + argument bag we hand the Rust IPC layer. That
|
||||
// argument bag is the contract — `invoke` maps camelCase JS keys onto
|
||||
// the snake_case Rust command parameters (`inviteCode` → `invite_code`
|
||||
// on `auth_register`), so a typo here surfaces at runtime as a null
|
||||
// argument, not at compile time.
|
||||
//
|
||||
// Covered:
|
||||
// * `session.register` — the code is forwarded as `inviteCode`,
|
||||
// alongside the unchanged `handle` / `password`;
|
||||
// * the empty / whitespace-only field — must reach the shell as
|
||||
// `null` ("no code given"), never as `""` ("my code is the empty
|
||||
// string"), because a PDS without the invite gate has to keep
|
||||
// accepting registrations;
|
||||
// * `errorMessage` — the PDS's `InvalidInviteCode` body becomes
|
||||
// German copy instead of the raw wire string.
|
||||
//
|
||||
// Run with:
|
||||
// npx vitest run src/lib/api/invite.test.ts
|
||||
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const invokeMock = vi.fn();
|
||||
|
||||
vi.mock("@tauri-apps/api/core", () => ({
|
||||
invoke: (...args: unknown[]) => invokeMock(...args),
|
||||
isTauri: () => true,
|
||||
}));
|
||||
|
||||
/// What the Rust `auth_register` command answers with on success.
|
||||
const SESSION = {
|
||||
did: "did:plc:alice",
|
||||
handle: "alice.tweet.maarcade.com",
|
||||
access_jwt: "acc",
|
||||
refresh_jwt: "ref",
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
invokeMock.mockReset();
|
||||
});
|
||||
|
||||
describe("session.register", () => {
|
||||
it("forwards the invite code as `inviteCode`", async () => {
|
||||
const { session } = await import("./client");
|
||||
invokeMock.mockResolvedValueOnce(SESSION);
|
||||
|
||||
const s = await session.register(
|
||||
"alice.tweet.maarcade.com",
|
||||
"hunter2hunter2",
|
||||
"mt-7k3qw-z9d2m",
|
||||
);
|
||||
|
||||
expect(invokeMock).toHaveBeenCalledTimes(1);
|
||||
expect(invokeMock).toHaveBeenCalledWith("auth_register", {
|
||||
handle: "alice.tweet.maarcade.com",
|
||||
password: "hunter2hunter2",
|
||||
inviteCode: "mt-7k3qw-z9d2m",
|
||||
});
|
||||
expect(s.did).toBe("did:plc:alice");
|
||||
});
|
||||
|
||||
it("trims the surrounding whitespace off a pasted code", async () => {
|
||||
const { session } = await import("./client");
|
||||
invokeMock.mockResolvedValueOnce(SESSION);
|
||||
|
||||
// Copying a code out of a chat message routinely drags a space or
|
||||
// a newline along. The server trims too (`invite::normalize`), but
|
||||
// sending the untrimmed string would mean the *client* and the
|
||||
// server disagree about whether the field is empty.
|
||||
await session.register("alice.test", "pw", " mt-7k3qw-z9d2m\n");
|
||||
|
||||
expect(invokeMock.mock.calls[0][1]).toMatchObject({
|
||||
inviteCode: "mt-7k3qw-z9d2m",
|
||||
});
|
||||
});
|
||||
|
||||
it("sends null — never an empty string — when the field is blank", async () => {
|
||||
const { session } = await import("./client");
|
||||
|
||||
// Three ways the UI can hand us "nothing": the argument omitted
|
||||
// entirely (login-shaped call), an untouched input, and an input
|
||||
// holding only whitespace. All three mean "the user gave no code"
|
||||
// and must arrive at the Rust `Option<String>` as `None`, so that
|
||||
// a PDS running without `PDS_INVITE_REQUIRED` still registers the
|
||||
// account instead of rejecting a blank code.
|
||||
for (const blank of [undefined, "", " "]) {
|
||||
invokeMock.mockReset();
|
||||
invokeMock.mockResolvedValueOnce(SESSION);
|
||||
|
||||
await session.register("alice.test", "pw", blank);
|
||||
|
||||
const args = invokeMock.mock.calls[0][1] as Record<string, unknown>;
|
||||
expect(args.inviteCode).toBeNull();
|
||||
expect(args.inviteCode).not.toBe("");
|
||||
// The rest of the bag is unaffected.
|
||||
expect(args.handle).toBe("alice.test");
|
||||
expect(args.password).toBe("pw");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("errorMessage for InvalidInviteCode", () => {
|
||||
/// Exactly what crosses the IPC boundary when the PDS refuses the
|
||||
/// code: `pds_client.rs` bails with `createAccount failed: {status}
|
||||
/// {body}` and `lib.rs` stringifies that into `Err(String)`, which
|
||||
/// `invoke` rejects with as a bare JS string.
|
||||
const RAW =
|
||||
'createAccount failed: 400 Bad Request {"error":"InvalidInviteCode",' +
|
||||
'"message":"a valid invite code is required to create an account on this server"}';
|
||||
|
||||
it("replaces the raw 400 body with copy the user can act on", async () => {
|
||||
const { errorMessage } = await import("./client");
|
||||
const msg = errorMessage(RAW);
|
||||
|
||||
expect(msg).toContain("Einladungscode");
|
||||
// None of the wire noise survives into the UI.
|
||||
expect(msg).not.toContain("InvalidInviteCode");
|
||||
expect(msg).not.toContain("400");
|
||||
// Both shapes a rejected `invoke` can produce — a bare string and
|
||||
// an Error — go through the same `errorText` normalisation.
|
||||
expect(errorMessage(new Error(RAW))).toBe(msg);
|
||||
});
|
||||
|
||||
it("leaves unrelated registration failures verbatim", async () => {
|
||||
const { errorMessage } = await import("./client");
|
||||
// A taken handle is a different 400 and has its own message worth
|
||||
// showing; the invite branch must not swallow it.
|
||||
expect(
|
||||
errorMessage(
|
||||
'createAccount failed: 400 Bad Request {"error":"HandleNotAvailable"}',
|
||||
),
|
||||
).toContain("HandleNotAvailable");
|
||||
expect(errorMessage("createAccount failed: 500 db down")).toContain("500");
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { onMount } from "svelte";
|
||||
import { session, describeServer, type Session } from "../api/client";
|
||||
import {
|
||||
session,
|
||||
describeServer,
|
||||
errorMessage,
|
||||
type Session,
|
||||
} from "../api/client";
|
||||
|
||||
let { onLogin }: { onLogin: (s: Session) => void } = $props();
|
||||
|
||||
@@ -10,6 +15,9 @@
|
||||
let mode: "login" | "register" = $state("login");
|
||||
let handle: string = $state("");
|
||||
let password: string = $state("");
|
||||
// Only meaningful in "register" mode — the field below is rendered
|
||||
// solely there, and `submit()` only forwards it on that branch.
|
||||
let inviteCode: string = $state("");
|
||||
let busy = $state(false);
|
||||
let error: string | null = $state(null);
|
||||
let serverInfo: any = $state(null);
|
||||
@@ -27,16 +35,40 @@
|
||||
busy = true;
|
||||
error = null;
|
||||
try {
|
||||
// The invite code is deliberately *not* part of the guard above.
|
||||
// Whether one is required is a server setting
|
||||
// (`PDS_INVITE_REQUIRED`): the public instance demands a code, a
|
||||
// dev PDS on localhost usually does not. Refusing to submit
|
||||
// without one would make the client unusable against the second
|
||||
// kind of server for a rule it cannot see. So we forward what
|
||||
// the user typed — `session.register` drops an empty string
|
||||
// instead of sending a blank code — and let the PDS answer.
|
||||
const s = mode === "register"
|
||||
? await session.register(handle, password)
|
||||
? await session.register(handle, password, inviteCode)
|
||||
: await session.login(handle, password);
|
||||
onLogin(s);
|
||||
} catch (e) {
|
||||
error = String(e);
|
||||
// `errorMessage` translates the failures worth naming — an
|
||||
// expired session, and a rejected `InvalidInviteCode` — into
|
||||
// German copy, and passes everything else through verbatim.
|
||||
error = errorMessage(e);
|
||||
} finally {
|
||||
busy = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear the form's mode-specific state when switching sides.
|
||||
///
|
||||
/// Without this, a code typed while registering would linger in the
|
||||
/// hidden field: switch to login, switch back, and the stale value
|
||||
/// is silently submitted again. The error goes too — the message
|
||||
/// from a failed registration says nothing about the login the user
|
||||
/// is now attempting.
|
||||
function toggleMode() {
|
||||
mode = mode === "register" ? "login" : "register";
|
||||
inviteCode = "";
|
||||
error = null;
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="login">
|
||||
@@ -75,6 +107,30 @@
|
||||
autocomplete={mode === "register" ? "new-password" : "current-password"}
|
||||
/>
|
||||
</label>
|
||||
{#if mode === "register"}
|
||||
<!--
|
||||
Registration only. Logging in never carries a code, and a
|
||||
field that is present but meaningless invites people to
|
||||
fill it in. `{#if}` removes it from the DOM rather than
|
||||
hiding it, so it also drops out of the tab order.
|
||||
-->
|
||||
<label class="field">
|
||||
<span class="key">einladungscode</span>
|
||||
<input
|
||||
type="text"
|
||||
bind:value={inviteCode}
|
||||
placeholder="mt-xxxxx-xxxxx"
|
||||
disabled={busy}
|
||||
onkeydown={(e) => e.key === "Enter" && submit()}
|
||||
autocomplete="off"
|
||||
autocapitalize="none"
|
||||
spellcheck="false"
|
||||
/>
|
||||
<span class="hint">
|
||||
// von dieser Instanz verlangt — ohne Code keine Registrierung
|
||||
</span>
|
||||
</label>
|
||||
{/if}
|
||||
</form>
|
||||
{#if error}
|
||||
<div class="err">err: {error}</div>
|
||||
@@ -83,7 +139,7 @@
|
||||
<button class="btn btn--primary" onclick={submit} disabled={busy || !handle || !password}>
|
||||
{busy ? "..." : mode === "register" ? "create account" : "log in"}
|
||||
</button>
|
||||
<button class="btn btn--ghost" onclick={() => (mode = mode === "register" ? "login" : "register")} disabled={busy}>
|
||||
<button class="btn btn--ghost" onclick={toggleMode} disabled={busy}>
|
||||
{mode === "register" ? "have an account? log in" : "no account? register"}
|
||||
</button>
|
||||
</div>
|
||||
@@ -172,6 +228,15 @@
|
||||
font-size: var(--fs-50);
|
||||
letter-spacing: var(--tracking-label);
|
||||
}
|
||||
/* Sub-label under the invite field. Same dim mono voice as the
|
||||
`// pds: …` server meta line above the form, so it reads as a
|
||||
comment on the field rather than as a second input label. */
|
||||
.hint {
|
||||
color: var(--text-dim);
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--fs-50);
|
||||
opacity: 0.8;
|
||||
}
|
||||
.form input {
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--line-2);
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
// Regression guard for the invite-code field on the login screen.
|
||||
//
|
||||
// The public instance runs the PDS with `PDS_INVITE_REQUIRED=true`, so
|
||||
// `createAccount` without a code is refused with `400
|
||||
// {"error":"InvalidInviteCode", …}`. Three things have to hold for the
|
||||
// screen to be usable against it:
|
||||
//
|
||||
// 1. the field exists in "register" mode and *not* in "login" mode —
|
||||
// a code is meaningless when signing in, and an input that is
|
||||
// present but ignored invites people to fill it in;
|
||||
// 2. what the user typed reaches `session.register` as its third
|
||||
// argument, and an untouched field does not become a blank code;
|
||||
// 3. a rejected code renders as German copy, not as the raw wire
|
||||
// body, which is where the user would otherwise read
|
||||
// `createAccount failed: 400 Bad Request {"error":…}`.
|
||||
//
|
||||
// Setup follows `NotificationsView.test.ts`: the component is mounted
|
||||
// against jsdom with `../api/client` partially mocked — the real
|
||||
// `errorMessage` is kept, since the error copy is part of what we are
|
||||
// asserting on, and only the calls that would need a Tauri runtime are
|
||||
// stubbed.
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { mount, unmount, tick } from "svelte";
|
||||
|
||||
const registerMock = vi.fn();
|
||||
const loginMock = vi.fn();
|
||||
const describeServerMock = vi.fn();
|
||||
|
||||
vi.mock("../api/client", async () => {
|
||||
const actual =
|
||||
await vi.importActual<typeof import("../api/client")>("../api/client");
|
||||
return {
|
||||
...actual,
|
||||
// Keep the real `errorMessage` from `actual` — the German copy for
|
||||
// `InvalidInviteCode` is exactly what test 3 checks.
|
||||
describeServer: (...args: unknown[]) => describeServerMock(...args),
|
||||
session: {
|
||||
...actual.session,
|
||||
register: (...args: unknown[]) => registerMock(...args),
|
||||
login: (...args: unknown[]) => loginMock(...args),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import LoginScreen from "./LoginScreen.svelte";
|
||||
|
||||
let target: HTMLDivElement;
|
||||
let app: ReturnType<typeof mount> | null = null;
|
||||
|
||||
const SESSION = {
|
||||
did: "did:plc:alice",
|
||||
handle: "alice.tweet.maarcade.com",
|
||||
access_jwt: "acc",
|
||||
refresh_jwt: "ref",
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
target = document.createElement("div");
|
||||
document.body.appendChild(target);
|
||||
registerMock.mockReset();
|
||||
loginMock.mockReset();
|
||||
describeServerMock.mockReset();
|
||||
// `onMount` calls this; a resolved stub keeps the meta line quiet.
|
||||
describeServerMock.mockResolvedValue({ did: "did:web:tweet.maarcade.com" });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (app) unmount(app);
|
||||
app = null;
|
||||
target.remove();
|
||||
});
|
||||
|
||||
/// Let `onMount`, the mocked promises and Svelte's flush settle.
|
||||
async function flush(turns = 6) {
|
||||
for (let i = 0; i < turns; i++) {
|
||||
await Promise.resolve();
|
||||
await tick();
|
||||
}
|
||||
}
|
||||
|
||||
/// The screen has no test ids; the inputs are addressed the way a user
|
||||
/// would, by the label text beside them.
|
||||
function fieldByLabel(label: string): HTMLInputElement | null {
|
||||
for (const el of target.querySelectorAll("label.field")) {
|
||||
if (el.querySelector(".key")?.textContent?.trim() === label) {
|
||||
return el.querySelector("input");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function typeInto(input: HTMLInputElement, value: string) {
|
||||
input.value = value;
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
}
|
||||
|
||||
/// Flip login ⇄ register via the ghost button under the form.
|
||||
async function toggleMode() {
|
||||
const buttons = [...target.querySelectorAll("button.btn--ghost")];
|
||||
buttons[buttons.length - 1].dispatchEvent(
|
||||
new MouseEvent("click", { bubbles: true }),
|
||||
);
|
||||
await tick();
|
||||
}
|
||||
|
||||
async function submitForm() {
|
||||
target
|
||||
.querySelector("button.btn--primary")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true }));
|
||||
await flush();
|
||||
}
|
||||
|
||||
async function mountScreen() {
|
||||
app = mount(LoginScreen, { target, props: { onLogin: vi.fn() } });
|
||||
await flush();
|
||||
}
|
||||
|
||||
describe("LoginScreen invite field", () => {
|
||||
it("shows the code field only while registering", async () => {
|
||||
await mountScreen();
|
||||
|
||||
// "login" is the default mode — no invite field, and nothing in
|
||||
// the tab order either, since `{#if}` removes it from the DOM.
|
||||
expect(fieldByLabel("einladungscode")).toBeNull();
|
||||
|
||||
await toggleMode();
|
||||
expect(fieldByLabel("einladungscode")).not.toBeNull();
|
||||
|
||||
await toggleMode();
|
||||
expect(fieldByLabel("einladungscode")).toBeNull();
|
||||
});
|
||||
|
||||
it("passes the typed code to session.register as the third argument", async () => {
|
||||
registerMock.mockResolvedValue(SESSION);
|
||||
await mountScreen();
|
||||
await toggleMode();
|
||||
|
||||
typeInto(fieldByLabel("handle")!, "alice.tweet.maarcade.com");
|
||||
typeInto(fieldByLabel("password")!, "hunter2hunter2");
|
||||
typeInto(fieldByLabel("einladungscode")!, "mt-7k3qw-z9d2m");
|
||||
await submitForm();
|
||||
|
||||
expect(loginMock).not.toHaveBeenCalled();
|
||||
expect(registerMock).toHaveBeenCalledTimes(1);
|
||||
expect(registerMock).toHaveBeenCalledWith(
|
||||
"alice.tweet.maarcade.com",
|
||||
"hunter2hunter2",
|
||||
"mt-7k3qw-z9d2m",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not turn an untouched field into a blank code", async () => {
|
||||
registerMock.mockResolvedValue(SESSION);
|
||||
await mountScreen();
|
||||
await toggleMode();
|
||||
|
||||
typeInto(fieldByLabel("handle")!, "alice.test");
|
||||
typeInto(fieldByLabel("password")!, "hunter2hunter2");
|
||||
// Invite field deliberately left alone. The submit must still go
|
||||
// through: whether a code is required is the *server's* call
|
||||
// (`PDS_INVITE_REQUIRED`), and a dev PDS on localhost runs without
|
||||
// the gate. What must not happen is a blank code travelling on as
|
||||
// if the user had entered one.
|
||||
await submitForm();
|
||||
|
||||
expect(registerMock).toHaveBeenCalledTimes(1);
|
||||
const code = registerMock.mock.calls[0][2];
|
||||
expect(code === "" || code === undefined).toBe(true);
|
||||
expect(code?.trim?.() ?? "").toBe("");
|
||||
});
|
||||
|
||||
it("clears a typed code when switching back to login", async () => {
|
||||
loginMock.mockResolvedValue(SESSION);
|
||||
await mountScreen();
|
||||
await toggleMode();
|
||||
|
||||
typeInto(fieldByLabel("handle")!, "alice.test");
|
||||
typeInto(fieldByLabel("password")!, "hunter2hunter2");
|
||||
typeInto(fieldByLabel("einladungscode")!, "mt-stale-code0");
|
||||
|
||||
// Back to login, then to register again: a stale code lingering in
|
||||
// the hidden field would be submitted silently on the next try.
|
||||
await toggleMode();
|
||||
await toggleMode();
|
||||
expect(fieldByLabel("einladungscode")!.value).toBe("");
|
||||
});
|
||||
|
||||
it("renders German copy when the PDS rejects the code", async () => {
|
||||
// Verbatim what reaches the component: `pds_client.rs` bails with
|
||||
// `createAccount failed: {status} {body}`, `lib.rs` stringifies it
|
||||
// into the command's `Err(String)`, and `invoke` rejects with that
|
||||
// bare string.
|
||||
registerMock.mockRejectedValue(
|
||||
'createAccount failed: 400 Bad Request {"error":"InvalidInviteCode",' +
|
||||
'"message":"a valid invite code is required to create an account on this server"}',
|
||||
);
|
||||
await mountScreen();
|
||||
await toggleMode();
|
||||
|
||||
typeInto(fieldByLabel("handle")!, "alice.test");
|
||||
typeInto(fieldByLabel("password")!, "hunter2hunter2");
|
||||
typeInto(fieldByLabel("einladungscode")!, "mt-wrong-code0");
|
||||
await submitForm();
|
||||
|
||||
const err = target.querySelector(".err");
|
||||
expect(err).not.toBeNull();
|
||||
expect(err!.textContent).toContain("Einladungscode");
|
||||
// The wire noise the user would otherwise be shown is gone.
|
||||
expect(err!.textContent).not.toContain("InvalidInviteCode");
|
||||
expect(err!.textContent).not.toContain("400 Bad Request");
|
||||
});
|
||||
|
||||
it("still shows an unrelated failure verbatim", async () => {
|
||||
// The invite branch must not swallow every registration error —
|
||||
// a taken handle has its own message worth reading.
|
||||
registerMock.mockRejectedValue(
|
||||
'createAccount failed: 400 Bad Request {"error":"HandleNotAvailable"}',
|
||||
);
|
||||
await mountScreen();
|
||||
await toggleMode();
|
||||
|
||||
typeInto(fieldByLabel("handle")!, "alice.test");
|
||||
typeInto(fieldByLabel("password")!, "hunter2hunter2");
|
||||
await submitForm();
|
||||
|
||||
expect(target.querySelector(".err")!.textContent).toContain(
|
||||
"HandleNotAvailable",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@
|
||||
import Avatar from "./Avatar.svelte";
|
||||
import Skeleton from "./Skeleton.svelte";
|
||||
import {
|
||||
errorMessage,
|
||||
fetchNotifications,
|
||||
markNotificationsSeen,
|
||||
notificationIcon,
|
||||
@@ -71,7 +72,7 @@
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
error = String(e);
|
||||
error = errorMessage(e);
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
@@ -89,7 +90,7 @@
|
||||
items = [...items, ...r.notifications.filter((n) => !seen.has(n.id))];
|
||||
cursor = r.cursor;
|
||||
} catch (e) {
|
||||
error = String(e);
|
||||
error = errorMessage(e);
|
||||
} finally {
|
||||
loading = false;
|
||||
}
|
||||
|
||||
@@ -194,6 +194,48 @@ describe("NotificationsView actor navigation", () => {
|
||||
expect(onThreadClick).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("shows actionable copy when the AppView rejects the session", async () => {
|
||||
// Since `/api/notifications` grew an auth guard, this is what a
|
||||
// rejected token looks like by the time it reaches the view: the
|
||||
// AppView's JSON body, wrapped by `appview_client.rs`'s
|
||||
// `status_error()` and stringified across the Tauri IPC boundary.
|
||||
// `safeInvoke` has already spent its one refresh attempt getting
|
||||
// here, so the only thing left to tell the user is "log in again" —
|
||||
// rendering the raw wire string would be accurate and useless.
|
||||
fetchNotificationsMock.mockRejectedValue(
|
||||
'appview: notifications returned 401 Unauthorized: ' +
|
||||
'{"error":"TokenInvalid","message":"ExpiredSignature"}',
|
||||
);
|
||||
|
||||
app = mount(NotificationsView, {
|
||||
target,
|
||||
props: { did: "did:plc:me" },
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(target.textContent).toContain("bitte neu anmelden");
|
||||
expect(target.textContent).not.toContain("TokenInvalid");
|
||||
expect(target.textContent).not.toContain("401");
|
||||
// A failed load must not leave the spinner up or ack a page it
|
||||
// never rendered.
|
||||
expect(markNotificationsSeenMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still shows a server error verbatim — there's nothing better to say", async () => {
|
||||
fetchNotificationsMock.mockRejectedValue(
|
||||
"appview: notifications returned 500 Internal Server Error: db down",
|
||||
);
|
||||
|
||||
app = mount(NotificationsView, {
|
||||
target,
|
||||
props: { did: "did:plc:me" },
|
||||
});
|
||||
await flush();
|
||||
|
||||
expect(target.textContent).toContain("500");
|
||||
expect(target.textContent).toContain("db down");
|
||||
});
|
||||
|
||||
it("opens the thread for a row that has a subject", async () => {
|
||||
fetchNotificationsMock.mockResolvedValue({
|
||||
notifications: [row()],
|
||||
|
||||
+47
-19
@@ -19,12 +19,14 @@ auf welchem Weg kommt ein Post vom Client bis in die Timeline zurück.
|
||||
┌──────────────────────────────────┐ ┌──────────────────────────────┐
|
||||
│ pds-server (axum, :2583) │ │ appview (axum, :2584) │
|
||||
│ │ │ │
|
||||
│ /xrpc/com.atproto.* │ │ GET /api/timeline/home │
|
||||
│ /xrpc/com.atproto.* │ │ GET /api/timeline/home 🔒 │
|
||||
│ /xrpc/app.bsky.actor.profile.* │ │ GET /api/profile[/:handle] │
|
||||
│ /blob/:cid │ │ GET /api/search │
|
||||
│ /healthz │ │ GET /api/post|thread/*uri │
|
||||
│ │ │ GET /api/notifications… │
|
||||
│ │ │ GET /api/followers|following│
|
||||
│ /.well-known/did.json ──────────┼───┼─▶ Schlüssel für 🔒 │
|
||||
│ │ │ GET /api/notifications… 🔒 │
|
||||
│ /xrpc/…sync.subscribeRepos ─────┼───┼─▶ pds_firehose.rs (WS) │
|
||||
│ (WebSocket, seq-Cursor) │ │ GET /api/followers|following│
|
||||
│ │ │ GET /healthz │
|
||||
│ at-lexicon Validierung (160) │ │ │
|
||||
│ at-repo/at-mst MST + Commit │ │ indexer.rs Upserts │
|
||||
@@ -40,37 +42,61 @@ auf welchem Weg kommt ein Post vom Client bis in die Timeline zurück.
|
||||
┌───────────┐ ┌────────┐ ┌─────────────────┐ │
|
||||
│ Postgres │ │ MinIO │ │ Postgres │ │
|
||||
│ pds :5434 │ │ :9100 │ │ appview :5435 │ │
|
||||
└───────────┘ └────────┘ └─────────────────┘ │
|
||||
│ │
|
||||
│ (heute: kein eigener Firehose-Ausgang) │
|
||||
▼ │
|
||||
│ +firehose │ └────────┘ │ +cursor │ │
|
||||
│ _events │ └─────────────────┘ │
|
||||
└───────────┘ │
|
||||
│
|
||||
┌──────────────────────────────────────────┐ │
|
||||
│ Jetstream-Relay (extern, WebSocket) │──────────────────┘
|
||||
│ JETSTREAM_URL │ at-firehose
|
||||
│ JETSTREAM_URL — kennt diese PDS nicht │ at-firehose
|
||||
└──────────────────────────────────────────┘ JetstreamConsumer
|
||||
```
|
||||
|
||||
Zwei Wege führen in die AppView, und das ist Absicht:
|
||||
🔒 = Bearer-Token nötig, und der `sub` des Tokens muss der angefragten DID
|
||||
entsprechen. Die AppView verifiziert die ES256-Signatur mit dem öffentlichen
|
||||
Schlüssel, den die PDS in ihrem DID-Dokument veröffentlicht — `PDS_JWT_SECRET`
|
||||
verlässt die PDS nie. Details in [`deployment.md`](deployment.md), Abschnitt 6.
|
||||
|
||||
Drei Wege führen in die AppView, und das ist Absicht:
|
||||
|
||||
1. **Direkter Push (schnell, lokal).** Jeder erfolgreiche Commit auf der PDS
|
||||
wird per `POST /internal/ingest-commit` an die AppView geschoben
|
||||
(`crates/pds-server/src/appview_push.rs`). Best effort, 5 s Timeout, blockiert
|
||||
den Record-Write nie. Damit sieht der Nutzer seinen eigenen Post sofort.
|
||||
2. **Jetstream (global, verzögert).** `at-firehose::JetstreamConsumer` hängt an
|
||||
2. **PDS-Firehose (lokal, garantiert).** Die PDS führt in derselben
|
||||
Transaktion wie den Commit ein Event in `firehose_events` und liefert es
|
||||
über `com.atproto.sync.subscribeRepos` als WebSocket aus
|
||||
(`crates/pds-server/src/firehose.rs`). Die AppView konsumiert das mit
|
||||
persistiertem Cursor (`crates/appview/src/pds_firehose.rs`).
|
||||
3. **Jetstream (global, verzögert).** `at-firehose::JetstreamConsumer` hängt an
|
||||
einem externen Jetstream-Relay und liefert alles, was in den konfigurierten
|
||||
Collections weltweit passiert.
|
||||
|
||||
Wichtig für das Verständnis der Topologie: **die eigene PDS speist den
|
||||
Jetstream nicht.** Es gibt keinen `com.atproto.sync.subscribeRepos`-Endpoint im
|
||||
PDS-Router. Der Firehose-Weg ist ein reiner Konsum-Pfad für fremde Repos; die
|
||||
eigenen Records erreichen die AppView ausschließlich über den Push aus
|
||||
Punkt 1 (noch offen).
|
||||
Warum 1 **und** 2: Der Push ist der schnelle Weg, der Firehose der
|
||||
verlässliche. Ein verlorener Push (AppView kurz weg, Netzwerkfehler) war
|
||||
früher endgültig — der öffentliche Jetstream kennt diese PDS nicht, also
|
||||
wäre der Post nie angekommen. Jetzt holt der Cursor-Replay ihn nach.
|
||||
Dass beide Wege denselben Commit liefern, ist unkritisch: die Indexer-Pfade
|
||||
sind Upserts, und der Dedupe-Index der Notifications fängt den Rest.
|
||||
|
||||
Wichtig für das Verständnis der Topologie bleibt: **die eigene PDS speist den
|
||||
*öffentlichen* Jetstream nicht.** Weg 2 ist ein lokaler Firehose zwischen den
|
||||
eigenen zwei Diensten; ein fremder Relay erfährt von dieser PDS weiterhin
|
||||
nichts.
|
||||
|
||||
Zur Spec-Treue: Die Frame-Hülle ist konformes DAG-CBOR mit Tag-42-CID-Links.
|
||||
Die Blöcke *darin* tragen die Konvention dieses Codebases — CIDs innerhalb von
|
||||
Commit-Blöcken sind Strings, nicht Links (`at-repo/src/commit.rs`). Ein
|
||||
fremder atproto-Consumer kann die Frames also lesen, scheitert aber beim
|
||||
Validieren der Blockinhalte. Das zu ändern hieße, die Blockkodierung zu
|
||||
ändern, und damit ändern sich sämtliche CIDs inklusive der
|
||||
`did:plc:`-Ableitung — eine eigene, bewusste Migration.
|
||||
|
||||
## Crates
|
||||
|
||||
| Crate | Typ | Aufgabe |
|
||||
|---|---|---|
|
||||
| `at-lexicon` | lib | Lexicon-Schemas laden (`Lex::from_json`) und Records validieren. `LexRegistry` in der PDS kennt `app.twi.post` (160 Zeichen), `app.bsky.feed.like`, `app.bsky.feed.repost`, `app.bsky.actor.profile` — alle vier per `include_str!` einkompiliert |
|
||||
| `at-lexicon` | lib | Lexicon-Schemas laden (`Lex::from_json`) und Records validieren. `LexRegistry` in der PDS kennt `app.twi.post` (160 Zeichen), `app.bsky.feed.like`, `app.bsky.feed.repost`, `app.bsky.graph.follow`, `app.bsky.actor.profile` — alle per `include_str!` einkompiliert |
|
||||
| `at-crypto` | lib | secp256k1/P-256-Keypairs, DAG-CBOR-CIDs, multibase/base58btc, JWT (`issue_jwt` / `verify_jwt`), PLC-Operationen inkl. `did_plc_from_op` |
|
||||
| `at-identity` | lib | Handle- und DID-Auflösung. Drei Resolver hinter dem Trait `DidHandleResolver`: `PlcClient` (PLC-Directory), `WebResolver` (`.well-known/did.json`), `PdsHandleResolver` (fragt die lokale PDS) |
|
||||
| `at-mst` | lib | Merkle-Search-Tree: Knoten, `split_around`, `wrap_with_split`, spec-konformes `encode_key` |
|
||||
@@ -158,9 +184,11 @@ Alle Schreibpfade sind Upserts, das Replay nach einem Reconnect ist damit
|
||||
unschädlich. Fehlerhafte Events rücken den Cursor **nicht** vor.
|
||||
|
||||
`ingest.rs` bedient denselben Indexer über HTTP, mit den Aktionen
|
||||
`create` / `delete`; für `app.bsky.graph.follow`-Deletes braucht der Aufrufer
|
||||
`subject_did` im Body, weil der Record-Wert bei Deletes nicht garantiert
|
||||
mitkommt.
|
||||
`create` / `delete`. Für `app.bsky.graph.follow`-Deletes gibt es zwei Wege:
|
||||
der Push schickt `subject_did` im Body mit, der Firehose kennt nur `did` +
|
||||
`rkey` — deshalb speichert `follows` seit Migration 0011 den rkey des
|
||||
Follow-Records und löst darüber auf. Ohne den rkey war ein Unfollow über den
|
||||
Firehose nicht anwendbar und hing allein am Push.
|
||||
|
||||
## Datenbanken und Tabellen
|
||||
|
||||
|
||||
+109
-21
@@ -287,7 +287,7 @@ Hinweise:
|
||||
* Es gibt **keinen** Signal-Handler für graceful Shutdown. `systemctl stop`
|
||||
beendet den Prozess hart; bei der AppView bedeutet das, dass der letzte
|
||||
Cursor-Flush nur passiert, wenn der Kanal regulär geschlossen wird —
|
||||
praktisch also mit bis zu 100 Events Verlust (siehe Abschnitt 8). Das ist
|
||||
praktisch also mit bis zu 100 Events Verlust (siehe Abschnitt 10). Das ist
|
||||
unkritisch, weil der Cursor beim Resume ohnehin leicht in die Vergangenheit
|
||||
zeigt und Events idempotent verarbeitet werden.
|
||||
* Eine Abhängigkeit `After=` auf Postgres/MinIO ist nur nötig, wenn diese auf
|
||||
@@ -296,7 +296,81 @@ Hinweise:
|
||||
`PgPoolOptions` mit `acquire_timeout(10s)`, bricht aber ebenfalls ab, wenn
|
||||
der erste Connect scheitert. `Restart=on-failure` fängt das ab.
|
||||
|
||||
## 6. Reverse-Proxy
|
||||
## 6. Authentifizierung
|
||||
|
||||
Die AppView prüft seit Phase 9 Bearer-Tokens. Wie das zusammenhängt:
|
||||
|
||||
1. Die PDS stellt beim Login ein ES256-Access-JWT aus (`sub` = DID,
|
||||
`scope` = `com.atproto.access`, eine Stunde gültig).
|
||||
2. Die PDS veröffentlicht den *öffentlichen* Teil ihres P-256-Schlüssels unter
|
||||
`GET /.well-known/did.json`. `PDS_JWT_SECRET` verlässt den PDS-Prozess nicht.
|
||||
3. Die AppView holt dieses Dokument beim Start von `PDS_INTERNAL_URL`
|
||||
(Fallback: `PDS_PUBLIC_URL`), cached den Schlüssel und lädt ihn bei einem
|
||||
Verifikationsfehler einmal nach — höchstens einmal pro Minute, damit
|
||||
Müll-Tokens kein Werkzeug werden, die PDS zu fluten. Ein Schlüsselwechsel
|
||||
braucht also keinen Neustart der AppView.
|
||||
|
||||
Ist die PDS beim Start nicht erreichbar, warnt die AppView nur und startet
|
||||
trotzdem — sie indiziert den Firehose, was von der lokalen PDS unabhängig ist.
|
||||
Der Schlüssel wird dann beim ersten authentifizierten Request geholt. Klappt
|
||||
auch das nicht, antwortet sie `503 AuthUnavailable`: **fail closed**, nie
|
||||
fail open.
|
||||
|
||||
### Welche Endpoints
|
||||
|
||||
| Endpoint | Zugriff |
|
||||
|---|---|
|
||||
| `/api/timeline/home`, `/api/notifications`, `/api/notifications/count`, `/api/notifications/seen` | Token nötig, `sub` muss dem `did`-Parameter entsprechen |
|
||||
| `/api/profile*`, `/api/search`, `/api/post/*`, `/api/thread*`, `/api/followers`, `/api/following` | öffentlich (in AT Proto öffentliche Records) |
|
||||
| `/internal/ingest-commit` | `APPVIEW_INGEST_SECRET`, server-zu-server |
|
||||
|
||||
### Fehlercodes
|
||||
|
||||
| Fall | Status | `error` |
|
||||
|---|---|---|
|
||||
| Header fehlt oder ist kein Bearer | 401 | `AuthMissing` |
|
||||
| Signatur falsch, abgelaufen, falscher `scope` | 401 | `TokenInvalid` |
|
||||
| Token gültig, aber `sub` ≠ `did` | 403 | `Forbidden` |
|
||||
| Schlüssel der PDS nicht beschaffbar | 503 | `AuthUnavailable` |
|
||||
|
||||
`TokenInvalid` ist ein Vertrag mit dem Desktop-Client: daran erkennt er, dass
|
||||
er sein Access-JWT erneuern und den Request einmal wiederholen muss. Wer den
|
||||
Code umbenennt, loggt jeden Nutzer eine Stunde nach dem Login aus.
|
||||
|
||||
### `APPVIEW_AUTH_REQUIRED=false`
|
||||
|
||||
Schaltet die Prüfung ab und stellt das alte Verhalten her — gedacht für eine
|
||||
Instanz hinter VPN und für die fail-open-Integrationstests. Die AppView warnt
|
||||
beim Start in Großbuchstaben. Öffentlich erreichbar heißt das: jeder kann die
|
||||
Notifications jeder DID lesen und als gelesen markieren.
|
||||
|
||||
## 7. Firehose
|
||||
|
||||
Die PDS liefert `com.atproto.sync.subscribeRepos` als WebSocket aus, die
|
||||
AppView konsumiert ihn. Betrieblich wichtig:
|
||||
|
||||
* **Das Event liegt in derselben Transaktion wie der Commit.** Es kann keinen
|
||||
Commit ohne Event geben und umgekehrt.
|
||||
* **Die `seq` ist lückenfrei.** Ein globaler `pg_advisory_xact_lock` sorgt
|
||||
dafür, dass Commit-Reihenfolge und `seq`-Reihenfolge übereinstimmen — sonst
|
||||
könnte ein Consumer eine Nummer überspringen, die erst danach sichtbar wird,
|
||||
und sie nie nachholen. Preis: das Ende jeder schreibenden Transaktion ist
|
||||
über alle Accounts hinweg serialisiert.
|
||||
* **Cursor:** `?cursor=<seq>` liefert alles mit `seq > cursor` aus der
|
||||
Datenbank nach und geht dann nahtlos live weiter. Ohne Cursor nur live. Ein
|
||||
Cursor aus der Zukunft ist ein Fehler-Frame, ein zu alter ein
|
||||
`#info`/`OutdatedCursor`.
|
||||
* **`firehose_events` wächst unbegrenzt.** Es gibt keine Retention. Beschneiden
|
||||
ist sicher, weil ein zu alter Cursor sauber behandelt wird — wer die Tabelle
|
||||
aufräumt, sollte aber wissen, wie weit die eigenen Consumer zurückhängen
|
||||
dürfen (`pds_firehose_seq` in `/healthz` der AppView gegen `MAX(seq)`).
|
||||
* **Reverse-Proxy:** die Route braucht ein WebSocket-Upgrade (`Upgrade`/
|
||||
`Connection`-Header durchreichen) und einen Read-Timeout, der längere
|
||||
Ruhephasen überlebt.
|
||||
* `PDS_FIREHOSE_ENABLED=false` schaltet den Consumer in der AppView ab; lokale
|
||||
Commits hängen dann wieder allein am Best-Effort-Push.
|
||||
|
||||
## 8. Reverse-Proxy
|
||||
|
||||
### PDS
|
||||
|
||||
@@ -357,25 +431,31 @@ im DID-Doc-`serviceEndpoint`, im JWT-`iss` und als Basis der Blob-URLs.
|
||||
kein `/.well-known/atproto-did` aus (im Router nicht vorhanden). `at-identity`
|
||||
kann solche Dokumente *auflösen*, aber wer `did:web`-Handles auf dieser PDS
|
||||
betreiben will, muss die Dateien vorerst statisch über den Proxy ausliefern.
|
||||
Ebenfalls offen: `describeServer` gibt die DID hart als
|
||||
`did:web:pds.maarcadetweet.local` zurück, unabhängig von `PDS_PUBLIC_URL`.
|
||||
Die Service-DID wird inzwischen aus `PDS_PUBLIC_URL` abgeleitet
|
||||
(`AppConfig::pds_did()`, did:web mit `%3A`-kodiertem Port) und von
|
||||
`describeServer` **und** `/.well-known/did.json` identisch ausgeliefert. Das
|
||||
heißt auch: ändert sich `PDS_PUBLIC_URL`, ändert sich die Service-DID.
|
||||
|
||||
### AppView
|
||||
|
||||
Die AppView setzt ihr CORS selbst — `crates/appview/src/routes.rs`:
|
||||
Die AppView setzt ihr CORS selbst (`cors_layer()` in
|
||||
`crates/appview/src/routes.rs`). Ohne `APPVIEW_CORS_ORIGINS` bleibt es beim
|
||||
alten `Access-Control-Allow-Origin: *` — die AppView warnt dann beim Start.
|
||||
Mit gesetzter Variable gilt eine Allowlist:
|
||||
|
||||
```rust
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods(Any)
|
||||
.allow_headers(Any);
|
||||
```
|
||||
APPVIEW_CORS_ORIGINS=tauri://localhost,http://tauri.localhost,http://127.0.0.1:1430
|
||||
```
|
||||
|
||||
Also `Access-Control-Allow-Origin: *` für alle Routen, inklusive
|
||||
`/internal/ingest-commit`. Der Grund steht im Code: der Tauri-Webview ruft die
|
||||
AppView von einem anderen Origin aus auf (`http://127.0.0.1:1430` im Dev,
|
||||
`tauri://` / `asset://` im Bundle), und die Read-Endpoints tragen keine
|
||||
Auth-Cookies.
|
||||
Der Tauri-Webview ruft die AppView von einem anderen Origin aus auf — im Dev
|
||||
die Vite-Adresse, im Bundle `tauri://localhost` (macOS/Linux) bzw.
|
||||
`http://tauri.localhost` (Windows). Alle drei gehören in die Liste, sonst
|
||||
scheitert der Preflight.
|
||||
|
||||
`/internal/ingest-commit` liegt bewusst **außerhalb** der CORS-Schicht: die
|
||||
Route wird von der PDS server-zu-server aufgerufen, nie von einem Browser. Ein
|
||||
`Access-Control-Allow-Origin` darauf würde ausschließlich einer Webseite
|
||||
helfen, in den Index zu schreiben.
|
||||
|
||||
Für ein öffentliches Deployment heißt das:
|
||||
|
||||
@@ -408,7 +488,7 @@ HTTP-Aufrufe an PDS/AppView laufen über den Rust-IPC-Layer
|
||||
(`src-tauri/src/pds_client.rs`, `appview_client.rs`), nicht aus dem Webview —
|
||||
die CSP muss also für neue Backend-URLs nicht angefasst werden.
|
||||
|
||||
## 7. Health-Checks und Logs
|
||||
## 9. Health-Checks und Logs
|
||||
|
||||
### PDS
|
||||
|
||||
@@ -472,7 +552,7 @@ Log-Zeilen, auf die es sich lohnt zu achten:
|
||||
| `s3 ping failed at startup` | MinIO beim PDS-Start nicht erreichbar |
|
||||
| `plc submit failed (dev ok)` | PLC-Directory nicht erreichbar; die DID bleibt lokal gültig, ist aber global nicht registriert |
|
||||
|
||||
## 8. Neustart-Verhalten
|
||||
## 10. Neustart-Verhalten
|
||||
|
||||
**PDS.** Zustandslos bis auf Postgres und MinIO. Der In-Memory-Blockstore
|
||||
(`MemoryBlockstore` in `state.rs`) wird beim Start neu aufgebaut; persistent
|
||||
@@ -515,14 +595,22 @@ nach — zuerst über die lokale PDS (`PdsHandleResolver`, 2 s Timeout), dann PL
|
||||
bzw. `did:web`. Nach einem Neustart holt der erste Durchlauf das nach; der
|
||||
Zustand ist reine Anzeigekosmetik.
|
||||
|
||||
## 9. Was noch offen ist
|
||||
## 11. Was noch offen ist
|
||||
|
||||
* Kein Compose-Service für `pds-server` / `appview` — das Compose-File deckt nur
|
||||
Postgres und MinIO ab. Es gibt kein Dockerfile im Repo.
|
||||
* `at-blob` spricht ausschließlich MinIO ohne Signature V4 (siehe Modul-Doku in
|
||||
`crates/at-blob/src/s3.rs`); echtes AWS S3 funktioniert damit nicht.
|
||||
* Kein Graceful-Shutdown, keine Readiness- (im Unterschied zur Liveness-)Probe.
|
||||
* Keine konfigurierbare CORS-Allowlist in der AppView.
|
||||
* Kein `.well-known`-Handling in der PDS, `describeServer` liefert eine
|
||||
hartkodierte DID.
|
||||
* `aud` wird beim Token-Check nicht validiert (`verify_jwt` setzt
|
||||
`validate_aud = false`), obwohl die PDS `did:web:appview.maarcadetweet.local`
|
||||
einsetzt. Signatur, Ablauf, `scope` und `sub` werden geprüft.
|
||||
* Notifications werden nie gelöscht; ein Unlike/Unfollow lässt die Zeile stehen.
|
||||
* Kein Backfill-Werkzeug für Jetstream-Lücken.
|
||||
* **Keine Retention für `firehose_events`.** Die Tabelle wächst mit jedem
|
||||
Commit und wird nie beschnitten. Pruning ist sicher — ein Consumer mit zu
|
||||
altem Cursor bekommt `#info`/`OutdatedCursor` und läuft ab der ältesten
|
||||
überlebenden Zeile weiter — aber es gibt weder Job noch Policy dafür.
|
||||
* Der globale Advisory-Lock, der die `seq`-Vergabe ordnet, serialisiert das
|
||||
Ende jeder schreibenden Transaktion über alle Accounts hinweg. Das
|
||||
begrenzt den Schreibdurchsatz auf ein COMMIT nach dem anderen.
|
||||
|
||||
+47
-3
@@ -296,13 +296,57 @@ beiden.
|
||||
5. Mit einer älteren installierten Version gegenprüfen, dass `check()` das
|
||||
Update findet und die Signaturprüfung durchgeht.
|
||||
|
||||
## 8. Offene Punkte
|
||||
## 8. CI: Tag-Release über Gitea Actions
|
||||
|
||||
`.gitea/workflows/release.yml` baut Schritt 2 der Checkliste für **Windows und
|
||||
Linux** automatisch. Trigger ist ein Tag `v*.*.*` (zusätzlich manuell per
|
||||
`workflow_dispatch`, dann ohne Release-Anlage).
|
||||
|
||||
| Job | `runs-on` | Runner | Bundles |
|
||||
|---|---|---|---|
|
||||
| `windows` | `windows` | winbuild, 192.168.1.69 (on-demand) | `msi/*.msi`, `nsis/*-setup.exe` |
|
||||
| `linux` | `ubuntu-latest` | ci-runner, 192.168.1.72 | `deb/*.deb`, `rpm/*.rpm`, `appimage/*.AppImage` |
|
||||
|
||||
Beide Jobs laufen `npm ci` (es gibt eine `package-lock.json`) und danach den
|
||||
npm-Skript-Umweg `npm run tauri -- build --ci` aus `crates/tauri-app/`, damit
|
||||
die im Lock gepinnte `@tauri-apps/cli` benutzt wird und nicht die zufällig auf
|
||||
dem Runner installierte. Die Pfade sind die aus Abschnitt 5. Beide laden ihre
|
||||
Bundles als Job-Artefakt hoch **und** hängen sie an dasselbe Gitea-Release zum
|
||||
Tag (anlegen, und falls der andere Job schneller war, das vorhandene per Tag
|
||||
holen). Die Release-Beschreibung kommt aus dem passenden
|
||||
`## [<version>]`-Abschnitt einer `CHANGELOG.md`, sobald es eine gibt — bis
|
||||
dahin steht dort der Commit-SHA.
|
||||
|
||||
Zwei Dinge, die der Workflow *nicht* tut:
|
||||
|
||||
* **macOS.** Es gibt keinen macOS-Runner. `.dmg`/`.app` werden lokal nach
|
||||
Abschnitt 4 gebaut und im Gitea-Release von Hand angehängt.
|
||||
* **Signierte Updater-Artefakte.** Der Workflow baut ohne Release-Overlay und
|
||||
ohne `TAURI_SIGNING_PRIVATE_KEY*`; es entstehen also keine `.sig`-Dateien
|
||||
(Abschnitt 2 und 5). Für ein echtes Auto-Update müssen Overlay-Datei und
|
||||
Secrets ergänzt und der Build-Aufruf um
|
||||
`--config src-tauri/tauri.release.conf.json` erweitert werden.
|
||||
|
||||
**Vor dem Tag zu bumpen** (Schritt 1 der Checkliste): Die Release-Version kommt
|
||||
aus dem Tag, die Version im *Dateinamen* aus `src-tauri/tauri.conf.json`. Ohne
|
||||
Bump heißt das Artefakt zu `v0.2.0` weiterhin
|
||||
`maarcadetweet_0.1.0_x64-setup.exe`. `src-tauri/Cargo.toml` und `package.json`
|
||||
mitziehen — alle drei stehen aktuell auf `0.1.0`.
|
||||
|
||||
> **Falle:** Sobald `.gitea/workflows/` existiert, ignoriert Gitea
|
||||
> `.github/workflows/` vollständig — kommentarlos, ohne roten Lauf. Im
|
||||
> Nachbarprojekt `lserver` waren Tests dadurch einen Tag lang still
|
||||
> abgeschaltet. Dieses Repo hat kein `.github/`, und das soll so bleiben: neue
|
||||
> Workflows gehören nach `.gitea/workflows/`.
|
||||
|
||||
## 9. Offene Punkte
|
||||
|
||||
* Kein Release-Overlay im Repo — die Datei aus Abschnitt 2 muss angelegt
|
||||
werden. Der Endpoint `https://releases.maarcadetweet.local/…` in der aktuellen
|
||||
Config ist ein Platzhalter und existiert nicht.
|
||||
* Kein Update-Server, kein CI-Workflow, kein Skript, das `latest.json` erzeugt
|
||||
(`scripts/` ist leer).
|
||||
* Kein Update-Server und kein Skript, das `latest.json` erzeugt (`scripts/` ist
|
||||
leer). Der CI-Workflow aus Abschnitt 8 baut und veröffentlicht Installer,
|
||||
aber keine Updater-Artefakte.
|
||||
* Keine Code-Signierung/Notarisierung für macOS und keine Authenticode-Signatur
|
||||
für Windows konfiguriert (`bundle` enthält weder `macOS.signingIdentity` noch
|
||||
`windows.certificateThumbprint`). Der Tauri-Updater-Schlüssel ersetzt das
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"lexicon": 1,
|
||||
"id": "app.bsky.graph.follow",
|
||||
"defs": {
|
||||
"main": {
|
||||
"type": "record",
|
||||
"key": "tid",
|
||||
"record": {
|
||||
"type": "object",
|
||||
"required": ["subject", "createdAt"],
|
||||
"properties": {
|
||||
"subject": { "type": "string", "format": "did" },
|
||||
"createdAt": { "type": "datetime" }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
-- AppView database schema 0009: indexes for handle → DID lookups.
|
||||
--
|
||||
-- `/api/profile/<handle>` took 9.5 s on a 3.3 M-row `posts` table
|
||||
-- (measured against the dev instance). Both halves of `resolve_profile`
|
||||
-- were unindexed:
|
||||
--
|
||||
-- 1. SELECT did FROM profiles WHERE LOWER(handle) = LOWER($1)
|
||||
-- 2. SELECT did FROM posts WHERE handle = $1
|
||||
-- ORDER BY indexed_at DESC LIMIT 1 -- the fallback
|
||||
--
|
||||
-- Step 2 was a parallel sequential scan over every post ever indexed
|
||||
-- (`Rows Removed by Filter: 1101310` per worker), and it runs on every
|
||||
-- profile view in the client.
|
||||
--
|
||||
-- On `profiles`: migration 0007 dropped exactly this index, reasoning
|
||||
-- that "every caller derives a DID first (via posts.handle or the
|
||||
-- handle-sync worker) and then queries profiles by PK". That stopped
|
||||
-- being true when `resolve_profile` learned to prefer the profiles
|
||||
-- cache — it now asks `profiles` by handle *first*, precisely the
|
||||
-- lookup 0007 removed the support for. Re-added, matching the
|
||||
-- expression in the query (`LOWER(handle)`) so the planner can use it.
|
||||
CREATE INDEX IF NOT EXISTS profiles_handle_lower_idx
|
||||
ON profiles (LOWER(handle));
|
||||
|
||||
-- On `posts`: `(handle, indexed_at DESC)` covers filter *and* sort, so
|
||||
-- the LIMIT 1 becomes an index scan that stops at the first row.
|
||||
--
|
||||
-- Partial on `handle <> ''`: empty handles are the un-backfilled
|
||||
-- majority on a firehose-fed instance and are never looked up by this
|
||||
-- path (the handle-sync worker queries them through its own predicate),
|
||||
-- so excluding them keeps the index small on the largest table we have.
|
||||
CREATE INDEX IF NOT EXISTS posts_handle_indexed_at_idx
|
||||
ON posts (handle, indexed_at DESC)
|
||||
WHERE handle <> '';
|
||||
|
||||
-- =====================================================
|
||||
-- posts: the cold-start global feed
|
||||
-- =====================================================
|
||||
--
|
||||
-- `/api/timeline/home` falls back to the global recent feed for users
|
||||
-- without a follow graph — every new account's first screen. It took
|
||||
-- 7.4 s (parallel seq scan + top-N sort over 3.3 M rows) and timed out
|
||||
-- the integration tests' 5 s client.
|
||||
--
|
||||
-- `posts_collection_indexed_at_uri_idx (collection, indexed_at DESC,
|
||||
-- uri DESC)` cannot serve it: the query filters
|
||||
-- `collection IN ('app.twi.post','app.bsky.feed.post')`, and with two
|
||||
-- leading values the index no longer yields rows in `indexed_at` order,
|
||||
-- so the planner falls back to scanning and sorting.
|
||||
--
|
||||
-- A partial index over exactly that predicate moves the collection
|
||||
-- filter into the index definition, which leaves `(indexed_at DESC,
|
||||
-- uri DESC)` as the sort key — the LIMIT then stops after the first
|
||||
-- page. Same shape as the existing `posts_did_indexed_at_uri_idx`,
|
||||
-- which is partial on the same two collections.
|
||||
CREATE INDEX IF NOT EXISTS posts_feed_indexed_at_uri_idx
|
||||
ON posts (indexed_at DESC, uri DESC)
|
||||
WHERE collection IN ('app.twi.post', 'app.bsky.feed.post');
|
||||
@@ -0,0 +1,44 @@
|
||||
-- AppView database schema 0010: cursor for the local PDS firehose.
|
||||
--
|
||||
-- Why a second cursor table
|
||||
--
|
||||
-- The AppView now consumes two event streams, and they are numbered in
|
||||
-- completely different spaces:
|
||||
--
|
||||
-- * `jetstream_cursor.cursor` is a Jetstream `time_us` — microseconds
|
||||
-- since the epoch, produced by a public relay we do not control.
|
||||
-- * this table's `cursor` is the `seq` of our own PDS's
|
||||
-- `com.atproto.sync.subscribeRepos` — a small monotonic counter
|
||||
-- that starts at 1 in a fresh PDS database.
|
||||
--
|
||||
-- Sharing one row between them would mean the larger of the two values
|
||||
-- (always the Jetstream timestamp) permanently swallowing the other:
|
||||
-- `cursor_advance` uses GREATEST, so the very first Jetstream event
|
||||
-- would push the PDS cursor to ~1.7e15 and every subsequent
|
||||
-- subscribeRepos connect would ask for a sequence the PDS will never
|
||||
-- reach. Hence a table of its own, deliberately in the same shape as
|
||||
-- `jetstream_cursor` so both read/advance the same way.
|
||||
--
|
||||
-- Shape
|
||||
-- id pinned to 1 by a CHECK — a single-row table, the same
|
||||
-- pattern `jetstream_cursor` uses. It makes "advance the
|
||||
-- cursor" a plain UPDATE with no upsert dance and makes a
|
||||
-- second row impossible to create by accident.
|
||||
-- cursor the last `seq` we durably applied. 0 means "nothing
|
||||
-- yet": the consumer then subscribes without a `cursor`
|
||||
-- query parameter, which the PDS reads as "start from the
|
||||
-- current head" rather than replaying the entire repo
|
||||
-- history into a fresh index.
|
||||
-- updated_at observability only — how stale the stream is can be
|
||||
-- read straight off the row.
|
||||
--
|
||||
-- The row is inserted here so `cursor_advance`'s UPDATE always has a
|
||||
-- target; `pds_firehose::cursor_get` still tolerates a missing row and
|
||||
-- returns 0.
|
||||
|
||||
CREATE TABLE pds_firehose_cursor (
|
||||
id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
cursor BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO pds_firehose_cursor (id, cursor) VALUES (1, 0);
|
||||
@@ -0,0 +1,79 @@
|
||||
-- AppView database schema 0011: remember which record a follow came from.
|
||||
--
|
||||
-- Why
|
||||
--
|
||||
-- A `follows` row was addressable only as `(follower_did, subject_did)`.
|
||||
-- That is the right identity for the *relationship*, but it is not the
|
||||
-- identity a delete event carries. A firehose / Jetstream delete op is
|
||||
-- just `did` + `rkey`:
|
||||
--
|
||||
-- {"action": "delete", "path": "app.bsky.graph.follow/3lmnop"}
|
||||
--
|
||||
-- There is no record body on a delete — the record is gone, that is the
|
||||
-- whole point of the event — so the subject DID is nowhere in it. With
|
||||
-- no rkey stored, the indexer had no way from `3lmnop` back to
|
||||
-- "did:plc:bob" and logged-and-skipped the op
|
||||
-- (`indexer::apply_commit`, `app.bsky.graph.follow` arm).
|
||||
--
|
||||
-- The practical consequence: unfollows only ever landed through the
|
||||
-- PDS's best-effort `POST /internal/ingest-commit` push, which knows the
|
||||
-- subject from its own snapshot. That push has no retry and no
|
||||
-- acknowledgement (see `pds_firehose`'s module docs). If it was lost —
|
||||
-- AppView restarting, request timing out — the follow stayed in the
|
||||
-- index forever, and the firehose, the stream that exists precisely to
|
||||
-- repair such gaps, could not repair this one. Storing the rkey closes
|
||||
-- that hole: the firehose replay can now apply the unfollow on its own.
|
||||
--
|
||||
-- The primary key deliberately stays `(follower_did, subject_did)`
|
||||
-- ------------------------------------------------------------------
|
||||
-- It is what makes `upsert_follow` idempotent. The same follow reaches
|
||||
-- us over both transports (push *and* firehose) and again after any
|
||||
-- replay, and every one of those must converge on one row. Keying on
|
||||
-- the rkey instead — or adding it to the key — would make a re-follow
|
||||
-- under a fresh rkey a *second* row for the same relationship, and then
|
||||
-- `follower_count` would count the same follower twice.
|
||||
--
|
||||
-- So `rkey` is not identity here; it is a second *access path* to a row
|
||||
-- the primary key already identifies.
|
||||
--
|
||||
-- Nullable, because history has no rkey
|
||||
-- -------------------------------------
|
||||
-- Every row written before this migration was inserted without one, and
|
||||
-- there is nothing to backfill it from: the AppView never stored the
|
||||
-- follow record itself. A NOT NULL column would need a fabricated
|
||||
-- placeholder that a later delete could accidentally match. NULL says
|
||||
-- exactly what is true — "we do not know which record this came from" —
|
||||
-- and a delete-by-rkey simply finds nothing for those rows, which is the
|
||||
-- documented no-op path in `indexer::delete_follow_by_rkey`. Those rows
|
||||
-- keep working through the push path (which sends `subject_did`) and
|
||||
-- heal on their own the next time the follow is re-created.
|
||||
--
|
||||
-- Re-follow under a new rkey
|
||||
-- --------------------------
|
||||
-- Follow → unfollow → follow again produces a *different* rkey each
|
||||
-- time (rkeys are TIDs; the client never reuses one). The upsert
|
||||
-- therefore hits the primary key and overwrites `rkey` with the newer
|
||||
-- record's: the youngest record wins. That ordering is what makes a
|
||||
-- late or replayed delete for the *old* rkey harmless — it matches no
|
||||
-- row and is skipped, instead of tearing down a follow that is
|
||||
-- currently live.
|
||||
--
|
||||
-- The index is NOT unique
|
||||
-- -----------------------
|
||||
-- `(follower_did, rkey)` is unique in practice — an rkey identifies one
|
||||
-- record inside one repo's collection — but a unique index would turn
|
||||
-- the one situation this migration exists for into a *write failure*:
|
||||
-- if a delete was lost and a create later reused that rkey, the insert
|
||||
-- would abort instead of the stale row being cleaned up. An index whose
|
||||
-- only job is to serve a lookup should not be able to reject a write.
|
||||
-- Partial (`WHERE rkey IS NOT NULL`) because a lookup key is never
|
||||
-- NULL, so the pre-migration rows have no business bloating it.
|
||||
|
||||
ALTER TABLE follows ADD COLUMN IF NOT EXISTS rkey TEXT;
|
||||
|
||||
-- Serves `DELETE FROM follows WHERE follower_did = $1 AND rkey = $2
|
||||
-- RETURNING subject_did` — the delete path for a firehose
|
||||
-- unfollow, which is the only lookup this column exists for.
|
||||
CREATE INDEX IF NOT EXISTS follows_follower_rkey_idx
|
||||
ON follows (follower_did, rkey)
|
||||
WHERE rkey IS NOT NULL;
|
||||
@@ -0,0 +1,112 @@
|
||||
-- PDS database schema 0003: the firehose event log.
|
||||
--
|
||||
-- Why
|
||||
--
|
||||
-- Until now the PDS produced no `com.atproto.sync.subscribeRepos` stream at
|
||||
-- all. The only way a local record reached the AppView was the best-effort
|
||||
-- HTTP push in `appview_push.rs` — a fire-and-forget `tokio::spawn` that is
|
||||
-- explicitly documented as "the Jetstream replay will catch up". There is no
|
||||
-- Jetstream replay for records that only exist on this PDS, so a dropped push
|
||||
-- meant the post was simply never indexed. Nothing retried it, and nothing
|
||||
-- could: the commit lived in `repos` / `repo_blocks` but there was no ordered
|
||||
-- log of *what changed* for a consumer to walk.
|
||||
--
|
||||
-- This table is that log. Every repo write appends exactly one row, in the
|
||||
-- same transaction as the head-pointer update, so the sequence and the repo
|
||||
-- head can never disagree. A consumer that reconnects with a cursor replays
|
||||
-- from here; a consumer that is live gets the same rows pushed over a
|
||||
-- broadcast channel.
|
||||
--
|
||||
-- Column choices
|
||||
--
|
||||
-- seq BIGSERIAL PRIMARY KEY — the cursor. It has to be a single
|
||||
-- monotonically increasing integer because that is what the
|
||||
-- `subscribeRepos` wire contract hands the client and takes
|
||||
-- back as `?cursor=`. BIGSERIAL (not an `(timestamp, id)`
|
||||
-- keyset like the AppView's notifications table) because the
|
||||
-- protocol's cursor is opaque-but-numeric and clients compare
|
||||
-- it with `>`.
|
||||
--
|
||||
-- Sequence values are handed out at INSERT time, which by
|
||||
-- itself does NOT guarantee that they become *visible* in seq
|
||||
-- order — two transactions can grab 5 and 6 and commit in the
|
||||
-- opposite order, leaving a reader that polls in between with a
|
||||
-- gap it would never fill. The write path therefore takes
|
||||
-- `pg_advisory_xact_lock` on a fixed key immediately before
|
||||
-- this INSERT (see `routes::helpers::apply_repo_write`), which
|
||||
-- serialises the tail of every firehose-writing transaction so
|
||||
-- commit order == seq order. That is what makes "give me
|
||||
-- everything with seq > N" an exact, gap-free replay rather
|
||||
-- than a best guess.
|
||||
--
|
||||
-- did the repo the event belongs to. Not a FK to `users(did)`:
|
||||
-- the log outlives the account. If a user is deleted we still
|
||||
-- want consumers that are mid-replay to see the events that
|
||||
-- already happened rather than have the rows cascade out from
|
||||
-- under their cursor.
|
||||
--
|
||||
-- rev the new commit's revision (TID string), mirrored from
|
||||
-- `repos.rev`. Goes out as the frame's `rev`.
|
||||
--
|
||||
-- since the *previous* commit's rev, or NULL for the first commit on
|
||||
-- a repo. The frame's `since` field; a consumer uses it to
|
||||
-- detect that it missed an intermediate commit.
|
||||
--
|
||||
-- commit_cid BYTEA holding the raw binary CID of the new commit, stored
|
||||
-- the same way `repos.head_cid` stores it so the two are
|
||||
-- directly comparable with `=` and no text/binary conversion
|
||||
-- is needed to join them.
|
||||
--
|
||||
-- blocks BYTEA holding a complete CAR v1 file: the commit block as the
|
||||
-- root plus every block this commit newly created (MST nodes
|
||||
-- and record values). Stored pre-serialised rather than
|
||||
-- reassembled from `repo_blocks` at read time because the
|
||||
-- *diff* — which blocks were new for this particular commit —
|
||||
-- is only knowable at write time. Recomputing it later would
|
||||
-- mean diffing two MST snapshots on every replayed event.
|
||||
--
|
||||
-- ops JSONB array of `{action, path, cid}`, the same objects that
|
||||
-- go into the frame's `ops` field. JSONB rather than a child
|
||||
-- table because it is always read as a whole, is never queried
|
||||
-- by content, and a child table would need its own ordering
|
||||
-- column to reproduce the array faithfully.
|
||||
--
|
||||
-- created_at when the event was appended. This is what the frame's `time`
|
||||
-- field carries, so a replayed frame is byte-identical to the
|
||||
-- live one that was broadcast at commit time — a consumer that
|
||||
-- deduplicates by hashing frames does not see two different
|
||||
-- frames for one event.
|
||||
--
|
||||
-- Retention: there is none
|
||||
-- ------------------------
|
||||
-- Nothing prunes this table. It grows by one row per repo write, and each row
|
||||
-- carries a CAR of the commit's new blocks (a few hundred bytes for a plain
|
||||
-- post, more when a record is large). At the volume this deployment sees that
|
||||
-- is fine for a long time, but it is unbounded, and an operator who wants a
|
||||
-- bound has to add one. Deleting the oldest rows is safe: a client whose
|
||||
-- cursor points before the surviving range gets an `#info`/`OutdatedCursor`
|
||||
-- frame and resumes from the oldest row that still exists. See the module
|
||||
-- header of `crates/pds-server/src/firehose.rs`.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS firehose_events (
|
||||
seq BIGSERIAL PRIMARY KEY,
|
||||
did TEXT NOT NULL,
|
||||
rev TEXT NOT NULL,
|
||||
since TEXT,
|
||||
commit_cid BYTEA NOT NULL,
|
||||
blocks BYTEA NOT NULL,
|
||||
ops JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
-- Cursor replay is `WHERE seq > $1 ORDER BY seq LIMIT $2`, which the
|
||||
-- BIGSERIAL primary key's own index already serves — no second index for
|
||||
-- that, on purpose: an extra index on `seq` would be pure write amplification
|
||||
-- on the hottest path in this table.
|
||||
--
|
||||
-- What the PK does *not* serve is "replay one repo", which is how an operator
|
||||
-- re-drives a single account into the AppView after an ingest bug, and how
|
||||
-- `getRepo`-style backfills are debugged. `(did, seq)` covers that and keeps
|
||||
-- the per-repo scan in seq order.
|
||||
CREATE INDEX IF NOT EXISTS firehose_events_did_seq_idx
|
||||
ON firehose_events (did, seq);
|
||||
@@ -0,0 +1,186 @@
|
||||
-- PDS database schema 0004: invite codes for `com.atproto.server.createAccount`.
|
||||
--
|
||||
-- Why
|
||||
--
|
||||
-- Until now `createAccount` had no gate of any kind: no invite code, no rate
|
||||
-- limit, and `describeServer` advertised `invite_code_required: false`. That
|
||||
-- was survivable while the only reachable instance was `127.0.0.1:2583`. It
|
||||
-- stops being survivable the moment the PDS answers on a public name, because
|
||||
-- every accepted account is not just a row in `users` — it creates a `repos`
|
||||
-- head, a key pair the server has to keep, an MST that grows with every write,
|
||||
-- and firehose events that every subscribed AppView is obliged to index. A
|
||||
-- single script could mint accounts until the disk filled up, and nothing in
|
||||
-- the write path would consider that abnormal.
|
||||
--
|
||||
-- This migration adds the smallest gate that actually closes that hole: an
|
||||
-- account may only be created by presenting a code the operator handed out.
|
||||
-- The gate is opt-in via `PDS_INVITE_REQUIRED` (default `false`, so the dozens
|
||||
-- of integration tests that create throwaway accounts keep working); the
|
||||
-- tables below exist unconditionally so that switching the flag on is a
|
||||
-- restart, not a migration.
|
||||
--
|
||||
-- Two tables, not one
|
||||
-- -------------------
|
||||
-- A code can be worth more than one account (`--uses 5` for a group of
|
||||
-- friends, a conference badge, a family). That means "who redeemed this code"
|
||||
-- is a *set*, not a single column, so it cannot live on the code row. Putting
|
||||
-- a `redeemed_by TEXT` column on `invite_codes` would have forced either
|
||||
-- one-code-one-account (losing the multi-use case the operator actually wants)
|
||||
-- or an array column that no foreign key, index or `COUNT(*)` can reason
|
||||
-- about. `invite_code_uses` is that set, one row per redemption.
|
||||
--
|
||||
-- The redemption *counter* still lives on the code row even though it is
|
||||
-- derivable from `COUNT(*)` over `invite_code_uses`. That duplication is
|
||||
-- deliberate and is the entire concurrency story — see below.
|
||||
--
|
||||
-- invite_codes
|
||||
-- ------------
|
||||
--
|
||||
-- code TEXT PRIMARY KEY — the code itself, and the natural key. No
|
||||
-- surrogate `id`: the code is what the user types, what the
|
||||
-- operator pastes into a chat window, and what the redeem query
|
||||
-- looks up, so a second identifier would only add a join.
|
||||
-- Codes are generated lowercase from a 32-character
|
||||
-- Crockford-style alphabet (`crates/pds-server/src/invite.rs`),
|
||||
-- and the server lowercases and trims what the client sends
|
||||
-- before looking it up. Because every stored code is already
|
||||
-- lowercase ASCII, that normalisation happens in Rust rather
|
||||
-- than as `WHERE lower(code) = …`, which would throw away this
|
||||
-- primary-key index on the hottest lookup this table has.
|
||||
--
|
||||
-- created_at when the operator minted it. Purely for the `invite list`
|
||||
-- output and for answering "where did this wave of signups come
|
||||
-- from" after the fact.
|
||||
--
|
||||
-- note free-text label the operator can attach at creation time
|
||||
-- (`--note "meetup 2026-09"`). Nullable, never interpreted.
|
||||
-- It exists because a bare list of random strings is unusable
|
||||
-- a month later.
|
||||
--
|
||||
-- max_uses how many accounts this code may create. `CHECK (max_uses > 0)`
|
||||
-- because a zero-use code is not a thing you would ever mean to
|
||||
-- create — it is a typo that would silently hand out a code that
|
||||
-- can never work.
|
||||
--
|
||||
-- used_count how many it has already created. Kept in sync with
|
||||
-- `invite_code_uses` inside the same transaction that writes
|
||||
-- both.
|
||||
--
|
||||
-- disabled a code the operator wants to stop honouring *without* losing
|
||||
-- the audit trail. Deleting the row would work for the future
|
||||
-- but would take the `invite_code_uses` rows with it (see the
|
||||
-- FK below) and with them the record of which accounts came
|
||||
-- from that code — which is the one question you ask when a
|
||||
-- code leaks. A boolean keeps the history and is checked in the
|
||||
-- same `WHERE` clause as the counter, so disabling costs nothing
|
||||
-- at redeem time.
|
||||
--
|
||||
-- CHECK (used_count <= max_uses) — the belt to the redeem query's braces.
|
||||
-- The application never over-redeems (the conditional UPDATE
|
||||
-- below makes that impossible), but this constraint means that
|
||||
-- *no* future query — a hand-written `UPDATE` during an
|
||||
-- incident, a bug in a later refactor — can hand out more
|
||||
-- accounts than the operator authorised. The database refuses.
|
||||
--
|
||||
-- How the redeem race is closed
|
||||
-- -----------------------------
|
||||
-- The obvious implementation is "SELECT the code, check `used_count <
|
||||
-- max_uses` in Rust, then UPDATE". That is a check-then-act, and two
|
||||
-- registrations arriving at the same instant with the same last remaining use
|
||||
-- both read `used_count = 0`, both decide they are allowed, and both write
|
||||
-- `used_count = 1` — two accounts from a one-use code, with the row still
|
||||
-- claiming a single redemption.
|
||||
--
|
||||
-- So the check and the act are one statement, and the database performs both:
|
||||
--
|
||||
-- UPDATE invite_codes
|
||||
-- SET used_count = used_count + 1
|
||||
-- WHERE code = $1
|
||||
-- AND NOT disabled
|
||||
-- AND used_count < max_uses
|
||||
-- RETURNING used_count, max_uses;
|
||||
--
|
||||
-- Under Postgres's READ COMMITTED isolation the second transaction to reach
|
||||
-- this row blocks on the row lock the first one took. When the first commits,
|
||||
-- the second does not proceed with its stale snapshot: it re-reads the updated
|
||||
-- row and re-evaluates the `WHERE` clause against it (EvalPlanQual). The
|
||||
-- counter is now `1`, `used_count < max_uses` is false, the row no longer
|
||||
-- matches, and the statement returns zero rows. Zero rows returned *is* the
|
||||
-- rejection — the route turns it into `400 InvalidInviteCode` without ever
|
||||
-- having formed an opinion of its own about whether the code was still valid.
|
||||
--
|
||||
-- If the first transaction instead rolls back — the handle turned out to be
|
||||
-- taken, key generation failed, anything — the lock is released with the
|
||||
-- counter back at `0` and the waiting transaction's re-check succeeds. The
|
||||
-- code is only consumed by a registration that actually completed, which is
|
||||
-- why the redemption is issued inside `create_account`'s existing
|
||||
-- transaction rather than before it.
|
||||
--
|
||||
-- This is also why `used_count` is stored rather than computed. A
|
||||
-- `COUNT(*) FROM invite_code_uses` has no row to lock — concurrent counters
|
||||
-- both see the same pre-insert count and both pass. The counter column gives
|
||||
-- the conditional UPDATE a single row to serialise on.
|
||||
CREATE TABLE IF NOT EXISTS invite_codes (
|
||||
code TEXT PRIMARY KEY,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
note TEXT,
|
||||
max_uses INTEGER NOT NULL DEFAULT 1,
|
||||
used_count INTEGER NOT NULL DEFAULT 0,
|
||||
disabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
CONSTRAINT invite_codes_max_uses_positive CHECK (max_uses > 0),
|
||||
CONSTRAINT invite_codes_used_count_sane CHECK (used_count >= 0 AND used_count <= max_uses)
|
||||
);
|
||||
|
||||
-- `invite list` shows the newest codes first, and that is the only listing
|
||||
-- this table has. Small table, but the operator runs it interactively and an
|
||||
-- ordered index keeps the output instant even after a few thousand codes.
|
||||
CREATE INDEX IF NOT EXISTS invite_codes_created_at_idx
|
||||
ON invite_codes (created_at DESC);
|
||||
|
||||
-- =====================================================
|
||||
-- invite_code_uses — which account came from which code
|
||||
-- =====================================================
|
||||
--
|
||||
-- code FK to `invite_codes(code)` ON DELETE CASCADE. Cascading is the
|
||||
-- right call *here* (unlike `firehose_events.did` in 0003, which
|
||||
-- deliberately has no FK) because these rows are meaningless
|
||||
-- without the code they describe: they exist to answer "which
|
||||
-- accounts did code X create", and a use-row whose code has been
|
||||
-- deleted answers nothing. The operator who wants to stop a code
|
||||
-- but keep the trail sets `disabled` instead of deleting — which
|
||||
-- is precisely why that column exists.
|
||||
--
|
||||
-- did the account that was created. Intentionally NOT a foreign key
|
||||
-- to `users(did)`: this is an audit record of something that
|
||||
-- happened, and it has to survive the account being deleted. If
|
||||
-- it cascaded from `users`, deleting a spam account would erase
|
||||
-- the evidence linking it to the code that let it in — the exact
|
||||
-- moment the link matters most. The trade-off is that a `did`
|
||||
-- here may point at a user that no longer exists; that is
|
||||
-- accepted and is what an audit log looks like.
|
||||
--
|
||||
-- handle the handle as it was at creation time, denormalised on
|
||||
-- purpose. Handles can change, and `users` may be gone entirely
|
||||
-- (see above); this column is a snapshot so the listing stays
|
||||
-- readable without a join that may find nothing.
|
||||
--
|
||||
-- used_at when the redemption happened.
|
||||
--
|
||||
-- PRIMARY KEY (code, did) — one account can only consume a given code once.
|
||||
-- This is not the mechanism that enforces the use limit (the
|
||||
-- conditional UPDATE is), it is a guard against a redemption
|
||||
-- being recorded twice for one account, which would make
|
||||
-- `used_count` and this table disagree.
|
||||
CREATE TABLE IF NOT EXISTS invite_code_uses (
|
||||
code TEXT NOT NULL REFERENCES invite_codes(code) ON DELETE CASCADE,
|
||||
did TEXT NOT NULL,
|
||||
handle TEXT NOT NULL,
|
||||
used_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (code, did)
|
||||
);
|
||||
|
||||
-- The reverse lookup: "which code let this account in?". Asked per-account
|
||||
-- during abuse triage, so it needs its own index — the (code, did) primary
|
||||
-- key cannot serve a query whose only predicate is `did`.
|
||||
CREATE INDEX IF NOT EXISTS invite_code_uses_did_idx
|
||||
ON invite_code_uses (did);
|
||||
Reference in New Issue
Block a user