feat(appview): PDS-Firehose konsumieren
Gegenstück zum subscribeRepos-Endpoint: WebSocket-Consumer mit persistiertem seq-Cursor, Reconnect-Backoff und Behandlung von #info/OutdatedCursor. Eigene Cursor-Tabelle statt einer Zeile in jetstream_cursor: dort steht ein time_us in der Größenordnung 1.7e15, die seq ist ein kleiner Zähler ab 1. Geteilt hätte GREATEST den PDS-Cursor sofort in eine Zukunft geschoben, die die PDS nie erreicht. Kein neuer Indexer-Pfad — jede Op wird in die Single-Op-Form übersetzt, die apply_commit schon vom Jetstream kennt. Push und Firehose liefern denselben Commit doppelt; das ist unkritisch, weil die Schreibpfade Upserts sind und der Dedupe-Index der Notifications den Rest abfängt. Mit einem Test festgehalten statt vorausgesetzt. Der CAR-Reader ist neu (es gab nur einen Writer, und der liegt in einem Binary-Crate ohne lib-Target). Der CBOR-Reader arbeitet mit explizitem Offset, weil ein Frame zwei hintereinander geschriebene Werte sind, und akzeptiert CID-Links in beiden Schreibweisen — die Blöcke tragen Strings. /healthz meldet beide Ströme getrennt; sie fallen unabhängig voneinander aus. Verifiziert mit totem Push-Ziel: der Post kam trotzdem an. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013HC9HLrUU1LNwkzp8nkDLX
This commit is contained in:
tomdebone
co-authored by
Claude Opus 5
Claude Opus 5
parent
d6947c2576
commit
124a90dc07
@@ -0,0 +1,394 @@
|
||||
//! CAR v1 *reader*.
|
||||
//!
|
||||
//! The repo has a writer (`pds-server/src/car.rs`) but no reader the
|
||||
//! AppView could use: `pds-server` is a binary crate with no library
|
||||
//! target, so its `parse` helper is unreachable from here. The PDS
|
||||
//! firehose hands us a CAR in every `#commit` frame's `blocks` field, so
|
||||
//! the AppView needs its own.
|
||||
//!
|
||||
//! Format (<https://ipld.io/specs/transport/car/carv1/>):
|
||||
//!
|
||||
//! ```text
|
||||
//! [ varint: header_len | DAG-CBOR header ] { version: 1, roots: [CID] }
|
||||
//! [ varint: section_len | CID | block bytes ] block 1
|
||||
//! [ varint: section_len | CID | block bytes ] block 2
|
||||
//! ...
|
||||
//! ```
|
||||
//!
|
||||
//! The header is decoded with [`crate::cbor`], which accepts both the
|
||||
//! spec's `tag(42) + bytes(0x00 || cid)` link and the bare
|
||||
//! `tag(42) + bytes(cid)` this codebase's writer emits.
|
||||
//!
|
||||
//! Only structure is validated. Block CIDs are *not* re-hashed here:
|
||||
//! the firehose connection is to our own PDS over the cluster-internal
|
||||
//! URL, and a mismatch would mean a bug rather than an attack. See
|
||||
//! [`verify_block_cids`] for the opt-in check the tests use.
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use cid::Cid;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::cbor;
|
||||
|
||||
/// The parsed CAR header.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CarHeader {
|
||||
pub version: u64,
|
||||
pub roots: Vec<Cid>,
|
||||
}
|
||||
|
||||
/// One `(CID, bytes)` pair out of a CAR file.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct CarBlock {
|
||||
pub cid: Cid,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// A CAR file's header plus its blocks, in file order.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Car {
|
||||
pub header: CarHeader,
|
||||
pub blocks: Vec<CarBlock>,
|
||||
}
|
||||
|
||||
impl Car {
|
||||
/// Index the blocks by CID for lookup by the commit's `ops`.
|
||||
///
|
||||
/// Duplicate CIDs keep the first occurrence, matching the writer's
|
||||
/// own de-duplication.
|
||||
pub fn block_map(&self) -> HashMap<Cid, &[u8]> {
|
||||
let mut map = HashMap::with_capacity(self.blocks.len());
|
||||
for b in &self.blocks {
|
||||
map.entry(b.cid).or_insert(b.data.as_slice());
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
/// The first root, if the header declares one.
|
||||
///
|
||||
/// `allow(dead_code)`: the ingest path doesn't need the commit
|
||||
/// block itself (the ops carry the record CIDs), but a CAR reader
|
||||
/// that cannot name its root is a reader with a hole in it, and the
|
||||
/// tests read it.
|
||||
#[allow(dead_code)]
|
||||
pub fn root(&self) -> Option<Cid> {
|
||||
self.roots().first().copied()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn roots(&self) -> &[Cid] {
|
||||
&self.header.roots
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a CAR v1 byte stream.
|
||||
pub fn parse(bytes: &[u8]) -> Result<Car> {
|
||||
let mut p = 0usize;
|
||||
|
||||
let (header_len, n) = read_varint(bytes, p)?;
|
||||
p += n;
|
||||
let header_end = checked_end(bytes, p, header_len, "CAR header")?;
|
||||
let header = decode_header(&bytes[p..header_end])?;
|
||||
p = header_end;
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
while p < bytes.len() {
|
||||
let (section_len, n) = read_varint(bytes, p)?;
|
||||
let section_start = p + n;
|
||||
let section_end = checked_end(bytes, section_start, section_len, "CAR section")?;
|
||||
let section = &bytes[section_start..section_end];
|
||||
let cid = Cid::read_bytes(section)
|
||||
.map_err(|e| anyhow!("invalid CID in CAR section at offset {section_start}: {e}"))?;
|
||||
let cid_len = cid.encoded_len();
|
||||
if cid_len > section.len() {
|
||||
bail!("CAR section at {section_start} is shorter than its CID");
|
||||
}
|
||||
blocks.push(CarBlock {
|
||||
cid,
|
||||
data: section[cid_len..].to_vec(),
|
||||
});
|
||||
p = section_end;
|
||||
}
|
||||
|
||||
Ok(Car { header, blocks })
|
||||
}
|
||||
|
||||
/// Re-hash every block and compare against its declared CID.
|
||||
///
|
||||
/// Not called on the ingest path (see the module docs); the CAR reader
|
||||
/// tests use it to prove the reader hands back the bytes the writer put
|
||||
/// in, unshifted by an off-by-one in the section framing.
|
||||
#[allow(dead_code)]
|
||||
pub fn verify_block_cids(car: &Car) -> Result<()> {
|
||||
for b in &car.blocks {
|
||||
let recomputed = at_crypto::cid::cid_for_cbor(&b.data)?;
|
||||
if recomputed != b.cid {
|
||||
bail!("CAR block CID mismatch: declared {}, computed {recomputed}", b.cid);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn decode_header(bytes: &[u8]) -> Result<CarHeader> {
|
||||
let value = cbor::decode(bytes)?;
|
||||
let version = value
|
||||
.get("version")
|
||||
.and_then(|v| v.as_i64())
|
||||
.ok_or_else(|| anyhow!("CAR header missing `version`"))?;
|
||||
if version != 1 {
|
||||
bail!("unsupported CAR version {version} (only v1 is defined for atproto)");
|
||||
}
|
||||
// `roots` is required by the spec but may legitimately be empty.
|
||||
let roots = match value.get("roots") {
|
||||
Some(v) => v
|
||||
.as_array()
|
||||
.ok_or_else(|| anyhow!("CAR header `roots` is not an array"))?
|
||||
.iter()
|
||||
.map(|item| {
|
||||
item.as_cid()
|
||||
.ok_or_else(|| anyhow!("CAR header root is not a CID link"))
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?,
|
||||
None => Vec::new(),
|
||||
};
|
||||
Ok(CarHeader {
|
||||
version: version as u64,
|
||||
roots,
|
||||
})
|
||||
}
|
||||
|
||||
/// LEB128 unsigned varint, the length prefix CAR uses.
|
||||
fn read_varint(bytes: &[u8], offset: usize) -> Result<(u64, usize)> {
|
||||
let mut value: u64 = 0;
|
||||
let mut shift = 0u32;
|
||||
let mut i = offset;
|
||||
loop {
|
||||
let b = *bytes
|
||||
.get(i)
|
||||
.ok_or_else(|| anyhow!("varint extends past end of CAR input at {offset}"))?;
|
||||
i += 1;
|
||||
value |= u64::from(b & 0x7f) << shift;
|
||||
if b & 0x80 == 0 {
|
||||
return Ok((value, i - offset));
|
||||
}
|
||||
shift += 7;
|
||||
if shift >= 64 {
|
||||
bail!("varint longer than 64 bits at offset {offset}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_end(bytes: &[u8], pos: usize, len: u64, what: &str) -> Result<usize> {
|
||||
let len = usize::try_from(len).map_err(|_| anyhow!("{what} length overflows usize"))?;
|
||||
let end = pos
|
||||
.checked_add(len)
|
||||
.ok_or_else(|| anyhow!("{what} length overflows"))?;
|
||||
if end > bytes.len() {
|
||||
bail!("{what} length {len} exceeds input (offset {pos}, total {})", bytes.len());
|
||||
}
|
||||
Ok(end)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_writer {
|
||||
//! A byte-for-byte copy of `pds-server/src/car.rs`'s encoder, so the
|
||||
//! reader's tests exercise *the writer's* output rather than a
|
||||
//! convenient fiction.
|
||||
//!
|
||||
//! Copied rather than imported because `pds-server` has no library
|
||||
//! target. If the writer ever changes shape, the integration test
|
||||
//! `pds_firehose_integration::car_reader_parses_a_real_repo_export`
|
||||
//! is the tripwire: it parses a CAR produced by the running PDS.
|
||||
|
||||
use cid::Cid;
|
||||
|
||||
fn cbor_head(out: &mut Vec<u8>, major: u8, n: u64) {
|
||||
let m = (major & 0x07) << 5;
|
||||
if n < 24 {
|
||||
out.push(m | n as u8);
|
||||
} else if n < 0x100 {
|
||||
out.push(m | 24);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x10000 {
|
||||
out.push(m | 25);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else if n < 0x100_0000 {
|
||||
out.push(m | 26);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
} else {
|
||||
out.push(m | 27);
|
||||
out.push((n >> 24) as u8);
|
||||
out.push((n >> 16) as u8);
|
||||
out.push((n >> 8) as u8);
|
||||
out.push(n as u8);
|
||||
}
|
||||
}
|
||||
|
||||
fn cbor_text(out: &mut Vec<u8>, s: &str) {
|
||||
cbor_head(out, 3, s.len() as u64);
|
||||
out.extend_from_slice(s.as_bytes());
|
||||
}
|
||||
|
||||
fn cbor_bytes(out: &mut Vec<u8>, b: &[u8]) {
|
||||
cbor_head(out, 2, b.len() as u64);
|
||||
out.extend_from_slice(b);
|
||||
}
|
||||
|
||||
pub fn encode_header(roots: &[Cid]) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
cbor_head(&mut out, 5, 2);
|
||||
cbor_text(&mut out, "version");
|
||||
cbor_head(&mut out, 0, 1);
|
||||
cbor_text(&mut out, "roots");
|
||||
cbor_head(&mut out, 4, roots.len() as u64);
|
||||
for cid in roots {
|
||||
cbor_head(&mut out, 6, 42);
|
||||
cbor_bytes(&mut out, &cid.to_bytes());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn write_varint(out: &mut Vec<u8>, mut n: u64) {
|
||||
loop {
|
||||
let mut byte = (n & 0x7f) as u8;
|
||||
n >>= 7;
|
||||
if n != 0 {
|
||||
byte |= 0x80;
|
||||
}
|
||||
out.push(byte);
|
||||
if n == 0 {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CarWriter {
|
||||
blocks: Vec<(Cid, Vec<u8>)>,
|
||||
}
|
||||
|
||||
impl CarWriter {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn append(&mut self, cid: Cid, data: &[u8]) {
|
||||
if self.blocks.iter().any(|(c, _)| *c == cid) {
|
||||
return;
|
||||
}
|
||||
self.blocks.push((cid, data.to_vec()));
|
||||
}
|
||||
|
||||
pub fn finish(&self, roots: &[Cid]) -> Vec<u8> {
|
||||
let header = encode_header(roots);
|
||||
let mut out = Vec::new();
|
||||
write_varint(&mut out, header.len() as u64);
|
||||
out.extend_from_slice(&header);
|
||||
for (cid, data) in &self.blocks {
|
||||
let cid_bytes = cid.to_bytes();
|
||||
write_varint(&mut out, (cid_bytes.len() + data.len()) as u64);
|
||||
out.extend_from_slice(&cid_bytes);
|
||||
out.extend_from_slice(data);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::test_writer::CarWriter;
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
#[test]
|
||||
fn round_trips_a_single_block() {
|
||||
let data = b"hello world".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[cid])).unwrap();
|
||||
assert_eq!(car.header.version, 1);
|
||||
assert_eq!(car.roots(), &[cid]);
|
||||
assert_eq!(car.blocks.len(), 1);
|
||||
assert_eq!(car.blocks[0].data, data);
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trips_many_blocks_in_order() {
|
||||
let payloads: Vec<Vec<u8>> = (0..6)
|
||||
.map(|i| format!("block-{i}").into_bytes())
|
||||
.collect();
|
||||
let cids: Vec<Cid> = payloads.iter().map(|p| cid_for_cbor(p).unwrap()).collect();
|
||||
let mut w = CarWriter::new();
|
||||
for (cid, data) in cids.iter().zip(&payloads) {
|
||||
w.append(*cid, data);
|
||||
}
|
||||
let car = parse(&w.finish(&[cids[3]])).unwrap();
|
||||
assert_eq!(car.root(), Some(cids[3]));
|
||||
assert_eq!(car.blocks.len(), 6);
|
||||
for (i, b) in car.blocks.iter().enumerate() {
|
||||
assert_eq!(b.cid, cids[i]);
|
||||
assert_eq!(b.data, payloads[i]);
|
||||
}
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_map_finds_blocks_by_cid() {
|
||||
let a = b"record a".to_vec();
|
||||
let b = b"record b".to_vec();
|
||||
let (ca, cb) = (cid_for_cbor(&a).unwrap(), cid_for_cbor(&b).unwrap());
|
||||
let mut w = CarWriter::new();
|
||||
w.append(ca, &a);
|
||||
w.append(cb, &b);
|
||||
let car = parse(&w.finish(&[ca])).unwrap();
|
||||
let map = car.block_map();
|
||||
assert_eq!(map.get(&ca).copied(), Some(a.as_slice()));
|
||||
assert_eq!(map.get(&cb).copied(), Some(b.as_slice()));
|
||||
assert!(!map.contains_key(&cid_for_cbor(b"absent").unwrap()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handles_a_multi_byte_varint_section_length() {
|
||||
// A >127 byte block forces a two-byte varint, which is where an
|
||||
// off-by-one in the length prefix would show up.
|
||||
let data = vec![0x42u8; 500];
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[cid])).unwrap();
|
||||
assert_eq!(car.blocks[0].data.len(), 500);
|
||||
verify_block_cids(&car).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_an_empty_root_list() {
|
||||
let data = b"orphan".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let car = parse(&w.finish(&[])).unwrap();
|
||||
assert!(car.roots().is_empty());
|
||||
assert_eq!(car.blocks.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_input() {
|
||||
let data = b"hello".to_vec();
|
||||
let cid = cid_for_cbor(&data).unwrap();
|
||||
let mut w = CarWriter::new();
|
||||
w.append(cid, &data);
|
||||
let bytes = w.finish(&[cid]);
|
||||
// Cut into the last block's payload.
|
||||
assert!(parse(&bytes[..bytes.len() - 3]).is_err());
|
||||
// Cut inside the header.
|
||||
assert!(parse(&bytes[..3]).is_err());
|
||||
// Nothing at all.
|
||||
assert!(parse(&[]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
//! A minimal, allocation-honest CBOR reader — just enough to decode the
|
||||
//! frames of `com.atproto.sync.subscribeRepos` and the header of a CAR
|
||||
//! file.
|
||||
//!
|
||||
//! ## Why not `ciborium`?
|
||||
//!
|
||||
//! Two reasons, and both come from what the wire format actually is.
|
||||
//!
|
||||
//! 1. **Two values per message.** A subscribeRepos frame is *two*
|
||||
//! DAG-CBOR values written back to back (header, then body) inside
|
||||
//! one WebSocket binary message. A `serde`-shaped reader gives us
|
||||
//! "decode one value from this slice" and no cursor we can resume
|
||||
//! from, so we would have to guess where the header ended.
|
||||
//! 2. **Tag 42.** DAG-CBOR encodes a CID link as `tag(42) +
|
||||
//! bytes(<cid>)`. `ciborium`'s `serde` mapping turns tags into a
|
||||
//! private newtype dance that does not survive a round trip through
|
||||
//! `serde_json::Value`, which is the shape the rest of the AppView
|
||||
//! speaks.
|
||||
//!
|
||||
//! So this module decodes CBOR into its own small [`Cbor`] tree with an
|
||||
//! explicit byte offset, which makes "read the header, then read the
|
||||
//! body from where the header stopped" a two-line function.
|
||||
//!
|
||||
//! ## What it deliberately does not do
|
||||
//!
|
||||
//! No indefinite-length items (DAG-CBOR forbids them; we reject them
|
||||
//! rather than guess), no half floats beyond a plain `f64` widening, no
|
||||
//! canonicalisation checks. It is a *reader* for input we already
|
||||
//! decided to trust at the transport layer, with hard length checks so
|
||||
//! a malformed frame returns `Err` instead of panicking.
|
||||
//!
|
||||
//! Record blocks inside the CAR payload are NOT decoded with this
|
||||
//! module: they are written by `ciborium::into_writer(&serde_json::Value)`
|
||||
//! on the PDS side (see `pds-server/src/routes/repo.rs`), which means
|
||||
//! CIDs inside a record are plain strings, not tag-42 links. Their
|
||||
//! inverse is `ciborium::from_reader::<serde_json::Value, _>`, and
|
||||
//! that is what [`crate::pds_firehose`] uses for them.
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use cid::Cid;
|
||||
|
||||
/// A decoded CBOR value.
|
||||
///
|
||||
/// `Nint` carries the already-negated value (CBOR stores `-1 - n`), so
|
||||
/// callers never have to remember the bias.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Cbor {
|
||||
Uint(u64),
|
||||
Nint(i64),
|
||||
Bytes(Vec<u8>),
|
||||
Text(String),
|
||||
Array(Vec<Cbor>),
|
||||
/// Kept as an ordered key/value list rather than a map: DAG-CBOR
|
||||
/// keys are text and already canonically ordered, and a `Vec` keeps
|
||||
/// the decoder free of hashing while the maps we read have a
|
||||
/// handful of entries at most.
|
||||
Map(Vec<(Cbor, Cbor)>),
|
||||
Tag(u64, Box<Cbor>),
|
||||
Bool(bool),
|
||||
Null,
|
||||
Undefined,
|
||||
Float(f64),
|
||||
}
|
||||
|
||||
impl Cbor {
|
||||
/// Look up a text key in a map. `None` for non-maps and misses.
|
||||
pub fn get(&self, key: &str) -> Option<&Cbor> {
|
||||
match self {
|
||||
Cbor::Map(entries) => entries.iter().find_map(|(k, v)| match k {
|
||||
Cbor::Text(s) if s == key => Some(v),
|
||||
_ => None,
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The value as a signed integer, accepting both CBOR integer
|
||||
/// majors. `None` when the value is not an integer, or when an
|
||||
/// unsigned value exceeds `i64::MAX` (which cannot happen for a
|
||||
/// `seq`, but silently wrapping would be worse than a miss).
|
||||
pub fn as_i64(&self) -> Option<i64> {
|
||||
match self {
|
||||
Cbor::Uint(n) => i64::try_from(*n).ok(),
|
||||
Cbor::Nint(n) => Some(*n),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> Option<&str> {
|
||||
match self {
|
||||
Cbor::Text(s) => Some(s.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bytes(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Cbor::Bytes(b) => Some(b.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_bool(&self) -> Option<bool> {
|
||||
match self {
|
||||
Cbor::Bool(b) => Some(*b),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_array(&self) -> Option<&[Cbor]> {
|
||||
match self {
|
||||
Cbor::Array(items) => Some(items.as_slice()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_null(&self) -> bool {
|
||||
matches!(self, Cbor::Null | Cbor::Undefined)
|
||||
}
|
||||
|
||||
/// Decode a DAG-CBOR CID link: `tag(42) + bytes(...)`.
|
||||
///
|
||||
/// The spec prefixes the CID bytes with a single `0x00` (the
|
||||
/// multibase "identity" marker), because a CID inside a byte string
|
||||
/// has no textual multibase prefix to carry. **This repository's own
|
||||
/// writer omits that byte** — `pds-server/src/car.rs` writes
|
||||
/// `cbor_bytes(&cid.to_bytes())` — so we accept both spellings: a
|
||||
/// leading `0x00` is skipped, anything else is parsed as-is. Being
|
||||
/// lenient here costs nothing (a real CIDv1 never starts with
|
||||
/// `0x00`, and CIDv0 starts with `0x12`) and it means the AppView
|
||||
/// keeps working whether the PDS follows the spec or the house
|
||||
/// convention.
|
||||
pub fn as_cid(&self) -> Option<Cid> {
|
||||
let inner = match self {
|
||||
Cbor::Tag(42, inner) => inner.as_ref(),
|
||||
// A bare byte string where a link is expected: CARs written by
|
||||
// older builds of this PDS tagged the root CID without the
|
||||
// `0x00` identity prefix, and some encoders drop the tag
|
||||
// entirely. Both still have to parse.
|
||||
Cbor::Bytes(_) => self,
|
||||
// A record block written from `serde_json::Value` spells a
|
||||
// CID as a plain string — accept that too, so callers do
|
||||
// not need a second code path for the block contents.
|
||||
Cbor::Text(s) => return s.parse::<Cid>().ok(),
|
||||
_ => return None,
|
||||
};
|
||||
let raw = inner.as_bytes()?;
|
||||
let raw = match raw.first() {
|
||||
Some(0x00) => &raw[1..],
|
||||
_ => raw,
|
||||
};
|
||||
Cid::read_bytes(raw).ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode exactly one CBOR value starting at `pos`.
|
||||
///
|
||||
/// Returns the value and the offset just past it, so a caller can read
|
||||
/// the next value from the same buffer — which is precisely what a
|
||||
/// two-value subscribeRepos frame needs.
|
||||
pub fn decode_at(bytes: &[u8], pos: usize) -> Result<(Cbor, usize)> {
|
||||
let (major, arg, mut p) = read_head(bytes, pos)?;
|
||||
match major {
|
||||
0 => Ok((Cbor::Uint(arg), p)),
|
||||
1 => {
|
||||
// CBOR negative integers store `-1 - n`. Values below
|
||||
// `i64::MIN` cannot occur in anything we consume, and
|
||||
// wrapping them would produce a positive number, so bail.
|
||||
let n = i64::try_from(arg)
|
||||
.map_err(|_| anyhow!("negative integer out of i64 range"))?;
|
||||
Ok((Cbor::Nint(-1 - n), p))
|
||||
}
|
||||
2 => {
|
||||
let end = checked_end(bytes, p, arg, "byte string")?;
|
||||
let v = bytes[p..end].to_vec();
|
||||
Ok((Cbor::Bytes(v), end))
|
||||
}
|
||||
3 => {
|
||||
let end = checked_end(bytes, p, arg, "text string")?;
|
||||
let s = std::str::from_utf8(&bytes[p..end])
|
||||
.map_err(|e| anyhow!("invalid UTF-8 in CBOR text: {e}"))?
|
||||
.to_string();
|
||||
Ok((Cbor::Text(s), end))
|
||||
}
|
||||
4 => {
|
||||
let mut items = Vec::with_capacity(sane_capacity(arg));
|
||||
for _ in 0..arg {
|
||||
let (v, next) = decode_at(bytes, p)?;
|
||||
items.push(v);
|
||||
p = next;
|
||||
}
|
||||
Ok((Cbor::Array(items), p))
|
||||
}
|
||||
5 => {
|
||||
let mut entries = Vec::with_capacity(sane_capacity(arg));
|
||||
for _ in 0..arg {
|
||||
let (k, next) = decode_at(bytes, p)?;
|
||||
let (v, next) = decode_at(bytes, next)?;
|
||||
entries.push((k, v));
|
||||
p = next;
|
||||
}
|
||||
Ok((Cbor::Map(entries), p))
|
||||
}
|
||||
6 => {
|
||||
let (inner, next) = decode_at(bytes, p)?;
|
||||
Ok((Cbor::Tag(arg, Box::new(inner)), next))
|
||||
}
|
||||
7 => match arg {
|
||||
20 => Ok((Cbor::Bool(false), p)),
|
||||
21 => Ok((Cbor::Bool(true), p)),
|
||||
22 => Ok((Cbor::Null, p)),
|
||||
23 => Ok((Cbor::Undefined, p)),
|
||||
// Floats arrive as the raw bit pattern in `arg`; the width
|
||||
// is implied by the additional-information byte, which
|
||||
// `read_head` has already consumed. We only ever see f64 in
|
||||
// practice (DAG-CBOR requires it), so the narrower widths
|
||||
// are decoded for completeness rather than need.
|
||||
_ => Ok((Cbor::Float(f64::from_bits(arg)), p)),
|
||||
},
|
||||
other => bail!("unsupported CBOR major type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a single CBOR value that must span the whole buffer.
|
||||
pub fn decode(bytes: &[u8]) -> Result<Cbor> {
|
||||
let (v, end) = decode_at(bytes, 0)?;
|
||||
if end != bytes.len() {
|
||||
bail!("trailing bytes after CBOR value ({} left)", bytes.len() - end);
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// Read a CBOR head: major type plus its argument.
|
||||
///
|
||||
/// Indefinite-length encodings (`additional information == 31`) are
|
||||
/// rejected: DAG-CBOR forbids them, and accepting them would mean
|
||||
/// implementing break-stop scanning for input that should never carry
|
||||
/// it.
|
||||
fn read_head(bytes: &[u8], pos: usize) -> Result<(u8, u64, usize)> {
|
||||
let first = *bytes
|
||||
.get(pos)
|
||||
.ok_or_else(|| anyhow!("CBOR read past end of input at {pos}"))?;
|
||||
let major = first >> 5;
|
||||
let low = first & 0x1f;
|
||||
let (arg, extra) = match low {
|
||||
0..=23 => (low as u64, 0usize),
|
||||
24 => (read_uint(bytes, pos + 1, 1)?, 1),
|
||||
25 => (read_uint(bytes, pos + 1, 2)?, 2),
|
||||
26 => (read_uint(bytes, pos + 1, 4)?, 4),
|
||||
27 => (read_uint(bytes, pos + 1, 8)?, 8),
|
||||
31 => bail!("indefinite-length CBOR item is not valid DAG-CBOR"),
|
||||
other => bail!("reserved CBOR additional information {other}"),
|
||||
};
|
||||
// For major 7 the "argument" of a float is the raw bit pattern, and
|
||||
// f32/f16 need widening before `f64::from_bits` makes sense.
|
||||
let arg = match (major, low) {
|
||||
(7, 25) => f64::from(half_to_f32(arg as u16)).to_bits(),
|
||||
(7, 26) => f64::from(f32::from_bits(arg as u32)).to_bits(),
|
||||
_ => arg,
|
||||
};
|
||||
Ok((major, arg, pos + 1 + extra))
|
||||
}
|
||||
|
||||
fn read_uint(bytes: &[u8], pos: usize, len: usize) -> Result<u64> {
|
||||
if pos + len > bytes.len() {
|
||||
bail!("truncated CBOR integer of {len} byte(s) at {pos}");
|
||||
}
|
||||
let mut n: u64 = 0;
|
||||
for b in &bytes[pos..pos + len] {
|
||||
n = (n << 8) | u64::from(*b);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// IEEE-754 half → single. Only reached for `f16` inputs, which nothing
|
||||
/// in this protocol emits; kept so a stray value decodes instead of
|
||||
/// erroring out mid-frame.
|
||||
fn half_to_f32(bits: u16) -> f32 {
|
||||
let sign = ((bits >> 15) & 1) as u32;
|
||||
let exp = ((bits >> 10) & 0x1f) as u32;
|
||||
let frac = (bits & 0x3ff) as u32;
|
||||
let out = match exp {
|
||||
0 if frac == 0 => sign << 31,
|
||||
0 => {
|
||||
// Subnormal: renormalise.
|
||||
let mut e = -1i32;
|
||||
let mut f = frac;
|
||||
while f & 0x400 == 0 {
|
||||
f <<= 1;
|
||||
e -= 1;
|
||||
}
|
||||
let exp32 = (127 - 15 + e) as u32;
|
||||
(sign << 31) | (exp32 << 23) | ((f & 0x3ff) << 13)
|
||||
}
|
||||
0x1f => (sign << 31) | (0xff << 23) | (frac << 13),
|
||||
_ => (sign << 31) | ((exp + 127 - 15) << 23) | (frac << 13),
|
||||
};
|
||||
f32::from_bits(out)
|
||||
}
|
||||
|
||||
/// Bounds-check a string/bytes payload before slicing it.
|
||||
fn checked_end(bytes: &[u8], pos: usize, len: u64, what: &str) -> Result<usize> {
|
||||
let len = usize::try_from(len).map_err(|_| anyhow!("{what} length overflows usize"))?;
|
||||
let end = pos
|
||||
.checked_add(len)
|
||||
.ok_or_else(|| anyhow!("{what} length overflows"))?;
|
||||
if end > bytes.len() {
|
||||
bail!("{what} of {len} byte(s) exceeds input at offset {pos}");
|
||||
}
|
||||
Ok(end)
|
||||
}
|
||||
|
||||
/// Cap the pre-allocation a declared array/map length can trigger. A
|
||||
/// corrupt frame claiming `map(2^40)` must not make us reserve 40 GiB
|
||||
/// before the first missing byte errors out; the collection still grows
|
||||
/// naturally for genuinely large inputs.
|
||||
fn sane_capacity(declared: u64) -> usize {
|
||||
usize::try_from(declared.min(1024)).unwrap_or(0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
|
||||
/// Build `{"op": 1, "t": "#commit"}` by hand — the exact bytes the
|
||||
/// PDS writes for a regular frame header.
|
||||
fn commit_header_bytes() -> Vec<u8> {
|
||||
let mut v = vec![0xA2]; // map(2)
|
||||
v.push(0x62); // text(2)
|
||||
v.extend_from_slice(b"op");
|
||||
v.push(0x01); // uint 1
|
||||
v.push(0x61); // text(1)
|
||||
v.extend_from_slice(b"t");
|
||||
v.push(0x67); // text(7)
|
||||
v.extend_from_slice(b"#commit");
|
||||
v
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_a_frame_header() {
|
||||
let v = decode(&commit_header_bytes()).unwrap();
|
||||
assert_eq!(v.get("op").unwrap().as_i64(), Some(1));
|
||||
assert_eq!(v.get("t").unwrap().as_str(), Some("#commit"));
|
||||
assert!(v.get("missing").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_negative_op_of_an_error_header() {
|
||||
// {"op": -1} → map(1), text(2)"op", nint(0) = -1
|
||||
let bytes = vec![0xA1, 0x62, b'o', b'p', 0x20];
|
||||
let v = decode(&bytes).unwrap();
|
||||
assert_eq!(v.get("op").unwrap().as_i64(), Some(-1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_multi_byte_integers() {
|
||||
// uint16 300, uint32 70000, uint64 2^33, nint -300
|
||||
assert_eq!(decode(&[0x19, 0x01, 0x2C]).unwrap().as_i64(), Some(300));
|
||||
assert_eq!(
|
||||
decode(&[0x1A, 0x00, 0x01, 0x11, 0x70]).unwrap().as_i64(),
|
||||
Some(70000)
|
||||
);
|
||||
assert_eq!(
|
||||
decode(&[0x1B, 0, 0, 0, 2, 0, 0, 0, 0]).unwrap().as_i64(),
|
||||
Some(8_589_934_592)
|
||||
);
|
||||
assert_eq!(decode(&[0x39, 0x01, 0x2B]).unwrap().as_i64(), Some(-300));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_at_reads_two_values_back_to_back() {
|
||||
// This is the whole reason the module exists: a frame is header
|
||||
// + body concatenated with no separator.
|
||||
let mut buf = commit_header_bytes();
|
||||
let body_start = buf.len();
|
||||
buf.extend_from_slice(&[0xA1, 0x63, b's', b'e', b'q', 0x18, 0x2A]); // {"seq": 42}
|
||||
let (header, next) = decode_at(&buf, 0).unwrap();
|
||||
assert_eq!(next, body_start);
|
||||
assert_eq!(header.get("t").unwrap().as_str(), Some("#commit"));
|
||||
let (body, end) = decode_at(&buf, next).unwrap();
|
||||
assert_eq!(end, buf.len());
|
||||
assert_eq!(body.get("seq").unwrap().as_i64(), Some(42));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_tag_42_cid_link_both_spellings() {
|
||||
let cid = cid_for_cbor(b"a block").unwrap();
|
||||
let raw = cid.to_bytes();
|
||||
|
||||
// House spelling: tag(42) + bytes(<cid>) with no 0x00 prefix.
|
||||
let mut bare = vec![0xD8, 42]; // tag(42) via 1-byte extension
|
||||
bare.push(0x58); // bytes, 1-byte length
|
||||
bare.push(raw.len() as u8);
|
||||
bare.extend_from_slice(&raw);
|
||||
assert_eq!(decode(&bare).unwrap().as_cid(), Some(cid));
|
||||
|
||||
// Spec spelling: the same, with the identity multibase prefix.
|
||||
let mut prefixed = vec![0xD8, 42, 0x58, (raw.len() + 1) as u8, 0x00];
|
||||
prefixed.extend_from_slice(&raw);
|
||||
assert_eq!(decode(&prefixed).unwrap().as_cid(), Some(cid));
|
||||
|
||||
// And the string spelling records use.
|
||||
let text = {
|
||||
let s = cid.to_string();
|
||||
let mut v = vec![0x78, s.len() as u8];
|
||||
v.extend_from_slice(s.as_bytes());
|
||||
v
|
||||
};
|
||||
assert_eq!(decode(&text).unwrap().as_cid(), Some(cid));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_simple_values_and_containers() {
|
||||
// [true, false, null] → array(3)
|
||||
let v = decode(&[0x83, 0xF5, 0xF4, 0xF6]).unwrap();
|
||||
let items = v.as_array().unwrap();
|
||||
assert_eq!(items[0].as_bool(), Some(true));
|
||||
assert_eq!(items[1].as_bool(), Some(false));
|
||||
assert!(items[2].is_null());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_truncated_and_indefinite_input() {
|
||||
// text(7) claiming 7 bytes but carrying 2.
|
||||
assert!(decode(&[0x67, b'a', b'b']).is_err());
|
||||
// Indefinite-length array.
|
||||
assert!(decode(&[0x9F, 0x01, 0xFF]).is_err());
|
||||
// Trailing garbage after a complete value.
|
||||
assert!(decode(&[0x01, 0x02]).is_err());
|
||||
// Empty input.
|
||||
assert!(decode(&[]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_declared_length_errors_instead_of_allocating() {
|
||||
// map(2^32) with nothing behind it. Must return Err quickly
|
||||
// rather than trying to reserve the declared capacity.
|
||||
let bytes = vec![0xBA, 0xFF, 0xFF, 0xFF, 0xFF];
|
||||
assert!(decode(&bytes).is_err());
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tracing::{debug, info, trace, warn};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
use crate::indexer;
|
||||
|
||||
@@ -31,6 +31,18 @@ pub struct Stats {
|
||||
/// writes this; `/healthz` reads it. Wrapped in `Arc` so the consumer
|
||||
/// can hold its own clone without borrowing from us.
|
||||
pub jetstream_connected: Arc<AtomicBool>,
|
||||
/// Whether the **local PDS** firehose WebSocket is currently up
|
||||
/// ([`crate::pds_firehose`]). Separate from `jetstream_connected`
|
||||
/// because the two streams fail independently and for different
|
||||
/// reasons: a dead Jetstream means no view of the wider network, a
|
||||
/// dead PDS firehose means the AppView has lost the guaranteed
|
||||
/// delivery path for its *own* users' records and is running on the
|
||||
/// best-effort push alone. `/healthz` has to be able to say which.
|
||||
pub pds_connected: AtomicBool,
|
||||
/// Number of `#commit` frames applied from the PDS firehose.
|
||||
pub pds_frames_processed: AtomicU64,
|
||||
/// Highest `seq` applied from the PDS firehose in this process.
|
||||
pub pds_last_seq: AtomicI64,
|
||||
}
|
||||
|
||||
impl Default for Stats {
|
||||
@@ -40,6 +52,9 @@ impl Default for Stats {
|
||||
last_event_time_us: AtomicI64::new(0),
|
||||
last_cursor_persisted_us: AtomicI64::new(0),
|
||||
jetstream_connected: Arc::new(AtomicBool::new(false)),
|
||||
pds_connected: AtomicBool::new(false),
|
||||
pds_frames_processed: AtomicU64::new(0),
|
||||
pds_last_seq: AtomicI64::new(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -50,6 +65,9 @@ impl std::fmt::Debug for Stats {
|
||||
.field("events_processed", &self.events_processed())
|
||||
.field("last_event_time_us", &self.last_event_time_us.load(Ordering::Relaxed))
|
||||
.field("jetstream_connected", &self.jetstream_connected())
|
||||
.field("pds_connected", &self.pds_connected())
|
||||
.field("pds_frames_processed", &self.pds_frames_processed())
|
||||
.field("pds_last_seq", &self.pds_last_seq())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -85,6 +103,23 @@ impl Stats {
|
||||
pub fn jetstream_connected(&self) -> bool {
|
||||
self.jetstream_connected.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Is the local PDS firehose connected right now?
|
||||
pub fn pds_connected(&self) -> bool {
|
||||
self.pds_connected.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn pds_frames_processed(&self) -> u64 {
|
||||
self.pds_frames_processed.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Highest PDS-firehose `seq` this process has applied. 0 before the
|
||||
/// first frame — note this is the *in-process* high-water mark, not
|
||||
/// the persisted cursor, which lives in `pds_firehose_cursor` and
|
||||
/// survives restarts.
|
||||
pub fn pds_last_seq(&self) -> i64 {
|
||||
self.pds_last_seq.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The thing the Jetstream consumer calls once per event.
|
||||
|
||||
@@ -4,9 +4,12 @@
|
||||
//! against a stub resolver without booting the binary.
|
||||
|
||||
pub mod auth;
|
||||
pub mod car;
|
||||
pub mod cbor;
|
||||
pub mod firehose;
|
||||
pub mod handle_sync;
|
||||
pub mod indexer;
|
||||
pub mod ingest;
|
||||
pub mod pds_firehose;
|
||||
pub mod routes;
|
||||
pub mod state;
|
||||
|
||||
@@ -8,10 +8,13 @@ use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
mod auth;
|
||||
mod car;
|
||||
mod cbor;
|
||||
mod firehose;
|
||||
mod handle_sync;
|
||||
mod indexer;
|
||||
mod ingest;
|
||||
mod pds_firehose;
|
||||
mod routes;
|
||||
mod state;
|
||||
|
||||
@@ -82,6 +85,30 @@ async fn main() -> Result<()> {
|
||||
});
|
||||
}
|
||||
|
||||
// Local PDS firehose. The push path (`/internal/ingest-commit`) is
|
||||
// the fast way a local commit reaches the index; this is the
|
||||
// guaranteed one — it carries a durable cursor, so anything the
|
||||
// push dropped while the AppView was down is replayed on connect.
|
||||
// See the module docs in `pds_firehose.rs` for why both exist.
|
||||
if cfg.pds_firehose_enabled {
|
||||
let start_seq = pds_firehose::cursor_get(&db).await.unwrap_or(0);
|
||||
let base = cfg.pds_base_url();
|
||||
info!(
|
||||
url = %pds_firehose::subscribe_url(&base, (start_seq > 0).then_some(start_seq)),
|
||||
cursor = start_seq,
|
||||
"starting the local PDS firehose consumer"
|
||||
);
|
||||
let consumer = pds_firehose::PdsFirehose::new(db.clone(), base, stats.clone())
|
||||
.with_max_backoff_secs(30);
|
||||
tokio::spawn(consumer.run_forever());
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"PDS_FIREHOSE_ENABLED=false — local commits reach the index only through \
|
||||
the best-effort `/internal/ingest-commit` push; a push lost to a restart \
|
||||
or a network error will NOT be recovered"
|
||||
);
|
||||
}
|
||||
|
||||
let state = AppState::new(cfg.clone(), db.clone(), stats.clone());
|
||||
|
||||
// Announce every relaxed security switch before we serve anything.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1538,6 +1538,16 @@ async fn actor_list(
|
||||
|
||||
// -- healthz ----------------------------------------------------------------
|
||||
|
||||
/// Liveness probe.
|
||||
///
|
||||
/// Both ingest streams report separately. `jetstream_connected` is the
|
||||
/// public network's view; `pds_firehose_connected` is the local PDS's
|
||||
/// guaranteed delivery path for our own users' records. A deployment
|
||||
/// can be perfectly healthy for reads with the first one down, but a
|
||||
/// `pds_firehose_enabled: true, pds_firehose_connected: false` pair
|
||||
/// means local commits are riding on the best-effort push alone — which
|
||||
/// is exactly the state an operator wants to see in a probe rather than
|
||||
/// discover from a missing post.
|
||||
async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let stats = &state.stats;
|
||||
Json(json!({
|
||||
@@ -1545,6 +1555,10 @@ async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
|
||||
"lag_ms": stats.lag_ms(),
|
||||
"events_processed": stats.events_processed(),
|
||||
"jetstream_connected": stats.jetstream_connected(),
|
||||
"pds_firehose_enabled": state.cfg.pds_firehose_enabled,
|
||||
"pds_firehose_connected": stats.pds_connected(),
|
||||
"pds_firehose_frames": stats.pds_frames_processed(),
|
||||
"pds_firehose_seq": stats.pds_last_seq(),
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user