tomdebone/maarcadetweet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pds): CAR-Header-Roots mit Multibase-Identity-Prefix schreiben

Browse Source 
Ein DAG-CBOR-Link ist tag(42) um einen Bytestring aus `0x00 || <CID>`. Der
CAR-Header taggte bisher die nackte CID ohne das 0x00 — keine
spec-konforme CAR-Bibliothek kann dem folgen: sie liest das erste Byte als
CID-Version und gibt auf. Betroffen war jede Antwort von getRepo,
getBlocks und getRecord.

Der Header ist nicht content-adressiert — nichts hasht ihn, keine CID hängt
an seinen Bytes. Die Korrektur ändert also ausschließlich, was über die
Leitung geht, und keinen einzigen Identifier. Deshalb ist sie hier gemacht
und nicht auf eine große Migration vertagt.

decode_header akzeptiert weiterhin beide Schreibweisen, damit ein
gespeicherter Repo-Export aus einem älteren Build lesbar bleibt. Das ist
eindeutig und kein Raten: eine echte CID beginnt nie mit 0x00, da steht das
Versions-Varint und Version 0 gibt es nicht.

Nebenbei: sync_list_repos_keyset_pagination lief von ganz vorn durch die
repos-Tabelle (inzwischen 4900 Zeilen) und riss bei zwei Zeilen pro Seite
den eigenen Iterationsdeckel — rot wegen Tabellengröße, nicht wegen
Paginierung. Der Test prüft jetzt die Invarianten, um die es geht:
Erreichbarkeit jedes DIDs über einen unmittelbar davor gesetzten Cursor,
streng aufsteigende Reihenfolge, keine Dubletten, und der zurückgegebene
Cursor ist der letzte DID der Seite.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_013HC9HLrUU1LNwkzp8nkDLX
This commit is contained in:
tomdebone
2026-09-10 07:08:23 +02:00
co-authored by Claude Opus 5
parent 0646fbeebe
commit d6947c2576
2 changed files with 203 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/pds-server/tests/pds_integration.rs
+72 -18
View File
@@ -435,7 +435,17 @@ fn parse_car(bytes: &[u8]) -> ParsedCar {
assert_eq!(maj, 2, "root CID must be a byte string");
p += c;
let cid_bytes = &bytes[p..p + ln as usize];
let cid_hex: String = cid_bytes
// A DAG-CBOR link wraps `0x00 || <binary CID>`. The 0x00 is
// the multibase identity prefix, not part of the CID, so it
// comes off before the hex comparison against a real CID's
// bytes. Asserted rather than skipped: this helper is the
// only place the header's wire shape is checked.
assert_eq!(
cid_bytes.first(),
Some(&0x00),
"root link must carry the multibase identity prefix"
);
let cid_hex: String = cid_bytes[1..]
.iter()
.map(|b| format!("{:02x}", b))
.collect();
@@ -1056,42 +1066,86 @@ async fn sync_list_repos_keyset_pagination() {
assert!(resp["uri"].is_string(), "createRecord: {:?}", resp);
created_dids.push(did);
}
let min_did = created_dids.iter().min().unwrap().clone();
let start_cursor = did_cursor_lt(&min_did);
// Two separate properties, deliberately not tested by one long walk
// from the top of the table: `repos` grows without bound on a
// long-lived instance (a few thousand rows here), the seeded DIDs are
// random `did:plc:bafy…` hashes scattered across that range, and a
// full scan at two rows per page ran into its own iteration cap —
// failing for table size rather than for anything about pagination.
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut cursor: Option<String> = Some(start_cursor);
let mut pages = 0;
loop {
pages += 1;
assert!(pages < 2000, "pagination did not terminate");
// 1. Every seeded DID is reachable: anchor the cursor immediately
// before it and it must be on the first page.
for did in &created_dids {
let url = format!(
"{}/xrpc/com.atproto.sync.listRepos?limit=2&cursor={}",
PDS_URL,
urlencode(cursor.as_deref().unwrap_or(""))
urlencode(&did_cursor_just_before(did))
);
let resp = client().await.get(&url).send().await.unwrap();
assert_eq!(resp.status().as_u16(), 200);
let body: Value = resp.json().await.unwrap();
let repos = body["repos"].as_array().expect("repos array");
assert!(
repos.iter().any(|r| r["did"].as_str() == Some(did.as_str())),
"DID not on the page starting immediately before it: {did}"
);
}
// 2. The keyset itself: walking forward from the lowest seeded DID
// yields strictly increasing DIDs, never a duplicate, and the
// cursor the server hands back is always the last DID of the page.
// A bounded number of pages is enough — these are properties of
// every step, not of the whole table.
let min_did = created_dids.iter().min().unwrap().clone();
let mut cursor = did_cursor_just_before(&min_did);
let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
let mut last: Option<String> = None;
for _ in 0..25 {
let url = format!(
"{}/xrpc/com.atproto.sync.listRepos?limit=2&cursor={}",
PDS_URL,
urlencode(&cursor)
);
let resp = client().await.get(&url).send().await.unwrap();
assert_eq!(resp.status().as_u16(), 200);
let body: Value = resp.json().await.unwrap();
let repos = body["repos"].as_array().expect("repos array");
if repos.is_empty() {
break;
}
for r in repos {
let did = r["did"].as_str().unwrap().to_string();
assert!(
seen.insert(did.clone()),
"duplicate DID across pages: {did}"
);
}
if created_dids.iter().all(|d| seen.contains(d)) {
break;
if let Some(prev) = &last {
assert!(
&did > prev,
"listRepos must be strictly ascending by DID: {prev} then {did}"
);
}
last = Some(did);
}
match body["cursor"].as_str() {
Some(c) => cursor = Some(c.to_string()),
None => panic!(
"pagination exhausted before all created DIDs were seen; missing {:?}",
created_dids.iter().filter(|d| !seen.contains(*d)).collect::<Vec<_>>()
),
Some(c) => {
assert_eq!(
Some(c),
last.as_deref(),
"cursor must be the last DID of the page just served"
);
cursor = c.to_string();
}
// Fewer rows than the limit: the end of the table, and the
// server correctly stops handing out a cursor.
None => break,
}
}
assert!(
seen.len() >= 2,
"expected the walk to cover at least two pages, saw {}",
seen.len()
);
}
fn urlencode(s: &str) -> String {