maarcadetweet: initial commit
AT Protocol PDS + AppView + Tauri Desktop Client, 160-char post limit. - PDS (Rust + axum + sqlx) - Auth: createAccount, createSession, refreshSession - Records: createRecord, deleteRecord (race-safe via SELECT FOR UPDATE) - Feed: feed.like.create, feed.repost.create - Sync: getRepo, getBlocks, getLatestCommit, getRecord (with MST proof), listRepos - Identity: resolveHandle - MST: spec-conformant (at-mst crate, 27 tests) - Repo: signed commits, TID counter (monotonic, 4096 wrap safe) - AppView (Rust + axum + sqlx) - Jetstream consumer (WebSocket, exponential backoff, 38k+ events indexed) - REST API: timeline/home (graph-aware), profile, search, post (with thread hydration) - Handle-sync worker (did:plc + did:web) - JSONB embed storage + thread columns (migration 0003) - Like/repost counter cache (migration 0004) - Tauri 2 + Svelte 5 Desktop Client - System tray (Show/Compose/Quit menu) - OS notifications (tauri-plugin-notification) - Auto-update (tauri-plugin-updater, placeholder endpoint) - Window-state (tauri-plugin-window-state) - 160-char compose with live counter - Image/Link embed rendering - LocalStorage-persisted like state - Timeline with poll (prepend new posts) - Custom TitleBar (transparent, no decorations) - Orange/IBM Plex Mono maarcade design Tests: 231 Rust + 9 vitest = 240 passed.
This commit is contained in:
tomdebone
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "at-repo"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
authors.workspace = true
|
||||
description = "Repository, commits, blocks for AT Protocol"
|
||||
|
||||
[lints.rust]
|
||||
unsafe_code = "forbid"
|
||||
|
||||
[dependencies]
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
ciborium = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
sqlx = { workspace = true }
|
||||
at-crypto = { workspace = true }
|
||||
at-lexicon = { workspace = true }
|
||||
at-mst = { workspace = true }
|
||||
at-shared = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
bytes = { workspace = true }
|
||||
k256 = { workspace = true }
|
||||
cid = { workspace = true }
|
||||
@@ -0,0 +1,53 @@
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use cid::Cid;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[async_trait]
|
||||
pub trait Blockstore: Send + Sync {
|
||||
async fn put(&self, cid: &Cid, block: Bytes) -> Result<()>;
|
||||
async fn get(&self, cid: &Cid) -> Result<Option<Bytes>>;
|
||||
async fn has(&self, cid: &Cid) -> Result<bool>;
|
||||
async fn list(&self) -> Result<Vec<(Cid, Bytes)>>;
|
||||
}
|
||||
|
||||
pub struct MemoryBlockstore {
|
||||
inner: parking_lot::Mutex<HashMap<Cid, Bytes>>,
|
||||
}
|
||||
|
||||
impl Default for MemoryBlockstore {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl MemoryBlockstore {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
inner: parking_lot::Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Blockstore for MemoryBlockstore {
|
||||
async fn put(&self, cid: &Cid, block: Bytes) -> Result<()> {
|
||||
self.inner.lock().insert(*cid, block);
|
||||
Ok(())
|
||||
}
|
||||
async fn get(&self, cid: &Cid) -> Result<Option<Bytes>> {
|
||||
Ok(self.inner.lock().get(cid).cloned())
|
||||
}
|
||||
async fn has(&self, cid: &Cid) -> Result<bool> {
|
||||
Ok(self.inner.lock().contains_key(cid))
|
||||
}
|
||||
async fn list(&self) -> Result<Vec<(Cid, Bytes)>> {
|
||||
Ok(self
|
||||
.inner
|
||||
.lock()
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, v.clone()))
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
use at_crypto::signing::verify_dag_cbor;
|
||||
use cid::Cid;
|
||||
use k256::ecdsa::VerifyingKey;
|
||||
use serde_json::Value;
|
||||
|
||||
/// A signed repository commit.
|
||||
///
|
||||
/// `Commit` carries the canonical DAG-CBOR serialization of a signed commit
|
||||
/// block (including the `sig` field) together with the parsed fields. The
|
||||
/// `cid` is the SHA-256 DAG-CBOR content-address of `signed_bytes`.
|
||||
///
|
||||
/// The `data` field is `Option<Cid>` to support commits on empty repositories
|
||||
/// — an empty repo has no MST root to point at, so the JSON payload's `data`
|
||||
/// is serialized as `null`.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Commit {
|
||||
pub cid: Cid,
|
||||
pub signed_bytes: Vec<u8>,
|
||||
pub did: String,
|
||||
pub rev: String,
|
||||
pub prev: Option<Cid>,
|
||||
pub data: Option<Cid>,
|
||||
}
|
||||
|
||||
impl Commit {
|
||||
/// Verify the commit's signature.
|
||||
///
|
||||
/// Delegates the actual cryptographic check to [`at_crypto::signing::verify_dag_cbor`],
|
||||
/// which uses the `pubkey` field embedded inside the signed commit. The
|
||||
/// `signing_pubkey` argument is the caller-trusted key — we additionally
|
||||
/// require the embedded pubkey to match it, so a malicious swap of the
|
||||
/// `pubkey` field (followed by a forged signature under the swapped key)
|
||||
/// is rejected.
|
||||
pub fn verify(&self, signing_pubkey: &VerifyingKey) -> Result<()> {
|
||||
let embedded = verify_dag_cbor(&self.signed_bytes)?;
|
||||
if &embedded != signing_pubkey {
|
||||
return Err(anyhow!(
|
||||
"commit embedded pubkey does not match expected signing pubkey"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Parse a signed commit block out of raw DAG-CBOR bytes.
|
||||
///
|
||||
/// This is used by `Repo::load` to reconstruct the head commit when
|
||||
/// re-hydrating a repository from a blockstore.
|
||||
pub fn from_signed_bytes(signed_bytes: Vec<u8>) -> Result<Self> {
|
||||
let cid = cid_for_cbor(&signed_bytes)?;
|
||||
let value: Value = ciborium::from_reader(&signed_bytes[..])
|
||||
.map_err(|e| anyhow!("invalid commit CBOR: {e}"))?;
|
||||
let obj = value
|
||||
.as_object()
|
||||
.ok_or_else(|| anyhow!("commit CBOR is not an object"))?;
|
||||
|
||||
let did = obj
|
||||
.get("did")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("commit missing `did`"))?
|
||||
.to_string();
|
||||
let rev = obj
|
||||
.get("rev")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| anyhow!("commit missing `rev`"))?
|
||||
.to_string();
|
||||
let prev = parse_optional_cid(obj.get("prev"), "prev")?;
|
||||
let data = parse_optional_cid(obj.get("data"), "data")?;
|
||||
|
||||
Ok(Self {
|
||||
cid,
|
||||
signed_bytes,
|
||||
did,
|
||||
rev,
|
||||
prev,
|
||||
data,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_optional_cid(value: Option<&Value>, field: &str) -> Result<Option<Cid>> {
|
||||
match value {
|
||||
None | Some(Value::Null) => Ok(None),
|
||||
Some(Value::String(s)) => s
|
||||
.parse::<Cid>()
|
||||
.map(Some)
|
||||
.map_err(|e| anyhow!("commit `{field}` is not a valid CID: {e}")),
|
||||
Some(_) => Err(anyhow!("commit `{field}` must be null or string")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::did_key::pubkey_to_multibase;
|
||||
use k256::ecdsa::SigningKey;
|
||||
use k256::SecretKey;
|
||||
use k256::PublicKey;
|
||||
|
||||
fn make_test_commit(
|
||||
sk: &SigningKey,
|
||||
did: &str,
|
||||
rev: &str,
|
||||
prev: Option<&str>,
|
||||
data: Option<&str>,
|
||||
) -> Commit {
|
||||
let pk: PublicKey = sk.verifying_key().into();
|
||||
let mb = pubkey_to_multibase(&pk).unwrap();
|
||||
let mut payload = serde_json::json!({
|
||||
"did": did,
|
||||
"version": 3,
|
||||
"prev": prev.map(|s| serde_json::Value::String(s.to_string()))
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
"data": data.map(|s| serde_json::Value::String(s.to_string()))
|
||||
.unwrap_or(serde_json::Value::Null),
|
||||
"rev": rev,
|
||||
"pubkey": mb,
|
||||
});
|
||||
let _ = payload.as_object_mut().unwrap().remove("sig");
|
||||
let signed = at_crypto::signing::sign_dag_cbor(sk, &payload).unwrap();
|
||||
Commit::from_signed_bytes(signed.signed_bytes).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn self_signed_commit_verifies() {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[7u8; 32]).unwrap());
|
||||
let commit = make_test_commit(&sk, "did:plc:abc", "0", None, None);
|
||||
assert!(commit.verify(&sk.verifying_key()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrong_key_fails_verify() {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[7u8; 32]).unwrap());
|
||||
let sk2 = SigningKey::from(SecretKey::from_slice(&[9u8; 32]).unwrap());
|
||||
let commit = make_test_commit(&sk, "did:plc:abc", "0", None, None);
|
||||
assert!(commit.verify(&sk2.verifying_key()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commit_with_prev_and_data_roundtrip() {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[11u8; 32]).unwrap());
|
||||
let prev_cid: Cid = "bafyreig7qfkqdk5v3jy3z6xgc4n3yh6ycjxqrvt5pqjpwxgvvcxyzw7tqy"
|
||||
.parse()
|
||||
.unwrap();
|
||||
let data_cid: Cid = "bafyreihzfgvyuwdq5i3qaqj2vnlv4bgw2xhcsoa2uh2pqkpcw55nuefzzi"
|
||||
.parse()
|
||||
.unwrap();
|
||||
let commit = make_test_commit(
|
||||
&sk,
|
||||
"did:plc:abc",
|
||||
"abc123",
|
||||
Some(&prev_cid.to_string()),
|
||||
Some(&data_cid.to_string()),
|
||||
);
|
||||
assert_eq!(commit.did, "did:plc:abc");
|
||||
assert_eq!(commit.rev, "abc123");
|
||||
assert_eq!(commit.prev, Some(prev_cid));
|
||||
assert_eq!(commit.data, Some(data_cid));
|
||||
assert!(commit.verify(&sk.verifying_key()).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tampered_signed_bytes_fail_verify() {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[15u8; 32]).unwrap());
|
||||
let commit = make_test_commit(&sk, "did:plc:abc", "0", None, None);
|
||||
// Flip a bit in `sig` (last bytes of the CBOR object). The simplest
|
||||
// way to land inside `sig` (a hex string of ~128 chars) is to flip a
|
||||
// byte near the tail of the payload.
|
||||
let mut tampered_bytes = commit.signed_bytes.clone();
|
||||
let len = tampered_bytes.len();
|
||||
tampered_bytes[len - 4] ^= 0x01;
|
||||
tampered_bytes[len - 3] ^= 0x01;
|
||||
let res = match Commit::from_signed_bytes(tampered_bytes) {
|
||||
Ok(c) => c.verify(&sk.verifying_key()),
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
assert!(
|
||||
res.is_err(),
|
||||
"tampered commit must not verify; got Ok"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tampered_field_fails_verify() {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[16u8; 32]).unwrap());
|
||||
let commit = make_test_commit(&sk, "did:plc:abc", "0", None, None);
|
||||
// Parse the signed CBOR object, swap the `did`, and re-encode. The
|
||||
// resulting CID is different, but the signature over the unsigned
|
||||
// payload is now stale — verification must fail.
|
||||
let mut value: Value = ciborium::from_reader(commit.signed_bytes.as_slice()).unwrap();
|
||||
{
|
||||
let obj = value.as_object_mut().unwrap();
|
||||
obj.insert("did".into(), Value::String("did:plc:imposter".into()));
|
||||
}
|
||||
let mut new_bytes = Vec::new();
|
||||
ciborium::into_writer(&value, &mut new_bytes).unwrap();
|
||||
let res = match Commit::from_signed_bytes(new_bytes) {
|
||||
Ok(c) => c.verify(&sk.verifying_key()),
|
||||
Err(e) => Err(e),
|
||||
};
|
||||
assert!(res.is_err(), "did swap must invalidate signature");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
pub mod blockstore;
|
||||
pub mod commit;
|
||||
pub mod repo;
|
||||
pub mod rev;
|
||||
|
||||
pub use blockstore::{Blockstore, MemoryBlockstore};
|
||||
pub use commit::Commit;
|
||||
pub use repo::Repo;
|
||||
pub use rev::Tid;
|
||||
@@ -0,0 +1,527 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use at_crypto::did_key::pubkey_to_multibase;
|
||||
use at_crypto::signing::sign_dag_cbor;
|
||||
use at_mst::util::encode_key;
|
||||
use at_mst::Mst;
|
||||
use bytes::Bytes;
|
||||
use cid::Cid;
|
||||
use k256::ecdsa::SigningKey;
|
||||
use k256::PublicKey;
|
||||
use serde::Deserialize;
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::blockstore::Blockstore;
|
||||
use crate::commit::Commit;
|
||||
use crate::rev::Tid;
|
||||
|
||||
/// A single repository: a content-addressed Merkle Search Tree backed by a
|
||||
/// [`Blockstore`], with a secp256k1 signing key used to authorize commits.
|
||||
///
|
||||
/// `Repo` is the mutable in-memory representation. The immutable history is
|
||||
/// encoded in the linked list of [`Commit`] blocks, and the current state is
|
||||
/// restored from that chain via [`Repo::load`].
|
||||
///
|
||||
/// Operations on the repo (`put_record`, `delete_record`, `commit`) all
|
||||
/// persist their newly produced blocks through `self.blockstore`. A
|
||||
/// production implementation would back the blockstore with durable storage
|
||||
/// (e.g. a Postgres-backed blockstore); tests use [`crate::MemoryBlockstore`].
|
||||
pub struct Repo<B: Blockstore> {
|
||||
pub did: String,
|
||||
pub signing_key: SigningKey,
|
||||
pub mst: Mst,
|
||||
pub blockstore: Arc<B>,
|
||||
pub prev_commit_cid: Option<Cid>,
|
||||
pub rev: String,
|
||||
/// CIDs of value blocks we've written, tracked so [`Repo::serialize_repo`]
|
||||
/// can include them in the output.
|
||||
value_cids: HashSet<Cid>,
|
||||
}
|
||||
|
||||
impl<B: Blockstore> Repo<B> {
|
||||
/// Construct a new empty repo for `did`, signed by `signing_key` and
|
||||
/// stored in `blockstore`. The repo has no MST and no prior commit.
|
||||
pub fn new(did: String, signing_key: SigningKey, blockstore: Arc<B>) -> Self {
|
||||
Self {
|
||||
did,
|
||||
signing_key,
|
||||
mst: Mst::new(),
|
||||
blockstore,
|
||||
prev_commit_cid: None,
|
||||
rev: Tid::new().as_str().to_string(),
|
||||
value_cids: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Add (or update) a record at `at://{did}/{collection}/{rkey}` pointing
|
||||
/// to `value_cid`.
|
||||
///
|
||||
/// The caller is responsible for storing the value's CBOR block in
|
||||
/// `self.blockstore` (typically before this call, via the route handler).
|
||||
/// This method only persists the new MST node blocks produced by the
|
||||
/// underlying [`Mst::put`].
|
||||
pub async fn put_record(
|
||||
&mut self,
|
||||
collection: &str,
|
||||
rkey: &str,
|
||||
value_cid: Cid,
|
||||
) -> Result<(String, Cid)> {
|
||||
let raw_key = format!("{collection}/{rkey}");
|
||||
// The MST encodes the key internally via encode_key; the redundant
|
||||
// call here is retained as documentation of the wire-format contract.
|
||||
let _ = encode_key(&raw_key);
|
||||
let new_mst = self.mst.clone().put(raw_key, value_cid, None)?;
|
||||
self.mst = new_mst;
|
||||
self.value_cids.insert(value_cid);
|
||||
self.persist_mst_blocks().await?;
|
||||
let uri = format!("at://{}/{}/{}", self.did, collection, rkey);
|
||||
Ok((uri, value_cid))
|
||||
}
|
||||
|
||||
/// Remove the record at `at://{did}/{collection}/{rkey}` if present.
|
||||
/// Persists the new MST node blocks produced by the underlying
|
||||
/// [`Mst::delete`].
|
||||
pub async fn delete_record(&mut self, collection: &str, rkey: &str) -> Result<()> {
|
||||
let raw_key = format!("{collection}/{rkey}");
|
||||
let new_mst = self.mst.clone().delete(raw_key)?;
|
||||
self.mst = new_mst;
|
||||
self.persist_mst_blocks().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Lookup the value CID for a record. Returns `Ok(None)` if absent.
|
||||
pub async fn get_record(&self, collection: &str, rkey: &str) -> Result<Option<Cid>> {
|
||||
let raw_key = format!("{collection}/{rkey}");
|
||||
self.mst.get(&raw_key)
|
||||
}
|
||||
|
||||
/// Build a signed commit over the current MST root, persist it in
|
||||
/// `self.blockstore`, and update `prev_commit_cid` + `rev` so subsequent
|
||||
/// commits link back to this one.
|
||||
///
|
||||
/// An empty repo (no MST entries) is allowed; the resulting commit's
|
||||
/// `data` field is `null`.
|
||||
pub async fn commit(&mut self) -> Result<Commit> {
|
||||
let data_cid = self.mst.root_cid();
|
||||
|
||||
let pk: PublicKey = self.signing_key.verifying_key().into();
|
||||
let pubkey_mb = pubkey_to_multibase(&pk)?;
|
||||
let prev_value = self
|
||||
.prev_commit_cid
|
||||
.map(|c| Value::String(c.to_string()))
|
||||
.unwrap_or(Value::Null);
|
||||
let data_value = data_cid
|
||||
.map(|c| Value::String(c.to_string()))
|
||||
.unwrap_or(Value::Null);
|
||||
let payload = json!({
|
||||
"did": self.did,
|
||||
"version": 3,
|
||||
"prev": prev_value,
|
||||
"data": data_value,
|
||||
"rev": self.rev,
|
||||
"pubkey": pubkey_mb,
|
||||
});
|
||||
|
||||
let signed = sign_dag_cbor(&self.signing_key, &payload)?;
|
||||
let signed_cid: Cid = signed
|
||||
.cid
|
||||
.parse()
|
||||
.map_err(|e| anyhow!("signed commit CID parse: {e}"))?;
|
||||
|
||||
self.blockstore
|
||||
.put(&signed_cid, Bytes::from(signed.signed_bytes.clone()))
|
||||
.await?;
|
||||
|
||||
let commit = Commit {
|
||||
cid: signed_cid,
|
||||
signed_bytes: signed.signed_bytes,
|
||||
did: self.did.clone(),
|
||||
rev: self.rev.clone(),
|
||||
prev: self.prev_commit_cid,
|
||||
data: data_cid,
|
||||
};
|
||||
|
||||
self.prev_commit_cid = Some(commit.cid);
|
||||
self.rev = Tid::new().as_str().to_string();
|
||||
|
||||
Ok(commit)
|
||||
}
|
||||
|
||||
/// Serialize the repo to a CAR-like pair `(header_bytes, blocks_map)`.
|
||||
///
|
||||
/// `header_bytes` is the latest signed commit block, or empty if no
|
||||
/// commit has been produced yet. `blocks_map` contains every MST block,
|
||||
/// every tracked value block, and the commit block.
|
||||
pub async fn serialize_repo(&self) -> Result<(Vec<u8>, HashMap<Cid, Vec<u8>>)> {
|
||||
let (_root_bytes, mut all_blocks) = self.mst.serialize()?;
|
||||
|
||||
let header = if let Some(commit_cid) = self.prev_commit_cid {
|
||||
match self.blockstore.get(&commit_cid).await? {
|
||||
Some(bytes) => {
|
||||
let v = bytes.to_vec();
|
||||
all_blocks.insert(commit_cid, v.clone());
|
||||
v
|
||||
}
|
||||
None => Vec::new(),
|
||||
}
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
for cid in &self.value_cids {
|
||||
if let Some(bytes) = self.blockstore.get(cid).await? {
|
||||
all_blocks.insert(*cid, bytes.to_vec());
|
||||
}
|
||||
}
|
||||
|
||||
Ok((header, all_blocks))
|
||||
}
|
||||
|
||||
/// Reconstruct a `Repo` from a previously-stored head commit.
|
||||
///
|
||||
/// `head_commit_cid` must resolve via `blockstore.get` to a signed commit
|
||||
/// block produced by `signing_key`. The blockstore must contain every MST
|
||||
/// node block reachable from `commit.data`, plus the commit block itself.
|
||||
pub async fn load(
|
||||
did: String,
|
||||
signing_key: SigningKey,
|
||||
blockstore: Arc<B>,
|
||||
head_commit_cid: Cid,
|
||||
) -> Result<Self> {
|
||||
let commit_bytes = blockstore
|
||||
.get(&head_commit_cid)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow!("head commit block not found in blockstore"))?;
|
||||
let commit = Commit::from_signed_bytes(commit_bytes.to_vec())?;
|
||||
|
||||
let mut mst = Mst::new();
|
||||
let mut value_cids: HashSet<Cid> = HashSet::new();
|
||||
if let Some(root) = commit.data {
|
||||
let blocks = collect_mst_blocks(blockstore.as_ref(), root).await?;
|
||||
mst = Mst::from_blocks(blocks, root);
|
||||
// Walk the loaded MST to populate `value_cids` so that
|
||||
// `serialize_repo` includes every value block produced by prior
|
||||
// writes. (We only know about values added since the last
|
||||
// in-process load otherwise.)
|
||||
if !mst.is_empty() {
|
||||
let empty = Mst::new();
|
||||
for entry in mst.diff(&empty)? {
|
||||
value_cids.insert(entry.cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
did,
|
||||
signing_key,
|
||||
mst,
|
||||
blockstore,
|
||||
prev_commit_cid: Some(head_commit_cid),
|
||||
rev: Tid::new().as_str().to_string(),
|
||||
value_cids,
|
||||
})
|
||||
}
|
||||
|
||||
async fn persist_mst_blocks(&self) -> Result<()> {
|
||||
for (cid, bytes) in self.mst.blocks() {
|
||||
self.blockstore
|
||||
.put(cid, Bytes::from(bytes.clone()))
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// -- internal helpers --------------------------------------------------------
|
||||
|
||||
/// Local mirror of `at_mst`'s on-the-wire node format. We need this to walk
|
||||
/// MST blocks from a `Blockstore` whose API is `get(cid) -> Option<Bytes>`
|
||||
/// rather than an iterator: `at_mst` exposes `Mst::from_blocks` but the tree
|
||||
/// walk is internal, so we parse the node-shape here and skip past value CIDs
|
||||
/// (which are not MST nodes).
|
||||
#[derive(Deserialize)]
|
||||
struct WireNode {
|
||||
#[serde(rename = "l")]
|
||||
left: Option<Cid>,
|
||||
#[serde(rename = "e")]
|
||||
entries: Vec<WireEntry>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct WireEntry {
|
||||
#[serde(rename = "v")]
|
||||
#[allow(dead_code)]
|
||||
value: Cid,
|
||||
#[serde(rename = "t")]
|
||||
tree: Option<Cid>,
|
||||
}
|
||||
|
||||
async fn collect_mst_blocks<B: Blockstore + ?Sized>(
|
||||
blockstore: &B,
|
||||
root: Cid,
|
||||
) -> Result<HashMap<Cid, Vec<u8>>> {
|
||||
let mut out: HashMap<Cid, Vec<u8>> = HashMap::new();
|
||||
let mut stack: Vec<Cid> = vec![root];
|
||||
while let Some(cid) = stack.pop() {
|
||||
if out.contains_key(&cid) {
|
||||
continue;
|
||||
}
|
||||
let bytes = blockstore
|
||||
.get(&cid)
|
||||
.await?
|
||||
.ok_or_else(|| anyhow!("missing block for cid {cid}"))?;
|
||||
// A block is only included if it parses as an MST node; value blocks
|
||||
// (and any other CBOR blocks) are skipped past.
|
||||
match ciborium::from_reader::<WireNode, _>(bytes.as_ref()) {
|
||||
Ok(node) => {
|
||||
if let Some(l) = node.left {
|
||||
stack.push(l);
|
||||
}
|
||||
for entry in node.entries {
|
||||
if let Some(t) = entry.tree {
|
||||
stack.push(t);
|
||||
}
|
||||
}
|
||||
out.insert(cid, bytes.to_vec());
|
||||
}
|
||||
Err(_) => {
|
||||
// Not an MST node — likely a value block. Skip.
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use at_crypto::cid::cid_for_cbor;
|
||||
use k256::ecdsa::SigningKey;
|
||||
use k256::SecretKey;
|
||||
|
||||
fn dummy_value_bytes(s: &str) -> Vec<u8> {
|
||||
let v = serde_json::json!({"text": s});
|
||||
let mut buf = Vec::new();
|
||||
ciborium::into_writer(&v, &mut buf).unwrap();
|
||||
buf
|
||||
}
|
||||
|
||||
fn dummy_repo() -> (Repo<crate::MemoryBlockstore>, SigningKey) {
|
||||
let sk = SigningKey::from(SecretKey::from_slice(&[42u8; 32]).unwrap());
|
||||
let bs = Arc::new(crate::MemoryBlockstore::new());
|
||||
let repo = Repo::new("did:plc:test".into(), sk.clone(), bs);
|
||||
(repo, sk)
|
||||
}
|
||||
|
||||
async fn put_value(
|
||||
repo: &mut Repo<crate::MemoryBlockstore>,
|
||||
coll: &str,
|
||||
rkey: &str,
|
||||
label: &str,
|
||||
) -> Cid {
|
||||
let bytes = dummy_value_bytes(label);
|
||||
let cid = cid_for_cbor(&bytes).unwrap();
|
||||
repo.blockstore.put(&cid, Bytes::from(bytes)).await.unwrap();
|
||||
repo.put_record(coll, rkey, cid).await.unwrap();
|
||||
cid
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_then_get_returns_value_cid() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let cid = put_value(&mut repo, "app.twi.post", "abc", "hello").await;
|
||||
let (uri, returned) = repo
|
||||
.put_record("app.twi.post", "abc", cid)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(uri, "at://did:plc:test/app.twi.post/abc");
|
||||
assert_eq!(returned, cid);
|
||||
let got = repo
|
||||
.get_record("app.twi.post", "abc")
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("record present");
|
||||
assert_eq!(got, cid);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_key_returns_none() {
|
||||
let (repo, _sk) = dummy_repo();
|
||||
let got = repo.get_record("c", "missing").await.unwrap();
|
||||
assert_eq!(got, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_record_removes_key() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let cid = put_value(&mut repo, "c", "a", "v1").await;
|
||||
assert_eq!(repo.get_record("c", "a").await.unwrap(), Some(cid));
|
||||
repo.delete_record("c", "a").await.unwrap();
|
||||
assert_eq!(repo.get_record("c", "a").await.unwrap(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_verifies_with_signing_key() {
|
||||
let (mut repo, sk) = dummy_repo();
|
||||
put_value(&mut repo, "c", "a", "v1").await;
|
||||
let commit = repo.commit().await.unwrap();
|
||||
assert!(commit.verify(&sk.verifying_key()).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_commits_produce_different_cids() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
put_value(&mut repo, "c", "a", "v1").await;
|
||||
let c1 = repo.commit().await.unwrap();
|
||||
put_value(&mut repo, "c", "b", "v2").await;
|
||||
let c2 = repo.commit().await.unwrap();
|
||||
assert_ne!(c1.cid, c2.cid);
|
||||
assert_eq!(c2.prev, Some(c1.cid));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_data_equals_mst_root() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
put_value(&mut repo, "c", "k", "v1").await;
|
||||
let commit = repo.commit().await.unwrap();
|
||||
assert_eq!(commit.data, repo.mst.root_cid());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn first_commit_prev_is_none() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let commit = repo.commit().await.unwrap();
|
||||
assert_eq!(commit.prev, None);
|
||||
assert_eq!(commit.data, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn commit_prev_links_to_previous_commit() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
put_value(&mut repo, "c", "a", "v1").await;
|
||||
let c1 = repo.commit().await.unwrap();
|
||||
put_value(&mut repo, "c", "b", "v2").await;
|
||||
let c2 = repo.commit().await.unwrap();
|
||||
assert_eq!(c2.prev, Some(c1.cid));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn serialize_repo_includes_mst_and_commit_blocks() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let value_cid = put_value(&mut repo, "c", "a", "v1").await;
|
||||
let commit = repo.commit().await.unwrap();
|
||||
let (header, blocks) = repo.serialize_repo().await.unwrap();
|
||||
assert_eq!(header, commit.signed_bytes);
|
||||
assert!(
|
||||
blocks.contains_key(&commit.cid),
|
||||
"commit block must be present"
|
||||
);
|
||||
let root_cid = repo.mst.root_cid().unwrap();
|
||||
assert!(
|
||||
blocks.contains_key(&root_cid),
|
||||
"MST root must be present"
|
||||
);
|
||||
assert!(
|
||||
blocks.contains_key(&value_cid),
|
||||
"value block must be present"
|
||||
);
|
||||
// Every block should be self-consistent under its CID.
|
||||
for (cid, bytes) in &blocks {
|
||||
let computed = cid_for_cbor(bytes).unwrap();
|
||||
assert_eq!(*cid, computed, "block CID mismatch for {cid}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn serialize_repo_before_commit_has_empty_header() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
put_value(&mut repo, "c", "a", "v1").await;
|
||||
let (header, blocks) = repo.serialize_repo().await.unwrap();
|
||||
assert!(header.is_empty(), "header bytes must be empty pre-commit");
|
||||
assert!(
|
||||
!blocks.is_empty(),
|
||||
"should have MST blocks even without a commit"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_repo_commit_has_null_data() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let commit = repo.commit().await.unwrap();
|
||||
assert_eq!(commit.data, None);
|
||||
// The signed CBOR must encode `data` as JSON null.
|
||||
let value: Value = ciborium::from_reader(commit.signed_bytes.as_slice()).unwrap();
|
||||
assert_eq!(value["data"], Value::Null);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_round_trip_preserves_mst_entries() {
|
||||
let (mut repo1, sk) = dummy_repo();
|
||||
let c1 = put_value(&mut repo1, "c", "a", "v1").await;
|
||||
let head = repo1.commit().await.unwrap();
|
||||
let c2 = put_value(&mut repo1, "c", "b", "v2").await;
|
||||
let head_with_b = repo1.commit().await.unwrap();
|
||||
assert_eq!(head_with_b.prev, Some(head.cid));
|
||||
// Sanity check before we load.
|
||||
assert_eq!(
|
||||
repo1.get_record("c", "a").await.unwrap(),
|
||||
Some(c1)
|
||||
);
|
||||
assert_eq!(
|
||||
repo1.get_record("c", "b").await.unwrap(),
|
||||
Some(c2)
|
||||
);
|
||||
|
||||
// Reconstruct from the second commit which contains both records.
|
||||
let mut repo2 = Repo::<crate::MemoryBlockstore>::load(
|
||||
"did:plc:test".into(),
|
||||
sk,
|
||||
repo1.blockstore.clone(),
|
||||
head_with_b.cid,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(repo2.get_record("c", "a").await.unwrap(), Some(c1));
|
||||
assert_eq!(repo2.get_record("c", "b").await.unwrap(), Some(c2));
|
||||
|
||||
// A subsequent commit links back to the reconstructed head.
|
||||
put_value(&mut repo2, "c", "c", "v3").await;
|
||||
let next = repo2.commit().await.unwrap();
|
||||
assert_eq!(next.prev, Some(head_with_b.cid));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_repopulates_value_cids_for_serialize() {
|
||||
// After Repo::load, serialize_repo should include pre-existing value
|
||||
// blocks (not just blocks added since the load).
|
||||
let (mut repo1, sk) = dummy_repo();
|
||||
let c1 = put_value(&mut repo1, "c", "a", "v1").await;
|
||||
let head = repo1.commit().await.unwrap();
|
||||
let repo2 = Repo::<crate::MemoryBlockstore>::load(
|
||||
"did:plc:test".into(),
|
||||
sk,
|
||||
repo1.blockstore.clone(),
|
||||
head.cid,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (_h, blocks) = repo2.serialize_repo().await.unwrap();
|
||||
assert!(
|
||||
blocks.contains_key(&c1),
|
||||
"value block must be present after load + serialize_repo"
|
||||
);
|
||||
assert!(blocks.contains_key(&head.cid));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rev_increments_after_commit() {
|
||||
let (mut repo, _sk) = dummy_repo();
|
||||
let r0 = repo.rev.clone();
|
||||
let commit = repo.commit().await.unwrap();
|
||||
assert_eq!(commit.rev, r0);
|
||||
// After commit(), rev has been bumped.
|
||||
assert_ne!(repo.rev, r0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
pub const TID_BASE32: &[u8] = b"234567abcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
/// Process-local monotonic counter used to disambiguate TIDs that would
|
||||
/// otherwise collide on the same microsecond.
|
||||
///
|
||||
/// The wall clock gives us 13 base32 chars of timestamp (≈52 bits of
|
||||
/// micros). Two writes in the same microsecond on the same PDS would
|
||||
/// otherwise produce the identical TID, and `put_record` would silently
|
||||
/// overwrite the prior record in the MST (same rkey, but actually
|
||||
/// different content — the value CIDs differ but the MST key is the
|
||||
/// TID, so the prior record becomes unreachable from the head commit).
|
||||
///
|
||||
/// We tack 12 low bits of a `fetch_add` counter into the encoded value
|
||||
/// so back-to-back calls — even inside the same microsecond — always
|
||||
/// yield different TIDs. The counter starts at 0; the first call's
|
||||
/// fetch_add returns 0 and produces a TID encoding
|
||||
/// `(now_micros << 12) | 0`. The counter is monotonic per process,
|
||||
/// not globally — a process restart will reset it to 0, which means
|
||||
/// a TID emitted by the new process may sort *before* a TID emitted
|
||||
/// by its predecessor on the same wall-clock microsecond. That's
|
||||
/// acceptable because TIDs are only used as MST rkeys within a
|
||||
/// single repo's history; the protocol doesn't require cross-process
|
||||
/// monotonicity.
|
||||
static TID_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct Tid {
|
||||
pub raw: String,
|
||||
}
|
||||
|
||||
impl Tid {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
raw: generate_tid(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_string(s: impl Into<String>) -> Self {
|
||||
Self { raw: s.into() }
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.raw
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Tid {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Tid {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.raw)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn generate_tid() -> String {
|
||||
// Phase 5b H10 — the counter is 12 bits wide, so it would wrap after
|
||||
// 4096 calls inside a single microsecond. To prevent that, we
|
||||
// block until the wall clock advances whenever the low-12 counter
|
||||
// has cycled back to 0 inside the same microsecond. In practice
|
||||
// this never fires (4096 TIDs/µs ≈ 4 billion/sec from one process)
|
||||
// but it's a cheap insurance policy.
|
||||
let mut now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_micros() as u64;
|
||||
let counter = loop {
|
||||
let prev = TID_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
// The counter is reset to 0 at process start; the low 12 bits
|
||||
// are an in-microsecond disambiguator. If we've wrapped back to
|
||||
// 0 mid-microsecond, spin until the clock advances.
|
||||
if (prev & 0xFFF) == 0 && prev != 0 {
|
||||
let next = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_micros() as u64;
|
||||
if next == now {
|
||||
std::hint::spin_loop();
|
||||
continue;
|
||||
}
|
||||
now = next;
|
||||
}
|
||||
break prev;
|
||||
};
|
||||
|
||||
// Combine timestamp + 12-bit disambiguator. The wall clock fits
|
||||
// comfortably in 52 bits, so the counter never spills into the
|
||||
// timestamp portion for any realistic process lifetime (~year 2400).
|
||||
let combined: u64 = (now << 12) | (counter & 0xFFF);
|
||||
|
||||
let mut s = String::with_capacity(13);
|
||||
let mut n = combined;
|
||||
for _ in 0..13 {
|
||||
let idx = (n & 0x1F) as usize;
|
||||
s.push(TID_BASE32[idx] as char);
|
||||
n >>= 5;
|
||||
}
|
||||
s.chars().rev().collect()
|
||||
}
|
||||
|
||||
pub fn compare_tid(a: &str, b: &str) -> std::cmp::Ordering {
|
||||
a.cmp(b)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn tid_increases() {
|
||||
let t1 = generate_tid();
|
||||
std::thread::sleep(std::time::Duration::from_millis(2));
|
||||
let t2 = generate_tid();
|
||||
// Strict ordering: t2 must be greater than t1. Accepting `is_le`
|
||||
// would mask the very bug this test exists to catch.
|
||||
assert!(compare_tid(&t1, &t2).is_lt());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tid_uses_lowercase_base32() {
|
||||
let t = generate_tid();
|
||||
for c in t.chars() {
|
||||
assert!(matches!(c, '2'..='7' | 'a'..='z'));
|
||||
}
|
||||
}
|
||||
|
||||
/// Phase 5b H10 — two writes in the same microsecond used to
|
||||
/// produce identical TIDs, which caused `put_record` to silently
|
||||
/// overwrite the prior record (different value CID, but the same
|
||||
/// rkey, so the new MST entry eclipsed the old). Verify a tight
|
||||
/// burst of N calls yields N distinct TIDs.
|
||||
#[test]
|
||||
fn generate_tid_is_monotonic_per_process() {
|
||||
let n = 1_000;
|
||||
let mut seen = HashSet::with_capacity(n);
|
||||
let mut prev: Option<String> = None;
|
||||
for _ in 0..n {
|
||||
let t = generate_tid();
|
||||
assert!(
|
||||
seen.insert(t.clone()),
|
||||
"duplicate TID produced in tight loop: {t}"
|
||||
);
|
||||
if let Some(p) = prev.as_ref() {
|
||||
assert!(
|
||||
compare_tid(p, &t).is_lt(),
|
||||
"TID must strictly increase per process: {p} >= {t}"
|
||||
);
|
||||
}
|
||||
prev = Some(t);
|
||||
}
|
||||
assert_eq!(seen.len(), n);
|
||||
}
|
||||
|
||||
/// Same as above but explicitly constructs the "same microsecond"
|
||||
/// worst case by sampling TIDs back-to-back without sleeping. The
|
||||
/// counter overlay must keep them distinct even when the wall
|
||||
/// clock doesn't tick.
|
||||
#[test]
|
||||
fn generate_tid_avoids_same_microsecond_collisions() {
|
||||
let n = 100;
|
||||
let mut seen = HashSet::with_capacity(n);
|
||||
for _ in 0..n {
|
||||
let t = generate_tid();
|
||||
assert!(seen.insert(t.clone()), "collision at {t}");
|
||||
}
|
||||
assert_eq!(seen.len(), n);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user