diff --git a/crates/appview/src/routes.rs b/crates/appview/src/routes.rs index 9e497f9..0b56455 100644 --- a/crates/appview/src/routes.rs +++ b/crates/appview/src/routes.rs @@ -318,20 +318,51 @@ async fn timeline_home( .map_err(db_err)?, } } else { - // Graph-aware branch: filter `posts.did` to the followee set - // plus the requesting user's own DID. `target_dids` has been - // deduped and capped at MAX_FOLLOWED_DIDS, and the user's own - // DID is guaranteed to be in the set. + // Graph-aware branch: the followee set plus the requesting + // user's own DID. `target_dids` has been deduped and capped at + // MAX_FOLLOWED_DIDS, and the user's own DID is guaranteed to + // be in the set. + // + // ## Why this is a LATERAL and not `did = ANY($2)` + // + // The straightforward `WHERE did = ANY($2) ORDER BY indexed_at + // DESC LIMIT n` is a plan-stability trap once + // `posts_feed_indexed_at_uri_idx` exists (migration 0009, added + // for the cold-start feed). The planner sees an index that + // already yields rows in `indexed_at DESC` order and assumes it + // will hit `n` matching rows early — so it walks the global + // feed and filters. When the followees are sparse (a fresh + // account following accounts that haven't posted), "early" + // means millions of rows: measured on the dev instance, 2.87 M + // rows discarded and 28 s per request, while the same query + // took 62 ms with the per-DID index. It also flipped between + // the two plans depending on how often the prepared statement + // had run, so it looked intermittent. + // + // Expressing the intent — "for each followee, their newest + // posts, merged" — takes that plan off the table: `unnest` is a + // relation the planner can size, and each iteration is a bounded + // range scan on `posts_did_indexed_at_uri_idx`. Fetching `$1` + // per followee is what makes it correct: the global top-N is + // always a subset of the union of the per-followee top-Ns. match cursor_ts { Some(ts) => sqlx::query_as::<_, PostRowWithIndexed>( - r#"SELECT uri, did, handle, rkey, collection, text, cid, - parent_uri, root_uri, embed, langs, created_at, - indexed_at - FROM posts - WHERE collection IN ('app.twi.post','app.bsky.feed.post') - AND did = ANY($2::text[]) - AND (indexed_at, uri) < ($3, $4) - ORDER BY indexed_at DESC, uri DESC + r#"SELECT t.uri, t.did, t.handle, t.rkey, t.collection, + t.text, t.cid, t.parent_uri, t.root_uri, + t.embed, t.langs, t.created_at, t.indexed_at + FROM unnest($2::text[]) AS f(did) + CROSS JOIN LATERAL ( + SELECT p.uri, p.did, p.handle, p.rkey, p.collection, + p.text, p.cid, p.parent_uri, p.root_uri, + p.embed, p.langs, p.created_at, p.indexed_at + FROM posts p + WHERE p.did = f.did + AND p.collection IN ('app.twi.post','app.bsky.feed.post') + AND (p.indexed_at, p.uri) < ($3, $4) + ORDER BY p.indexed_at DESC, p.uri DESC + LIMIT $1 + ) t + ORDER BY t.indexed_at DESC, t.uri DESC LIMIT $1"#, ) .bind(fetch) @@ -342,13 +373,21 @@ async fn timeline_home( .await .map_err(db_err)?, None => sqlx::query_as::<_, PostRowWithIndexed>( - r#"SELECT uri, did, handle, rkey, collection, text, cid, - parent_uri, root_uri, embed, langs, created_at, - indexed_at - FROM posts - WHERE collection IN ('app.twi.post','app.bsky.feed.post') - AND did = ANY($2::text[]) - ORDER BY indexed_at DESC, uri DESC + r#"SELECT t.uri, t.did, t.handle, t.rkey, t.collection, + t.text, t.cid, t.parent_uri, t.root_uri, + t.embed, t.langs, t.created_at, t.indexed_at + FROM unnest($2::text[]) AS f(did) + CROSS JOIN LATERAL ( + SELECT p.uri, p.did, p.handle, p.rkey, p.collection, + p.text, p.cid, p.parent_uri, p.root_uri, + p.embed, p.langs, p.created_at, p.indexed_at + FROM posts p + WHERE p.did = f.did + AND p.collection IN ('app.twi.post','app.bsky.feed.post') + ORDER BY p.indexed_at DESC, p.uri DESC + LIMIT $1 + ) t + ORDER BY t.indexed_at DESC, t.uri DESC LIMIT $1"#, ) .bind(fetch) diff --git a/migrations/appview/0009_handle_lookup_indexes.sql b/migrations/appview/0009_handle_lookup_indexes.sql new file mode 100644 index 0000000..3b2d984 --- /dev/null +++ b/migrations/appview/0009_handle_lookup_indexes.sql @@ -0,0 +1,58 @@ +-- AppView database schema 0009: indexes for handle → DID lookups. +-- +-- `/api/profile/` took 9.5 s on a 3.3 M-row `posts` table +-- (measured against the dev instance). Both halves of `resolve_profile` +-- were unindexed: +-- +-- 1. SELECT did FROM profiles WHERE LOWER(handle) = LOWER($1) +-- 2. SELECT did FROM posts WHERE handle = $1 +-- ORDER BY indexed_at DESC LIMIT 1 -- the fallback +-- +-- Step 2 was a parallel sequential scan over every post ever indexed +-- (`Rows Removed by Filter: 1101310` per worker), and it runs on every +-- profile view in the client. +-- +-- On `profiles`: migration 0007 dropped exactly this index, reasoning +-- that "every caller derives a DID first (via posts.handle or the +-- handle-sync worker) and then queries profiles by PK". That stopped +-- being true when `resolve_profile` learned to prefer the profiles +-- cache — it now asks `profiles` by handle *first*, precisely the +-- lookup 0007 removed the support for. Re-added, matching the +-- expression in the query (`LOWER(handle)`) so the planner can use it. +CREATE INDEX IF NOT EXISTS profiles_handle_lower_idx + ON profiles (LOWER(handle)); + +-- On `posts`: `(handle, indexed_at DESC)` covers filter *and* sort, so +-- the LIMIT 1 becomes an index scan that stops at the first row. +-- +-- Partial on `handle <> ''`: empty handles are the un-backfilled +-- majority on a firehose-fed instance and are never looked up by this +-- path (the handle-sync worker queries them through its own predicate), +-- so excluding them keeps the index small on the largest table we have. +CREATE INDEX IF NOT EXISTS posts_handle_indexed_at_idx + ON posts (handle, indexed_at DESC) + WHERE handle <> ''; + +-- ===================================================== +-- posts: the cold-start global feed +-- ===================================================== +-- +-- `/api/timeline/home` falls back to the global recent feed for users +-- without a follow graph — every new account's first screen. It took +-- 7.4 s (parallel seq scan + top-N sort over 3.3 M rows) and timed out +-- the integration tests' 5 s client. +-- +-- `posts_collection_indexed_at_uri_idx (collection, indexed_at DESC, +-- uri DESC)` cannot serve it: the query filters +-- `collection IN ('app.twi.post','app.bsky.feed.post')`, and with two +-- leading values the index no longer yields rows in `indexed_at` order, +-- so the planner falls back to scanning and sorting. +-- +-- A partial index over exactly that predicate moves the collection +-- filter into the index definition, which leaves `(indexed_at DESC, +-- uri DESC)` as the sort key — the LIMIT then stops after the first +-- page. Same shape as the existing `posts_did_indexed_at_uri_idx`, +-- which is partial on the same two collections. +CREATE INDEX IF NOT EXISTS posts_feed_indexed_at_uri_idx + ON posts (indexed_at DESC, uri DESC) + WHERE collection IN ('app.twi.post', 'app.bsky.feed.post');