From 5ca18c440434f233fcbb3a23694f1cd7bbf37a3f Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 17 Sep 2026 08:35:05 +0100 Subject: [PATCH 1/2] fix(vahter): rebuild idx_event_rawmessage_trgm with fastupdate off Every vahter 504 in Envoy (response_flags=UT, 15s upstream timeout on the Telegram webhook POST /bot) traces to a single INSERT INTO event stalling for 8-30s, and Tempo + the event table show the stall recurring every 474-526 MessageReceived rows (~600-900 kB of rawMessage), 3-5 times a day. That cadence is the GIN pending list of idx_event_rawmessage_trgm (V47 forensic index, 653 MB, gin_pending_list_limit 4 MB) hitting its limit: the inserting backend flushes it synchronously, and on B1ms/P4 (120 IOPS) that takes tens of seconds while blocking other inserters, so Telegram's 15s redelivery to the second pod stalls on the same flush. Recreate the index CONCURRENTLY with fastupdate=off so trigram entries go straight into the index on each insert instead of accumulating into a periodic flush. Same idempotent DROP/CREATE CONCURRENTLY shape as V49. Co-Authored-By: Claude Fable 5.1 --- .../migrations/V51__rawmessage_trgm_fastupdate_off.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql diff --git a/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql b/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql new file mode 100644 index 0000000..ce6d1ab --- /dev/null +++ b/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql @@ -0,0 +1,7 @@ +-- fastupdate off: with it on, the inserting backend flushes the GIN pending list every +-- ~500 MessageReceived rows, which blocks the webhook INSERT for 10-30s on this server tier. +DROP INDEX CONCURRENTLY IF EXISTS idx_event_rawmessage_trgm; +CREATE INDEX CONCURRENTLY idx_event_rawmessage_trgm + ON event USING gin ((data->>'rawMessage') gin_trgm_ops) + WITH (fastupdate = off) + WHERE event_type = 'MessageReceived'; From 90b0233cc862939eecb8b48f9f1b5b1db0f98fb9 Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Thu, 17 Sep 2026 08:42:13 +0100 Subject: [PATCH 2/2] fix(vahter): idx_forensic_ prefix for debug-only indexes The rebuilt trigram index becomes idx_forensic_event_rawmessage_trgm so its name records that it is a debug-session index built with fastupdate=off. V52 gives the same prefix to the other indexes no bot query reads: idx_event_username_lower and idx_event_user_ban_by_stream (V30 "debug indexes"), idx_event_mlscored_score_created_at (V48, LLM-band analysis query run by hand), and the three snapshot_user indexes (V38/V43; snapshot_user is only ever written by bot code, the debug/ops queries read it). Verified by grepping DB.fs for each index's predicate; the only low-scan event index that IS read by prod code, idx_event_msg_marked_ham_text (GetUserStats' is_ham subquery), keeps its name. Co-Authored-By: Claude Fable 5.1 --- .../migrations/V51__rawmessage_trgm_fastupdate_off.sql | 7 ++++--- src/vahter-bot/migrations/V52__mark_forensic_indexes.sql | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 src/vahter-bot/migrations/V52__mark_forensic_indexes.sql diff --git a/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql b/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql index ce6d1ab..2b07767 100644 --- a/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql +++ b/src/vahter-bot/migrations/V51__rawmessage_trgm_fastupdate_off.sql @@ -1,7 +1,8 @@ --- fastupdate off: with it on, the inserting backend flushes the GIN pending list every --- ~500 MessageReceived rows, which blocks the webhook INSERT for 10-30s on this server tier. +-- idx_forensic_*: debug-session indexes, not read by bot code. GIN ones are built with fastupdate +-- off, since a pending-list flush stalls the inserting webhook request for 10-30s on this tier. DROP INDEX CONCURRENTLY IF EXISTS idx_event_rawmessage_trgm; -CREATE INDEX CONCURRENTLY idx_event_rawmessage_trgm +DROP INDEX CONCURRENTLY IF EXISTS idx_forensic_event_rawmessage_trgm; +CREATE INDEX CONCURRENTLY idx_forensic_event_rawmessage_trgm ON event USING gin ((data->>'rawMessage') gin_trgm_ops) WITH (fastupdate = off) WHERE event_type = 'MessageReceived'; diff --git a/src/vahter-bot/migrations/V52__mark_forensic_indexes.sql b/src/vahter-bot/migrations/V52__mark_forensic_indexes.sql new file mode 100644 index 0000000..26249a7 --- /dev/null +++ b/src/vahter-bot/migrations/V52__mark_forensic_indexes.sql @@ -0,0 +1,8 @@ +-- Remaining debug-session indexes (V30, V38, V43, V48) get the idx_forensic_ prefix so the +-- catalog itself shows which indexes no bot query depends on. +ALTER INDEX IF EXISTS idx_event_username_lower RENAME TO idx_forensic_event_username_lower; +ALTER INDEX IF EXISTS idx_event_user_ban_by_stream RENAME TO idx_forensic_event_user_ban_by_stream; +ALTER INDEX IF EXISTS idx_event_mlscored_score_created_at RENAME TO idx_forensic_event_mlscored_score_created_at; +ALTER INDEX IF EXISTS idx_snapshot_user_username RENAME TO idx_forensic_snapshot_user_username; +ALTER INDEX IF EXISTS idx_snapshot_user_banned RENAME TO idx_forensic_snapshot_user_banned; +ALTER INDEX IF EXISTS idx_snapshot_user_spam_protection RENAME TO idx_forensic_snapshot_user_spam_protection;