CLEAN-96: retry transient LightRAG failures without a person

admin/slices/reins/components/knowledge/SourceStatusBadge.vue

@@ -10,6 +10,8 @@ const label = computed(() => {
       return 'Indexed';
     case 'failed':
       return 'Failed';
+    case 'retrying':
+      return 'Retrying';
     case 'pending':
       return 'Pending';
     default:

admin/slices/reins/components/knowledge/overview/Provider.vue

@@ -15,16 +15,23 @@ const knowledgeId = computed(() => current?.value?.id ?? '');
 // reports.
 const counts = computed(() => {
   const k = current?.value;
-  if (!k) return { total: 0, indexed: 0, processing: 0, failed: 0, pending: 0 };
+  if (!k) {
+    return { total: 0, indexed: 0, processing: 0, failed: 0, retrying: 0, pending: 0 };
+  }
   const pending = Math.max(
     0,
-    k.sourceCount - k.indexedCount - k.processingCount - k.failedCount,
+    k.sourceCount -
+      k.indexedCount -
+      k.processingCount -
+      k.failedCount -
+      k.retryingCount,
   );
   return {
     total: k.sourceCount,
     indexed: k.indexedCount,
     processing: k.processingCount,
     failed: k.failedCount,
+    retrying: k.retryingCount,
     pending,
   };
 });
@@ -177,6 +184,9 @@ async function save(): Promise<void> {
         >
           {{ counts.failed }}
         </p>
+        <p v-if="counts.retrying" class="text-xs text-muted-foreground">
+          {{ counts.retrying }} retrying automatically
+        </p>
         <p v-if="counts.pending" class="text-xs text-muted-foreground">
           {{ counts.pending }} never sent
         </p>

admin/slices/reins/components/knowledge/sources/Provider.vue

@@ -9,8 +9,13 @@ import type {
   SourceIndexStatus,
   SourceType,
 } from '#reins/stores/knowledge';
-import { Download, Eye, ScanText, Trash2 } from 'lucide-vue-next';
-import { errorMessageOf, formatBytes, formatDate } from '#reins/domain';
+import { Download, Eye, RefreshCw, ScanText, Trash2 } from 'lucide-vue-next';
+import {
+  errorMessageOf,
+  formatBytes,
+  formatDate,
+  formatTime,
+} from '#reins/domain';
 import { Badge } from '#theme/components/ui/badge';
 import { Button } from '#theme/components/ui/button';
 import { Checkbox } from '#theme/components/ui/checkbox';
@@ -46,8 +51,12 @@ const STATUS_OPTIONS: { value: SourceIndexStatus | 'all'; label: string }[] = [
   { value: 'all', label: 'All statuses' },
   { value: 'indexed', label: 'Indexed' },
   { value: 'pending', label: 'Pending' },
+  { value: 'retrying', label: 'Retrying' },
   { value: 'failed', label: 'Failed' },
 ];
+// One upload plus the API's three automatic retries (MAX_INDEX_RETRIES in
+// reins/source/domain/indexFailure.ts); only shown, never enforced here.
+const MAX_INDEX_ATTEMPTS = 4;
 const TYPE_OPTIONS: { value: SourceType | 'all'; label: string }[] = [
   { value: 'all', label: 'All types' },
   { value: 'file', label: 'File' },
@@ -279,6 +288,15 @@ async function handleReextract(source: ISource) {
   await load();
 }
 
+// A failed row gets another go with a fresh attempt count. When LightRAG
+// still holds the document the API schedules it for its reconciler (the row
+// reads "Retrying" at once); otherwise it is uploaded again in the background.
+async function handleRetry(source: ISource) {
+  await store.reindexSource(knowledgeId.value, source.id);
+  await load();
+  if (refresh) await refresh();
+}
+
 async function onAdded() {
   // The new rows land at the end of the list (oldest first), so a user sitting
   // on a later page or a filter would not see them; go back to a clean view.
@@ -439,11 +457,22 @@ async function onAdded() {
                 >
                   {{ s.textError }}
                 </span>
+                <span
+                  v-else-if="s.indexStatus === 'retrying'"
+                  class="text-xs text-muted-foreground"
+                  :title="s.indexError ?? undefined"
+                >
+                  Attempt {{ s.indexAttempts + 1 }} of {{ MAX_INDEX_ATTEMPTS }},
+                  next try {{ formatTime(s.indexRetryAt) }}
+                </span>
                 <span
                   v-else-if="s.indexStatus === 'failed' && s.indexError"
                   class="line-clamp-2 text-xs break-words text-destructive"
                   :title="s.indexError"
                 >
+                  <template v-if="s.indexAttempts > 1">
+                    Gave up after {{ s.indexAttempts }} attempts:
+                  </template>
                   {{ s.indexError }}
                 </span>
               </div>
@@ -485,6 +514,16 @@ async function onAdded() {
                   <ScanText class="size-4" />
                   <span class="sr-only">Re-extract text from {{ s.name }}</span>
                 </Button>
+                <Button
+                  v-if="s.indexStatus === 'failed' || s.indexStatus === 'retrying'"
+                  size="icon-sm"
+                  variant="ghost"
+                  :title="s.indexStatus === 'retrying' ? 'Retry now' : 'Retry'"
+                  @click="handleRetry(s)"
+                >
+                  <RefreshCw class="size-4" />
+                  <span class="sr-only">Retry {{ s.name }}</span>
+                </Button>
                 <Button
                   size="icon-sm"
                   variant="ghost"

admin/slices/reins/data/knowledge.mapper.ts

@@ -38,6 +38,7 @@ const SOURCE_TYPES = new Set<SourceType>(['file', 'url', 'text']);
 const SOURCE_INDEX_STATUSES = new Set<SourceIndexStatus>([
   'indexed',
   'pending',
+  'retrying',
   'failed',
 ]);
 const IMPORT_JOB_STATUSES = new Set<ImportJobStatus>([
@@ -131,6 +132,7 @@ export class KnowledgeMapper {
       sourceCount: num(o.sourceCount),
       indexedCount: num(o.indexedCount),
       failedCount: num(o.failedCount),
+      retryingCount: num(o.retryingCount),
       processingCount: num(o.processingCount),
       indexRunAlive: o.indexRunAlive === true,
       instanceState:
@@ -165,6 +167,7 @@ export class KnowledgeMapper {
       sourceCount: num(o.sourceCount),
       indexedCount: num(o.indexedCount),
       failedCount: num(o.failedCount),
+      retryingCount: num(o.retryingCount),
       processingCount: num(o.processingCount),
       byType: { file: num(t.file), url: num(t.url), text: num(t.text) },
       totalSizeBytes: num(o.totalSizeBytes),
@@ -222,6 +225,8 @@ export class KnowledgeMapper {
           : 'queued',
       indexError: nullableStr(o.indexError),
       indexedAt: nullableStr(o.indexedAt),
+      indexAttempts: num(o.indexAttempts),
+      indexRetryAt: nullableStr(o.indexRetryAt),
       textState: readTextState(o.textState),
       textError: nullableStr(o.textError),
       createdAt: str(o.createdAt),

admin/slices/reins/domain/format.ts

@@ -47,3 +47,13 @@ export function formatDate(iso: string | null | undefined): string {
 export function formatDateTime(iso: string | null | undefined): string {
   return parseDate(iso)?.toLocaleString() ?? 'never';
 }
+
+/** Time of day only, for something due within the hour. */
+export function formatTime(iso: string | null | undefined): string {
+  return (
+    parseDate(iso)?.toLocaleTimeString([], {
+      hour: '2-digit',
+      minute: '2-digit',
+    }) ?? '-'
+  );
+}

admin/slices/reins/domain/knowledge.types.ts

@@ -23,7 +23,11 @@ export type IndexStatus =
   | 'empty'
   | 'partial';
 export type SourceType = 'file' | 'url' | 'text';
-export type SourceIndexStatus = 'indexed' | 'pending' | 'failed';
+/**
+ * `retrying` is a failure the API will try again on its own (a model outage,
+ * a lost connection); `failed` is one nobody will touch without a person.
+ */
+export type SourceIndexStatus = 'indexed' | 'pending' | 'retrying' | 'failed';
 /**
  * Text extraction for a PDF without a text layer: `none` (not a PDF, or it
  * has its own text), `pending` (probing or OCR running), `ready` (recognised
@@ -52,7 +56,10 @@ export interface IKnowledge {
   /** Index progress over the attached sources, as counted by the API. */
   sourceCount: number;
   indexedCount: number;
+  /** Terminal failures only. */
   failedCount: number;
+  /** Failed for a reason that passes; the API retries them on its own. */
+  retryingCount: number;
   /** Handed to LightRAG, not finished yet. Not an error, just not done. */
   processingCount: number;
   /**
@@ -95,6 +102,10 @@ export interface ISource {
   indexState: SourceIndexState;
   indexError: string | null;
   indexedAt: string | null;
+  /** Failed attempts since the last success or manual retry. */
+  indexAttempts: number;
+  /** When the API will retry a failed row by itself; null when it will not. */
+  indexRetryAt: string | null;
   textState: SourceTextState;
   textError: string | null;
   createdAt: string;
@@ -216,6 +227,7 @@ export interface IKnowledgeOverview {
   sourceCount: number;
   indexedCount: number;
   failedCount: number;
+  retryingCount: number;
   processingCount: number;
   byType: Record<SourceType, number>;
   totalSizeBytes: number;

admin/slices/reins/pages/knowledges/[id].vue

@@ -84,9 +84,12 @@ async function handleIndex() {
   // Sources LightRAG is still chunking are waited on, not re-sent, so they do
   // not belong in the "never confirmed" bucket that reads as work to redo.
   const processing = current.value.processingCount;
-  const neverConfirmed = toIndex.value - current.value.failedCount - processing;
+  const retrying = current.value.retryingCount;
+  const neverConfirmed =
+    toIndex.value - current.value.failedCount - retrying - processing;
   const breakdown = [
     `${current.value.failedCount} failed earlier`,
+    ...(retrying > 0 ? [`${retrying} being retried`] : []),
     ...(processing > 0 ? [`${processing} still processing`] : []),
     `${neverConfirmed} never confirmed`,
   ].join(', ');
@@ -194,7 +197,11 @@ const queuedCount = computed(() => {
   if (!k) return 0;
   // Never sent and not failed: what the Index button will actually process.
   return Math.max(
-    k.sourceCount - k.indexedCount - k.processingCount - k.failedCount,
+    k.sourceCount -
+      k.indexedCount -
+      k.processingCount -
+      k.failedCount -
+      k.retryingCount,
     0,
   );
 });
@@ -231,6 +238,12 @@ provide('knowledge-refresh', refresh);
           >
             · {{ current.processingCount }} processing
           </span>
+          <span
+            v-if="current.retryingCount"
+            title="Failed for a reason that passes (a model outage, a lost connection). The API retries them by itself."
+          >
+            · {{ current.retryingCount }} retrying
+          </span>
           <span v-if="current.failedCount" class="text-destructive">
             · {{ current.failedCount }} failed
           </span>

api/prisma/migrations/20260917120000_source_index_retry/migration.sql

@@ -0,0 +1,13 @@
+-- Automatic retry of index failures that pass on their own (a model outage, a
+-- lost connection). `indexAttempts` counts the failures since the last success
+-- or manual retry and sets the pause before the next try; `indexRetryAt` is
+-- when the reconciler may try again, null when it never will. Existing failed
+-- rows keep null: nothing is retried behind anyone's back by the migration.
+ALTER TABLE "Source" ADD COLUMN     "indexAttempts" INTEGER NOT NULL DEFAULT 0,
+ADD COLUMN     "indexRetryAt" TIMESTAMP(3),
+-- LightRAG's timestamp on the failed verdict a row was last re-queued over,
+-- so the reconciler can tell that verdict from a new one.
+ADD COLUMN     "indexRequeuedOverAt" TIMESTAMP(3);
+
+-- The reconciler asks for due rows every minute across every knowledge.
+CREATE INDEX "Source_indexRetryAt_idx" ON "Source"("indexRetryAt");

api/src/slices/reins/README.md

@@ -83,6 +83,41 @@ force. The manifest itself is not in this repo - the Mazda cluster's LightRAG is
 an ArgoCD app (`ranch-lightrag`) from `connectsolutions/mazda-ai-gitops`, path
 `helm/ranch-lightrag`.
 
+## A failure that passes on its own is retried on its own
+
+Bedrock answers `ServiceUnavailableException` in waves of ten to fifteen
+minutes (on dev 2026-09-17, 170 in a row between 00:14 and 00:27). LightRAG's
+own retry is five attempts a few seconds apart, so it gives up inside the wave
+and marks the document `failed`; the row records that, and nothing a person
+could press recovered it: an Index run or a Reindex re-uploads, LightRAG
+refuses the duplicate of the failed copy it still holds (`Original doc_id: X,
+Status: failed`), and the row fails again with that message. The only thing
+that clears a failed LightRAG document is `POST /documents/reprocess_failed`,
+which re-queues everything it holds in PENDING, PROCESSING or FAILED.
+
+So the reconciler does that. A failure whose message reads as transient
+(`source/domain/indexFailure.ts`: the Bedrock wrapper's `RetryError`, a 5xx
+from LightRAG, a lost connection, a refusal naming a *failed* original) earns
+a retry slot on the row, `indexRetryAt`, after 5, then 15, then 60 minutes;
+three retries and the row is an honest `failed`. When the slot comes and the
+base's pipeline is idle, `retryFailed` puts the row back in flight under the
+handle LightRAG really holds (a refusal's original, not the rejected copy) and
+the pipeline is nudged; a document LightRAG no longer holds at all is uploaded
+again. The usual confirm pass stamps it or records the next failure. Messages
+not on the list are permanent on purpose: a wrong "permanent" costs one look,
+a wrong "transient" hides a broken file behind "Retrying" for over an hour.
+Add to the list from a log line, not from a guess.
+
+Two consequences for anyone reading a row. `retrying` is a fourth source
+status, derived (`indexError` set, `indexRetryAt` set, no `indexedAt`), and
+is neither `failed` nor `processing` in the counts. And the reconciler now
+records a LightRAG-side failure it meets on a row it was confirming instead of
+dropping the handle; before that the row stayed `processing` with nothing to
+confirm, which is how a base read "Indexing…" for four days over one document.
+LightRAG's `updated_at` on the failed verdict travels with the row when it is
+re-queued (`indexRequeuedOverAt`); seeing that same verdict again is "not
+reached yet", any other timestamp is a new failure.
+
 ## Changing the extraction model requires clearing the LLM cache
 
 LightRAG caches every extraction and summary response in `lightrag_llm_cache`,

api/src/slices/reins/extraction/domain/textExtraction.service.spec.ts

@@ -33,6 +33,8 @@ function pdf(id: string, overrides: Partial<ISourceData> = {}): ISourceData {
     indexState: 'queued',
     indexError: 'left over from an earlier run',
     indexedAt: null,
+    indexAttempts: 0,
+    indexRetryAt: null,
     textState: 'none',
     textUrl: null,
     textError: null,

api/src/slices/reins/knowledge/data/knowledge.gateway.ts

@@ -96,16 +96,36 @@ export class KnowledgeGateway extends IKnowledgeGateway {
       for (let i = 0; i < row._count._all; i += 1) list.push(state);
       statesOf.set(row.knowledgeId, list);
     }
+    // A failed row the reconciler still owes a retry is stored as `failed`
+    // (it did fail) but is not one for the counts: the list must agree with
+    // the detail page, which counts through the source slice's filters.
+    const retryRows = ids.length
+      ? await this.prisma.source.groupBy({
+          by: ['knowledgeId'],
+          where: {
+            knowledgeId: { in: ids },
+            indexedAt: null,
+            indexError: { not: null },
+            indexRetryAt: { not: null },
+          },
+          _count: { _all: true },
+        })
+      : [];
+    const retryingOf = new Map(
+      retryRows.map((r) => [r.knowledgeId, r._count._all]),
+    );
 
     return {
       items: records.map((r) => {
         const states = statesOf.get(r.id) ?? [];
+        const retrying = retryingOf.get(r.id) ?? 0;
         return {
           ...this.mapper.toEntity(r),
           indexStatus: deriveIndexStatus(states),
           sourceCount: r._count.sources,
           indexedCount: states.filter((s) => s === 'indexed').length,
-          failedCount: states.filter((s) => s === 'failed').length,
+          failedCount: states.filter((s) => s === 'failed').length - retrying,
+          retryingCount: retrying,
           processingCount: states.filter((s) => s === 'processing').length,
           sourcesCount: r._count.sources,
           totalSizeBytes: sizeOf.get(r.id) ?? 0,

api/src/slices/reins/knowledge/domain/failureSummary.spec.ts

@@ -4,6 +4,7 @@ function failures(n: number) {
   return Array.from({ length: n }, (_, i) => ({
     name: `doc-${i + 1}.pdf`,
     error: 'only whitespace',
+    retryAt: null,
   }));
 }
 
@@ -28,8 +29,18 @@ describe('summarizeFailures', () => {
   });
 
   it('falls back to a wording for a failure with no message', () => {
-    expect(summarizeFailures([{ name: 'a.pdf', error: null }])).toBe(
-      '1 source(s) failed: a.pdf (unknown error)',
-    );
+    expect(
+      summarizeFailures([{ name: 'a.pdf', error: null, retryAt: null }]),
+    ).toBe('1 source(s) failed: a.pdf (unknown error)');
+  });
+
+  it('says how many of them the reconciler will retry on its own', () => {
+    const line = summarizeFailures([
+      { name: 'a.pdf', error: 'RetryError[...]', retryAt: new Date() },
+      { name: 'b.pdf', error: 'RetryError[...]', retryAt: new Date() },
+      { name: 'c.pdf', error: 'only whitespace', retryAt: null },
+    ])!;
+    expect(line.startsWith('3 source(s) failed: a.pdf')).toBe(true);
+    expect(line.endsWith('; 2 of them will be retried automatically')).toBe(true);
   });
 });