Files
gbrain/test/sync-failures.test.ts
Garry Tan b5fa3d044a fix: 8 root-cause fixes from /investigate (v0.14.2) (#259)
* fix: 8 root-cause fixes from /investigate wave

Consolidated bundle of bug fixes from /investigate on the 8 deferred bugs.
Each fix was designed to go at the structural gap, not the symptom. Codex
verified 20 load-bearing claims on the plan; 12 triggered plan revisions.

Bug 2  — GBRAIN_POOL_SIZE env knob + init finally blocks (no auto-detect).
         Covers both the singleton pool (db.ts) and instance pool (import.ts:140).
Bug 3  — Centralize migration ledger writes in apply-migrations runner.
         Removed appendCompletedMigration from v0_11_0, v0_12_0, v0_12_2,
         v0_13_0, v0_13_1. Added 3-partial wedge cap + --force-retry reset.
         'complete wins' preserved; no partial can regress a completed migration.
Bug 5  — v0.14.0 migration registered. src/commands/migrations/v0_14_0.ts
         ships Phase A (ALTER minion_jobs.max_stalled SET DEFAULT 3) + Phase B
         (pending-host-work ping for shell-jobs adoption).
Bug 6/10 — jsonb_agg(DISTINCT ...) in legacy traverseGraph (both engines).
         Presentation-level dedup; schema still preserves provenance rows.
Bug 7  — doctor --fast reads DB URL source via getDbUrlSource() in config.ts.
         Precise message: 'Skipping DB checks (--fast mode, URL present from env)'
         replaces the misleading 'No database configured'.
Bug 8  — max_stalled default bumped 1→3 in schema-embedded.ts, pglite-schema.ts,
         schema.sql (new installs). v0_14_0 Phase A ALTER for existing installs.
         autopilot-cycle handler yields to event loop between phases so the
         worker's lock-renewal timer fires on huge brains. (Deep AbortSignal
         threading through runEmbedCore/runExtractCore/runBacklinksCore/performSync
         deferred to v0.15 queue polish.)
Bug 9  — Gate sync.last_commit on no-failures across all three sync paths
         (incremental, full via runImport, gbrain import git continuity).
         recordSyncFailures() helper + ~/.gbrain/sync-failures.jsonl with
         dedup key path+commit+error-hash. New flags: --skip-failed (ack) +
         --retry-failed (re-attempt). Doctor surfaces unacknowledged failures.
Bug 11 — brain_score breakdown fields on BrainHealth (embed_coverage_score,
         link_density_score, timeline_coverage_score, no_orphans_score,
         no_dead_links_score); sum equals brain_score by construction.
         dead_links now on the type (resolves featuresTeaserForDoctor drift).
         orphan_pages kept as 'islanded' (no inbound AND no outbound) and
         docs updated to match — explicit semantic instead of doc drift.

New tests: test/traverse-graph-dedup.test.ts, test/sync-failures.test.ts,
test/brain-score-breakdown.test.ts, test/migration-resume.test.ts,
test/migrations-v0_14_0.test.ts. Extended: migrate, doctor, apply-migrations.

All 1696 unit tests pass locally. postgres-jsonb E2E regression unchanged
(none of these touch the JSONB write surface).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* docs: v0.14.2 CHANGELOG + CLAUDE.md; align migration-flow E2E with runner-owned ledger

CHANGELOG: v0.14.2 entry in the standard release-summary format
(two-line headline + lead + numbers table + "what this means" +
"To take advantage of v0.14.2" self-repair block + itemized
changes grouped by reliability / observability / graph correctness /
new migration / tests / deferred-to-v0.15).

CLAUDE.md: new "Key commands added in v0.14.2" section covers
--skip-failed, --retry-failed, --force-retry, GBRAIN_POOL_SIZE env,
and the new doctor checks (sync_failures, brain_score breakdown).
Migration orchestrator docs updated to describe v0_14_0.ts + the
runner-owned ledger contract from Bug 3.

test/e2e/migration-flow.test.ts: three assertions updated to match
the Bug 3 contract — orchestrators no longer append to completed.jsonl
directly, so direct-orchestrator E2E calls leave the ledger empty.
Preferences assertions remain (that's still the orchestrator's side
of the contract). Runner's ledger write is covered by the unit suite
(test/apply-migrations.test.ts + test/migration-resume.test.ts).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-20 23:14:38 +08:00

161 lines
6.6 KiB
TypeScript

/**
* Bug 9 regression — sync silently drops files with broken YAML.
*
* Before the fix, sync.ts caught per-file parse errors, printed a warning,
* and still advanced sync.last_commit. The failed file was never retried
* because it was behind the bookmark. Silent data loss.
*
* After the fix:
* - failures append to ~/.gbrain/sync-failures.jsonl (with dedup)
* - incremental + full-sync + import git-continuity paths gate the
* sync.last_commit advance on "no failures"
* - `gbrain sync --skip-failed` acknowledges the current set
* - `gbrain doctor` surfaces unacknowledged failures
*
* This suite exercises the helper + the dedup behavior. The full CLI
* round-trip is covered by E2E tests.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, rmSync, readFileSync, existsSync, writeFileSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
// Point HOME at a tmpdir so we don't stomp the real ~/.gbrain/sync-failures.jsonl
let tmpHome: string;
const originalHome = process.env.HOME;
beforeEach(async () => {
tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-sync-failures-'));
process.env.HOME = tmpHome;
// Belt-and-suspenders: explicitly clear the jsonl at the resolved path.
const { syncFailuresPath } = await import('../src/core/sync.ts');
try { rmSync(syncFailuresPath(), { force: true }); } catch { /* none */ }
});
afterEach(() => {
if (originalHome) process.env.HOME = originalHome;
else delete process.env.HOME;
try { rmSync(tmpHome, { recursive: true, force: true }); } catch { /* ignore */ }
});
describe('Bug 9 — sync-failures JSONL helpers', () => {
test('recordSyncFailures appends one line per failure with dedup', async () => {
const { recordSyncFailures, loadSyncFailures, syncFailuresPath } = await import('../src/core/sync.ts');
recordSyncFailures([
{ path: 'people/alice.md', error: 'YAML: unexpected colon in title' },
{ path: 'notes/broken.md', error: 'YAML: duplicated key' },
], 'abc123def456');
expect(existsSync(syncFailuresPath())).toBe(true);
const entries = loadSyncFailures();
expect(entries.length).toBe(2);
expect(entries[0].path).toBe('people/alice.md');
expect(entries[0].commit).toBe('abc123def456');
expect(entries[0].acknowledged).toBeUndefined();
// Same failure on same commit should NOT re-append.
recordSyncFailures([
{ path: 'people/alice.md', error: 'YAML: unexpected colon in title' },
], 'abc123def456');
expect(loadSyncFailures().length).toBe(2);
// Different commit → new entry.
recordSyncFailures([
{ path: 'people/alice.md', error: 'YAML: unexpected colon in title' },
], 'zzz999');
expect(loadSyncFailures().length).toBe(3);
});
test('acknowledgeSyncFailures marks unacked entries, leaves acked alone', async () => {
const { recordSyncFailures, acknowledgeSyncFailures, loadSyncFailures } = await import('../src/core/sync.ts');
recordSyncFailures([
{ path: 'a.md', error: 'err1' },
{ path: 'b.md', error: 'err2' },
], 'commit1');
const n = acknowledgeSyncFailures();
expect(n).toBe(2);
const after = loadSyncFailures();
expect(after.every(e => e.acknowledged === true)).toBe(true);
expect(after.every(e => typeof e.acknowledged_at === 'string')).toBe(true);
// Second ack: nothing new to mark.
expect(acknowledgeSyncFailures()).toBe(0);
// Adding a fresh failure then ack: only the new one flips.
recordSyncFailures([{ path: 'c.md', error: 'err3' }], 'commit2');
expect(acknowledgeSyncFailures()).toBe(1);
expect(loadSyncFailures().length).toBe(3);
expect(loadSyncFailures().every(e => e.acknowledged === true)).toBe(true);
});
test('unacknowledgedSyncFailures filters correctly', async () => {
const { recordSyncFailures, acknowledgeSyncFailures, unacknowledgedSyncFailures } = await import('../src/core/sync.ts');
recordSyncFailures([{ path: 'a.md', error: 'err1' }], 'c1');
acknowledgeSyncFailures();
recordSyncFailures([{ path: 'b.md', error: 'err2' }], 'c2');
const unacked = unacknowledgedSyncFailures();
expect(unacked.length).toBe(1);
expect(unacked[0].path).toBe('b.md');
});
test('loadSyncFailures returns [] when file is missing', async () => {
const { loadSyncFailures } = await import('../src/core/sync.ts');
expect(loadSyncFailures()).toEqual([]);
});
test('loadSyncFailures tolerates malformed lines', async () => {
const { loadSyncFailures, syncFailuresPath, recordSyncFailures } = await import('../src/core/sync.ts');
// Seed one valid entry.
recordSyncFailures([{ path: 'a.md', error: 'err1' }], 'c1');
// Append garbage.
writeFileSync(syncFailuresPath(), readFileSync(syncFailuresPath(), 'utf-8') + 'NOT-JSON\n', { flag: 'w' });
const out = loadSyncFailures();
expect(out.length).toBe(1);
expect(out[0].path).toBe('a.md');
});
});
describe('Bug 9 — doctor surfaces sync failures', () => {
test('doctor source contains sync_failures check', async () => {
const source = await Bun.file(new URL('../src/commands/doctor.ts', import.meta.url)).text();
expect(source).toContain('sync_failures');
expect(source).toContain('unacknowledgedSyncFailures');
expect(source).toContain("'gbrain sync --skip-failed'");
});
});
describe('Bug 9 — sync.ts CLI flag wiring', () => {
test('runSync parses --skip-failed and --retry-failed flags', async () => {
const source = await Bun.file(new URL('../src/commands/sync.ts', import.meta.url)).text();
expect(source).toContain("args.includes('--skip-failed')");
expect(source).toContain("args.includes('--retry-failed')");
expect(source).toContain('skipFailed');
expect(source).toContain('retryFailed');
});
test('performSync gates sync.last_commit on failedFiles.length', async () => {
const source = await Bun.file(new URL('../src/commands/sync.ts', import.meta.url)).text();
// The gate exists and references the failure set.
expect(source).toContain('failedFiles.length > 0');
expect(source).toContain('blocked_by_failures');
});
test('performFullSync gates on result.failures from runImport', async () => {
const source = await Bun.file(new URL('../src/commands/sync.ts', import.meta.url)).text();
expect(source).toContain('result.failures.length > 0');
});
test('runImport returns RunImportResult with failures list', async () => {
const source = await Bun.file(new URL('../src/commands/import.ts', import.meta.url)).text();
expect(source).toContain('RunImportResult');
expect(source).toContain('failures: Array<{ path: string; error: string }>');
expect(source).toContain('recordSyncFailures');
});
});