v0.37.1.0 feat: brainstorm + lsd — bisociation idea generator grounded in your own brain (#1214)
* feat: brainstorm + lsd (v0.37 wave, pre-merge snapshot) Brainstorm + LSD bisociation idea generator. Will rebase + bump after master merge. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * docs: CHANGELOG voice tightening — strip meta-content from v0.37.1.0 entry CLAUDE.md gains an IRON RULE for CHANGELOG entries: the changelog describes what the user gets, not how the work happened. No mentions of review processes, plan files, decision tags, migration version drama, or 'what we caught and fixed before merging.' If a fact only exists because of the development workflow, it does not belong in release notes. Rewrite v0.37.1.0 entry to comply: cut the 'what we caught' section (architectural review drama), the 'plan + reviews' bullet, and the migration-renumbering aside. Entry shrinks 67 → 56 lines, every sentence now answers 'what can I do / how do I use it / what should I watch for.' Regenerated llms-full.txt to absorb the CLAUDE.md voice update. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -11,6 +11,9 @@ import type {
|
||||
FactRow, FactKind, FactVisibility, FactInsertStatus,
|
||||
NewFact, FactListOpts, FactsHealth,
|
||||
} from './engine.ts';
|
||||
import type {
|
||||
DomainBankSampleOpts, CorpusSampleOpts, DomainBankRow,
|
||||
} from './types.ts';
|
||||
import { MAX_SEARCH_LIMIT, clampSearchLimit } from './engine.ts';
|
||||
import { deriveResolutionTuple, finalizeScorecard } from './takes-resolution.ts';
|
||||
import { normalizeWeightForStorage } from './takes-fence.ts';
|
||||
@@ -403,7 +406,9 @@ export class PostgresEngine implements BrainEngine {
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'sources' AND column_name = 'archived_at') AS sources_archived_at_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'sources' AND column_name = 'archive_expires_at') AS sources_archive_expires_at_exists
|
||||
WHERE table_schema = current_schema() AND table_name = 'sources' AND column_name = 'archive_expires_at') AS sources_archive_expires_at_exists,
|
||||
EXISTS (SELECT 1 FROM information_schema.columns
|
||||
WHERE table_schema = current_schema() AND table_name = 'pages' AND column_name = 'last_retrieved_at') AS pages_last_retrieved_at_exists
|
||||
`;
|
||||
const probe = probeRows[0]!;
|
||||
|
||||
@@ -451,12 +456,18 @@ export class PostgresEngine implements BrainEngine {
|
||||
&& (!probe.sources_archived_exists
|
||||
|| !probe.sources_archived_at_exists
|
||||
|| !probe.sources_archive_expires_at_exists);
|
||||
// v0.37.0 (v79): pages_last_retrieved_at_idx in SCHEMA_SQL references
|
||||
// last_retrieved_at. Pre-v79 brains crash without the column; bootstrap
|
||||
// adds it before SCHEMA_SQL replay creates the index. v79 runs later
|
||||
// via runMigrations and is idempotent.
|
||||
const needsPagesLastRetrievedAt = probe.pages_exists && !(probe as { pages_last_retrieved_at_exists?: boolean }).pages_last_retrieved_at_exists;
|
||||
|
||||
if (!needsPagesBootstrap && !needsLinksBootstrap && !needsChunksBootstrap
|
||||
&& !needsPagesDeletedAt && !needsMcpLogBootstrap && !needsSubagentProviderId
|
||||
&& !needsChunksEmbeddingImage && !needsPagesRecency
|
||||
&& !needsIngestLogSourceId && !needsFilesBootstrap
|
||||
&& !needsOauthClientsBootstrap && !needsSourcesArchive) return;
|
||||
&& !needsOauthClientsBootstrap && !needsSourcesArchive
|
||||
&& !needsPagesLastRetrievedAt) return;
|
||||
|
||||
console.log(' Pre-v0.21 brain detected, applying forward-reference bootstrap');
|
||||
|
||||
@@ -635,6 +646,16 @@ export class PostgresEngine implements BrainEngine {
|
||||
ALTER TABLE sources ADD COLUMN IF NOT EXISTS archive_expires_at TIMESTAMPTZ;
|
||||
`);
|
||||
}
|
||||
|
||||
if (needsPagesLastRetrievedAt) {
|
||||
// v79 (pages_last_retrieved_at): adds the real stale-page signal column
|
||||
// + full B-tree index. SCHEMA_SQL's CREATE INDEX
|
||||
// pages_last_retrieved_at_idx crashes without the column. v79 runs
|
||||
// later via runMigrations and is idempotent.
|
||||
await conn.unsafe(`
|
||||
ALTER TABLE pages ADD COLUMN IF NOT EXISTS last_retrieved_at TIMESTAMPTZ;
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
async transaction<T>(fn: (engine: BrainEngine) => Promise<T>): Promise<T> {
|
||||
@@ -904,6 +925,166 @@ export class PostgresEngine implements BrainEngine {
|
||||
return rows.map((r) => ({ slug: r.slug as string, source_id: r.source_id as string }));
|
||||
}
|
||||
|
||||
// v0.37.0 — domain-bank engine methods (D14 + D5 + D10).
|
||||
//
|
||||
// `listPrefixSampledPages`: one page per prefix, tiebroken by inbound-link
|
||||
// count (connection_count via LEFT JOIN to page_links). Stale-bias optional
|
||||
// for LSD mode (D5). Source-scoped (D5). Excludes close-set slugs.
|
||||
//
|
||||
// Ranking inside each prefix partition:
|
||||
// 1. stale_score DESC (when staleBias) — never-retrieved beats >90d-stale beats fresh
|
||||
// 2. connection_count DESC — structural-centrality tiebreaker (D10)
|
||||
// 3. slug ASC — deterministic for tests
|
||||
async listPrefixSampledPages(opts: DomainBankSampleOpts): Promise<DomainBankRow[]> {
|
||||
const sql = this.sql;
|
||||
if (opts.prefixes.length === 0) return [];
|
||||
const exclude = opts.excludeSlugs ?? [];
|
||||
const staleBias = opts.staleBias === true;
|
||||
const staleThreshold = opts.staleThresholdDays ?? 90;
|
||||
// Source scoping (D5, codex r2 #2 — federated array wins over scalar).
|
||||
const sourceIds = opts.sourceIds ?? null;
|
||||
const sourceId = opts.sourceId ?? null;
|
||||
const rows = await sql`
|
||||
WITH prefix_pages AS (
|
||||
SELECT
|
||||
p.id AS page_id,
|
||||
p.slug,
|
||||
p.source_id,
|
||||
p.title,
|
||||
p.compiled_truth,
|
||||
p.last_retrieved_at,
|
||||
substring(p.slug from '^[^/]+/[^/]+') AS prefix,
|
||||
COUNT(pl.id) AS connection_count
|
||||
FROM pages p
|
||||
LEFT JOIN page_links pl ON pl.to_page_id = p.id
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND substring(p.slug from '^[^/]+/[^/]+') = ANY(${opts.prefixes}::text[])
|
||||
AND (cardinality(${exclude}::text[]) = 0 OR NOT (p.slug = ANY(${exclude}::text[])))
|
||||
AND (
|
||||
(${sourceIds}::text[] IS NOT NULL AND p.source_id = ANY(${sourceIds}::text[]))
|
||||
OR (${sourceIds}::text[] IS NULL AND ${sourceId}::text IS NOT NULL AND p.source_id = ${sourceId})
|
||||
OR (${sourceIds}::text[] IS NULL AND ${sourceId}::text IS NULL)
|
||||
)
|
||||
GROUP BY p.id, p.slug, p.source_id, p.title, p.compiled_truth, p.last_retrieved_at
|
||||
),
|
||||
ranked AS (
|
||||
SELECT
|
||||
pp.*,
|
||||
(CASE WHEN ${staleBias}::boolean THEN
|
||||
CASE
|
||||
WHEN pp.last_retrieved_at IS NULL THEN 2
|
||||
WHEN pp.last_retrieved_at < NOW() - (${staleThreshold}::int * INTERVAL '1 day') THEN 1
|
||||
ELSE 0
|
||||
END
|
||||
ELSE 0
|
||||
END) AS stale_score,
|
||||
ROW_NUMBER() OVER (
|
||||
PARTITION BY pp.prefix
|
||||
ORDER BY
|
||||
(CASE WHEN ${staleBias}::boolean THEN
|
||||
CASE
|
||||
WHEN pp.last_retrieved_at IS NULL THEN 2
|
||||
WHEN pp.last_retrieved_at < NOW() - (${staleThreshold}::int * INTERVAL '1 day') THEN 1
|
||||
ELSE 0
|
||||
END
|
||||
ELSE 0
|
||||
END) DESC,
|
||||
pp.connection_count DESC,
|
||||
pp.slug ASC
|
||||
) AS rn
|
||||
FROM prefix_pages pp
|
||||
),
|
||||
with_chunk AS (
|
||||
SELECT
|
||||
r.*,
|
||||
(
|
||||
SELECT cc.id FROM content_chunks cc
|
||||
WHERE cc.page_id = r.page_id AND cc.embedding IS NOT NULL
|
||||
ORDER BY cc.chunk_index ASC
|
||||
LIMIT 1
|
||||
) AS representative_chunk_id
|
||||
FROM ranked r
|
||||
WHERE r.rn = 1
|
||||
)
|
||||
SELECT page_id, slug, source_id, title, compiled_truth, last_retrieved_at,
|
||||
prefix, connection_count, representative_chunk_id
|
||||
FROM with_chunk
|
||||
ORDER BY prefix
|
||||
`;
|
||||
return rows.map((r): DomainBankRow => ({
|
||||
slug: r.slug as string,
|
||||
source_id: r.source_id as string,
|
||||
prefix: r.prefix as string | null,
|
||||
page_id: Number(r.page_id),
|
||||
title: r.title as string | null,
|
||||
compiled_truth: (r.compiled_truth as string | null) ?? '',
|
||||
connection_count: Number(r.connection_count),
|
||||
last_retrieved_at: r.last_retrieved_at as Date | null,
|
||||
representative_chunk_id: r.representative_chunk_id == null ? null : Number(r.representative_chunk_id),
|
||||
}));
|
||||
}
|
||||
|
||||
// v0.37.0 — corpus-sampling fallback when prefix-stratified can't fill M.
|
||||
// Deterministic with opts.seed (setseed before SELECT); random otherwise.
|
||||
async listCorpusSample(opts: CorpusSampleOpts): Promise<DomainBankRow[]> {
|
||||
const sql = this.sql;
|
||||
if (opts.n <= 0) return [];
|
||||
const exclude = opts.excludeSlugs ?? [];
|
||||
const sourceIds = opts.sourceIds ?? null;
|
||||
const sourceId = opts.sourceId ?? null;
|
||||
// setseed deterministic path: use SELECT setseed($1) + RANDOM(). PGLite/Postgres
|
||||
// both honor setseed for the same session/transaction. For tests this gives
|
||||
// identical ordering across runs.
|
||||
if (typeof opts.seed === 'number') {
|
||||
// Clamp to [-1, 1] required by setseed.
|
||||
const clamped = Math.max(-1, Math.min(1, opts.seed));
|
||||
await sql`SELECT setseed(${clamped}::float8)`;
|
||||
}
|
||||
const rows = await sql`
|
||||
WITH sampled AS (
|
||||
SELECT
|
||||
p.id AS page_id,
|
||||
p.slug,
|
||||
p.source_id,
|
||||
p.title,
|
||||
p.compiled_truth,
|
||||
p.last_retrieved_at,
|
||||
substring(p.slug from '^[^/]+/[^/]+') AS prefix,
|
||||
(SELECT COUNT(*) FROM page_links pl WHERE pl.to_page_id = p.id) AS connection_count
|
||||
FROM pages p
|
||||
WHERE p.deleted_at IS NULL
|
||||
AND (cardinality(${exclude}::text[]) = 0 OR NOT (p.slug = ANY(${exclude}::text[])))
|
||||
AND (
|
||||
(${sourceIds}::text[] IS NOT NULL AND p.source_id = ANY(${sourceIds}::text[]))
|
||||
OR (${sourceIds}::text[] IS NULL AND ${sourceId}::text IS NOT NULL AND p.source_id = ${sourceId})
|
||||
OR (${sourceIds}::text[] IS NULL AND ${sourceId}::text IS NULL)
|
||||
)
|
||||
ORDER BY RANDOM()
|
||||
LIMIT ${opts.n}
|
||||
)
|
||||
SELECT
|
||||
s.*,
|
||||
(
|
||||
SELECT cc.id FROM content_chunks cc
|
||||
WHERE cc.page_id = s.page_id AND cc.embedding IS NOT NULL
|
||||
ORDER BY cc.chunk_index ASC
|
||||
LIMIT 1
|
||||
) AS representative_chunk_id
|
||||
FROM sampled s
|
||||
`;
|
||||
return rows.map((r): DomainBankRow => ({
|
||||
slug: r.slug as string,
|
||||
source_id: r.source_id as string,
|
||||
prefix: r.prefix as string | null,
|
||||
page_id: Number(r.page_id),
|
||||
title: r.title as string | null,
|
||||
compiled_truth: (r.compiled_truth as string | null) ?? '',
|
||||
connection_count: Number(r.connection_count),
|
||||
last_retrieved_at: r.last_retrieved_at as Date | null,
|
||||
representative_chunk_id: r.representative_chunk_id == null ? null : Number(r.representative_chunk_id),
|
||||
}));
|
||||
}
|
||||
|
||||
async resolveSlugs(partial: string): Promise<string[]> {
|
||||
const sql = this.sql;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user