fix: statement_timeout scoped to search, upload-raw writes pointer, publish inlines marked.js

1. statement_timeout: 8s moved from global connection config to
   searchKeyword/searchVector only. Prevents DoS on search without
   killing embed --all or bulk imports that need longer than 8s.

2. upload-raw now writes the .redirect.yaml pointer file to disk
   (was creating the pointer object but never calling writeFileSync).

3. publish inlines marked.js from node_modules instead of loading
   from cdn.jsdelivr.net. Generated HTML is now truly self-contained
   with no external dependencies.

4. v0.9.1 migration doc updated with slug authority breaking change
   warning for brains that use frontmatter slug: overrides.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-04-12 13:23:55 -10:00
parent fa62e61994
commit 004ac6c66f
8 changed files with 86 additions and 52 deletions

View File

@@ -240,9 +240,10 @@ async function uploadRaw(args: string[]) {
uploaded: new Date().toISOString(),
...(fileType ? { type: fileType } : {}),
});
// Write pointer next to the page that references it
pointerPath = `${pageSlug}/${filename}.redirect.yaml`;
console.error(`Pointer: ${pointerPath}`);
// Write pointer next to the original file
pointerPath = filePath + '.redirect.yaml';
writeFileSync(pointerPath, pointer);
console.error(`Pointer written: ${pointerPath}`);
}
// Record in DB

View File

@@ -14,7 +14,12 @@
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
import { dirname, basename } from 'path';
import { dirname, basename, join } from 'path';
import { createRequire } from 'module';
// Inline marked.js so published HTML is truly self-contained (no CDN dependency)
const require = createRequire(import.meta.url);
const MARKED_JS = readFileSync(join(dirname(require.resolve('marked')), 'marked.umd.js'), 'utf8');
// ── Content stripping ──────────────────────────────────────────────
@@ -314,7 +319,7 @@ export function generateHtml({ title, markdown, encrypted }: GenerateHtmlOptions
${passwordHtml}
<div id="content"></div>
${encryptedVars}
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"><\/script>
<script>${MARKED_JS}<\/script>
${contentScript}
</body>
</html>`;

View File

@@ -39,7 +39,6 @@ export async function connect(config: EngineConfig): Promise<void> {
max: 10,
idle_timeout: 20,
connect_timeout: 10,
connection: { statement_timeout: '8s' },
types: {
// Register pgvector type
bigint: postgres.BigInt,

View File

@@ -188,34 +188,40 @@ export class PostgresEngine implements BrainEngine {
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
}
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
const rows = await sql`
WITH ranked_pages AS (
SELECT p.id, p.slug, p.title, p.type,
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
FROM pages p
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
// Search-only timeout: prevents DoS via expensive queries without
// affecting long-running operations like embed --all or bulk import
await sql`SET statement_timeout = '8s'`;
try {
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
const rows = await sql`
WITH ranked_pages AS (
SELECT p.id, p.slug, p.title, p.type,
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
FROM pages p
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY score DESC
LIMIT ${limit}
OFFSET ${offset}
),
best_chunks AS (
SELECT DISTINCT ON (rp.slug)
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
cc.chunk_text, cc.chunk_source
FROM ranked_pages rp
JOIN content_chunks cc ON cc.page_id = rp.id
ORDER BY rp.slug, cc.chunk_index
)
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
false AS stale
FROM best_chunks
ORDER BY score DESC
LIMIT ${limit}
OFFSET ${offset}
),
best_chunks AS (
SELECT DISTINCT ON (rp.slug)
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
cc.chunk_text, cc.chunk_source
FROM ranked_pages rp
JOIN content_chunks cc ON cc.page_id = rp.id
ORDER BY rp.slug, cc.chunk_index
)
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
false AS stale
FROM best_chunks
ORDER BY score DESC
`;
return rows.map(rowToSearchResult);
`;
return rows.map(rowToSearchResult);
} finally {
await sql`SET statement_timeout = '0'`;
}
}
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
@@ -231,23 +237,28 @@ export class PostgresEngine implements BrainEngine {
const vecStr = '[' + Array.from(embedding).join(',') + ']';
const rows = await sql`
SELECT
p.slug, p.id as page_id, p.title, p.type,
cc.chunk_text, cc.chunk_source,
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
false AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE cc.embedding IS NOT NULL
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY cc.embedding <=> ${vecStr}::vector
LIMIT ${limit}
OFFSET ${offset}
`;
return rows.map(rowToSearchResult);
// Search-only timeout (see searchKeyword for rationale)
await sql`SET statement_timeout = '8s'`;
try {
const rows = await sql`
SELECT
p.slug, p.id as page_id, p.title, p.type,
cc.chunk_text, cc.chunk_source,
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
false AS stale
FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE cc.embedding IS NOT NULL
${type ? sql`AND p.type = ${type}` : sql``}
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
ORDER BY cc.embedding <=> ${vecStr}::vector
LIMIT ${limit}
OFFSET ${offset}
`;
return rows.map(rowToSearchResult);
} finally {
await sql`SET statement_timeout = '0'`;
}
}
// Chunks