fix: statement_timeout scoped to search, upload-raw writes pointer, publish inlines marked.js
1. statement_timeout: 8s moved from global connection config to searchKeyword/searchVector only. Prevents DoS on search without killing embed --all or bulk imports that need longer than 8s. 2. upload-raw now writes the .redirect.yaml pointer file to disk (was creating the pointer object but never calling writeFileSync). 3. publish inlines marked.js from node_modules instead of loading from cdn.jsdelivr.net. Generated HTML is now truly self-contained with no external dependencies. 4. v0.9.1 migration doc updated with slug authority breaking change warning for brains that use frontmatter slug: overrides. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -240,9 +240,10 @@ async function uploadRaw(args: string[]) {
|
||||
uploaded: new Date().toISOString(),
|
||||
...(fileType ? { type: fileType } : {}),
|
||||
});
|
||||
// Write pointer next to the page that references it
|
||||
pointerPath = `${pageSlug}/${filename}.redirect.yaml`;
|
||||
console.error(`Pointer: ${pointerPath}`);
|
||||
// Write pointer next to the original file
|
||||
pointerPath = filePath + '.redirect.yaml';
|
||||
writeFileSync(pointerPath, pointer);
|
||||
console.error(`Pointer written: ${pointerPath}`);
|
||||
}
|
||||
|
||||
// Record in DB
|
||||
|
||||
@@ -14,7 +14,12 @@
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync } from 'fs';
|
||||
import { randomBytes, createCipheriv, pbkdf2Sync } from 'crypto';
|
||||
import { dirname, basename } from 'path';
|
||||
import { dirname, basename, join } from 'path';
|
||||
import { createRequire } from 'module';
|
||||
|
||||
// Inline marked.js so published HTML is truly self-contained (no CDN dependency)
|
||||
const require = createRequire(import.meta.url);
|
||||
const MARKED_JS = readFileSync(join(dirname(require.resolve('marked')), 'marked.umd.js'), 'utf8');
|
||||
|
||||
// ── Content stripping ──────────────────────────────────────────────
|
||||
|
||||
@@ -314,7 +319,7 @@ export function generateHtml({ title, markdown, encrypted }: GenerateHtmlOptions
|
||||
${passwordHtml}
|
||||
<div id="content"></div>
|
||||
${encryptedVars}
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"><\/script>
|
||||
<script>${MARKED_JS}<\/script>
|
||||
${contentScript}
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
@@ -39,7 +39,6 @@ export async function connect(config: EngineConfig): Promise<void> {
|
||||
max: 10,
|
||||
idle_timeout: 20,
|
||||
connect_timeout: 10,
|
||||
connection: { statement_timeout: '8s' },
|
||||
types: {
|
||||
// Register pgvector type
|
||||
bigint: postgres.BigInt,
|
||||
|
||||
@@ -188,34 +188,40 @@ export class PostgresEngine implements BrainEngine {
|
||||
console.warn(`[gbrain] Warning: search limit clamped from ${opts.limit} to ${MAX_SEARCH_LIMIT}`);
|
||||
}
|
||||
|
||||
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
||||
const rows = await sql`
|
||||
WITH ranked_pages AS (
|
||||
SELECT p.id, p.slug, p.title, p.type,
|
||||
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
||||
FROM pages p
|
||||
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
// Search-only timeout: prevents DoS via expensive queries without
|
||||
// affecting long-running operations like embed --all or bulk import
|
||||
await sql`SET statement_timeout = '8s'`;
|
||||
try {
|
||||
// CTE: rank pages by FTS score, then pick the best chunk per page in SQL
|
||||
const rows = await sql`
|
||||
WITH ranked_pages AS (
|
||||
SELECT p.id, p.slug, p.title, p.type,
|
||||
ts_rank(p.search_vector, websearch_to_tsquery('english', ${query})) AS score
|
||||
FROM pages p
|
||||
WHERE p.search_vector @@ websearch_to_tsquery('english', ${query})
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
),
|
||||
best_chunks AS (
|
||||
SELECT DISTINCT ON (rp.slug)
|
||||
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
||||
cc.chunk_text, cc.chunk_source
|
||||
FROM ranked_pages rp
|
||||
JOIN content_chunks cc ON cc.page_id = rp.id
|
||||
ORDER BY rp.slug, cc.chunk_index
|
||||
)
|
||||
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
||||
false AS stale
|
||||
FROM best_chunks
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
),
|
||||
best_chunks AS (
|
||||
SELECT DISTINCT ON (rp.slug)
|
||||
rp.slug, rp.id as page_id, rp.title, rp.type, rp.score,
|
||||
cc.chunk_text, cc.chunk_source
|
||||
FROM ranked_pages rp
|
||||
JOIN content_chunks cc ON cc.page_id = rp.id
|
||||
ORDER BY rp.slug, cc.chunk_index
|
||||
)
|
||||
SELECT slug, page_id, title, type, chunk_text, chunk_source, score,
|
||||
false AS stale
|
||||
FROM best_chunks
|
||||
ORDER BY score DESC
|
||||
`;
|
||||
|
||||
return rows.map(rowToSearchResult);
|
||||
`;
|
||||
return rows.map(rowToSearchResult);
|
||||
} finally {
|
||||
await sql`SET statement_timeout = '0'`;
|
||||
}
|
||||
}
|
||||
|
||||
async searchVector(embedding: Float32Array, opts?: SearchOpts): Promise<SearchResult[]> {
|
||||
@@ -231,23 +237,28 @@ export class PostgresEngine implements BrainEngine {
|
||||
|
||||
const vecStr = '[' + Array.from(embedding).join(',') + ']';
|
||||
|
||||
const rows = await sql`
|
||||
SELECT
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
||||
false AS stale
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NOT NULL
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY cc.embedding <=> ${vecStr}::vector
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
`;
|
||||
|
||||
return rows.map(rowToSearchResult);
|
||||
// Search-only timeout (see searchKeyword for rationale)
|
||||
await sql`SET statement_timeout = '8s'`;
|
||||
try {
|
||||
const rows = await sql`
|
||||
SELECT
|
||||
p.slug, p.id as page_id, p.title, p.type,
|
||||
cc.chunk_text, cc.chunk_source,
|
||||
1 - (cc.embedding <=> ${vecStr}::vector) AS score,
|
||||
false AS stale
|
||||
FROM content_chunks cc
|
||||
JOIN pages p ON p.id = cc.page_id
|
||||
WHERE cc.embedding IS NOT NULL
|
||||
${type ? sql`AND p.type = ${type}` : sql``}
|
||||
${excludeSlugs?.length ? sql`AND p.slug != ALL(${excludeSlugs})` : sql``}
|
||||
ORDER BY cc.embedding <=> ${vecStr}::vector
|
||||
LIMIT ${limit}
|
||||
OFFSET ${offset}
|
||||
`;
|
||||
return rows.map(rowToSearchResult);
|
||||
} finally {
|
||||
await sql`SET statement_timeout = '0'`;
|
||||
}
|
||||
}
|
||||
|
||||
// Chunks
|
||||
|
||||
Reference in New Issue
Block a user