612f0f8396
- fix(file_upload): call storage.upload() in all 3 paths (operation, CLI upload, CLI sync) with rollback semantics (#22 Bug #9) - fix(import): use atomic index counter for parallel queue instead of array.shift() race, preserve checkpoint on errors (#22 Bug #3) - fix(s3): replace unsigned fetch with @aws-sdk/client-s3 for proper SigV4 auth, supports R2/MinIO via forcePathStyle (#22 Bug #10) - fix(redirect): verify remote file exists before deleting local copy, skip files not found in storage (#22 Bug #5) - deps: add @aws-sdk/client-s3 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
486 lines
17 KiB
TypeScript
486 lines
17 KiB
TypeScript
import { readFileSync, readdirSync, statSync, existsSync, writeFileSync, unlinkSync } from 'fs';
|
|
import { join, relative, extname, basename } from 'path';
|
|
import { createHash } from 'crypto';
|
|
import type { BrainEngine } from '../core/engine.ts';
|
|
import * as db from '../core/db.ts';
|
|
|
|
interface FileRecord {
|
|
id: number;
|
|
page_slug: string | null;
|
|
filename: string;
|
|
storage_path: string;
|
|
mime_type: string | null;
|
|
size_bytes: number;
|
|
content_hash: string;
|
|
metadata: Record<string, unknown>;
|
|
created_at: string;
|
|
}
|
|
|
|
const MIME_TYPES: Record<string, string> = {
|
|
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
|
|
'.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml',
|
|
'.pdf': 'application/pdf', '.mp4': 'video/mp4', '.m4a': 'audio/mp4',
|
|
'.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.heic': 'image/heic',
|
|
'.tiff': 'image/tiff', '.tif': 'image/tiff', '.dng': 'image/x-adobe-dng',
|
|
'.doc': 'application/msword', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
'.xls': 'application/vnd.ms-excel', '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
};
|
|
|
|
function getMimeType(filePath: string): string | null {
|
|
const ext = extname(filePath).toLowerCase();
|
|
return MIME_TYPES[ext] || null;
|
|
}
|
|
|
|
function fileHash(filePath: string): string {
|
|
const content = readFileSync(filePath);
|
|
return createHash('sha256').update(content).digest('hex');
|
|
}
|
|
|
|
export async function runFiles(engine: BrainEngine, args: string[]) {
|
|
const subcommand = args[0];
|
|
|
|
switch (subcommand) {
|
|
case 'list':
|
|
await listFiles(args[1]);
|
|
break;
|
|
case 'upload':
|
|
await uploadFile(args.slice(1));
|
|
break;
|
|
case 'sync':
|
|
await syncFiles(args[1]);
|
|
break;
|
|
case 'verify':
|
|
await verifyFiles();
|
|
break;
|
|
case 'mirror':
|
|
await mirrorFiles(args.slice(1));
|
|
break;
|
|
case 'unmirror':
|
|
await unmirrorFiles(args.slice(1));
|
|
break;
|
|
case 'redirect':
|
|
await redirectFiles(args.slice(1));
|
|
break;
|
|
case 'restore':
|
|
await restoreFiles(args.slice(1));
|
|
break;
|
|
case 'clean':
|
|
await cleanFiles(args.slice(1));
|
|
break;
|
|
case 'status':
|
|
await filesStatus(args.slice(1));
|
|
break;
|
|
default:
|
|
console.error(`Usage: gbrain files <list|upload|sync|verify|mirror|unmirror|redirect|restore|clean|status> [args]`);
|
|
console.error(` list [slug] List files for a page (or all)`);
|
|
console.error(` upload <file> --page <slug> Upload file linked to page`);
|
|
console.error(` sync <dir> Upload directory to storage`);
|
|
console.error(` verify Verify all uploads match local`);
|
|
console.error(` mirror <dir> [--dry-run] Mirror files to cloud storage`);
|
|
console.error(` unmirror <dir> Remove mirror marker (files stay in storage)`);
|
|
console.error(` redirect <dir> [--dry-run] Replace files with .redirect breadcrumbs`);
|
|
console.error(` restore <dir> Download from storage, recreate local files`);
|
|
console.error(` clean <dir> [--yes] Delete .redirect breadcrumbs (irreversible)`);
|
|
console.error(` status Show migration status of directories`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
async function listFiles(slug?: string) {
|
|
const sql = db.getConnection();
|
|
let rows;
|
|
if (slug) {
|
|
rows = await sql`SELECT * FROM files WHERE page_slug = ${slug} ORDER BY filename`;
|
|
} else {
|
|
rows = await sql`SELECT * FROM files ORDER BY page_slug, filename LIMIT 100`;
|
|
}
|
|
|
|
if (rows.length === 0) {
|
|
console.log(slug ? `No files for page: ${slug}` : 'No files stored.');
|
|
return;
|
|
}
|
|
|
|
console.log(`${rows.length} file(s):`);
|
|
for (const row of rows) {
|
|
const size = row.size_bytes ? `${Math.round(row.size_bytes / 1024)}KB` : '?';
|
|
console.log(` ${row.page_slug || '(unlinked)'} / ${row.filename} [${size}, ${row.mime_type || '?'}]`);
|
|
}
|
|
}
|
|
|
|
async function uploadFile(args: string[]) {
|
|
const filePath = args.find(a => !a.startsWith('--'));
|
|
const pageSlug = args.find((a, i) => args[i - 1] === '--page') || null;
|
|
|
|
if (!filePath || !existsSync(filePath)) {
|
|
console.error('Usage: gbrain files upload <file> --page <slug>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const stat = statSync(filePath);
|
|
const hash = fileHash(filePath);
|
|
const filename = basename(filePath);
|
|
const storagePath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
|
|
const mimeType = getMimeType(filePath);
|
|
|
|
const sql = db.getConnection();
|
|
|
|
// Check for existing file by hash
|
|
const existing = await sql`SELECT id FROM files WHERE content_hash = ${hash} AND storage_path = ${storagePath}`;
|
|
if (existing.length > 0) {
|
|
console.log(`File already uploaded (hash match): ${storagePath}`);
|
|
return;
|
|
}
|
|
|
|
// Upload to storage backend if configured
|
|
const { loadConfig } = await import('../core/config.ts');
|
|
const config = loadConfig();
|
|
if (config?.storage) {
|
|
const { createStorage } = await import('../core/storage.ts');
|
|
const storage = await createStorage(config.storage as any);
|
|
const content = readFileSync(filePath);
|
|
await storage.upload(storagePath, content, mimeType || undefined);
|
|
}
|
|
|
|
await sql`
|
|
INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
|
VALUES (${pageSlug}, ${filename}, ${storagePath}, ${mimeType}, ${stat.size}, ${hash}, ${'{}'}::jsonb)
|
|
ON CONFLICT (storage_path) DO UPDATE SET
|
|
content_hash = EXCLUDED.content_hash,
|
|
size_bytes = EXCLUDED.size_bytes,
|
|
mime_type = EXCLUDED.mime_type
|
|
`;
|
|
|
|
console.log(`Uploaded: ${storagePath} (${Math.round(stat.size / 1024)}KB)`);
|
|
}
|
|
|
|
async function syncFiles(dir?: string) {
|
|
if (!dir || !existsSync(dir)) {
|
|
console.error('Usage: gbrain files sync <directory>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const files = collectFiles(dir);
|
|
console.log(`Found ${files.length} files to sync`);
|
|
|
|
let uploaded = 0;
|
|
let skipped = 0;
|
|
|
|
for (let i = 0; i < files.length; i++) {
|
|
const filePath = files[i];
|
|
const relativePath = relative(dir, filePath);
|
|
|
|
if ((i + 1) % 50 === 0 || i === files.length - 1) {
|
|
process.stdout.write(`\r ${i + 1}/${files.length} processed, ${uploaded} uploaded, ${skipped} skipped`);
|
|
}
|
|
|
|
const hash = fileHash(filePath);
|
|
const filename = basename(filePath);
|
|
const storagePath = relativePath.replace(/\\/g, '/');
|
|
const mimeType = getMimeType(filePath);
|
|
const stat = statSync(filePath);
|
|
|
|
const sql = db.getConnection();
|
|
const existing = await sql`SELECT id FROM files WHERE content_hash = ${hash} AND storage_path = ${storagePath}`;
|
|
if (existing.length > 0) {
|
|
skipped++;
|
|
continue;
|
|
}
|
|
|
|
// Infer page slug from directory structure
|
|
const pathParts = relativePath.split('/');
|
|
const pageSlug = pathParts.length > 1 ? pathParts.slice(0, -1).join('/') : null;
|
|
|
|
await sql`
|
|
INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
|
|
VALUES (${pageSlug}, ${filename}, ${storagePath}, ${mimeType}, ${stat.size}, ${hash}, ${'{}'}::jsonb)
|
|
ON CONFLICT (storage_path) DO UPDATE SET
|
|
content_hash = EXCLUDED.content_hash,
|
|
size_bytes = EXCLUDED.size_bytes,
|
|
mime_type = EXCLUDED.mime_type
|
|
`;
|
|
|
|
uploaded++;
|
|
}
|
|
|
|
console.log(`\n\nFiles sync complete: ${uploaded} uploaded, ${skipped} skipped (unchanged)`);
|
|
}
|
|
|
|
async function verifyFiles() {
|
|
const sql = db.getConnection();
|
|
const rows = await sql`SELECT * FROM files ORDER BY storage_path`;
|
|
|
|
if (rows.length === 0) {
|
|
console.log('No files to verify.');
|
|
return;
|
|
}
|
|
|
|
let verified = 0;
|
|
let mismatches = 0;
|
|
let missing = 0;
|
|
|
|
for (const row of rows) {
|
|
// Note: full verification would check Supabase Storage hash
|
|
// For now, verify the DB record exists and has valid data
|
|
if (!row.content_hash || !row.storage_path) {
|
|
mismatches++;
|
|
console.error(` MISMATCH: ${row.storage_path} (missing hash or path)`);
|
|
} else {
|
|
verified++;
|
|
}
|
|
}
|
|
|
|
if (mismatches === 0 && missing === 0) {
|
|
console.log(`${verified} files verified, 0 mismatches, 0 missing`);
|
|
} else {
|
|
console.error(`VERIFY FAILED: ${mismatches} mismatches, ${missing} missing.`);
|
|
console.error(`Run: gbrain files sync --retry-failed`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
// ─────────────────────────────────────────────────────────────────
|
|
// File Migration Commands (mirror → redirect → clean lifecycle)
|
|
// ─────────────────────────────────────────────────────────────────
|
|
|
|
async function mirrorFiles(args: string[]) {
|
|
const dir = args.find(a => !a.startsWith('--'));
|
|
const dryRun = args.includes('--dry-run');
|
|
if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files mirror <dir> [--dry-run]'); process.exit(1); }
|
|
|
|
const { createStorage } = await import('../core/storage.ts');
|
|
const { loadConfig } = await import('../core/config.ts');
|
|
const { stringify } = await import('../core/yaml-lite.ts');
|
|
const config = loadConfig();
|
|
if (!config?.storage) { console.error('No storage backend configured. Run gbrain init with storage settings.'); process.exit(1); }
|
|
|
|
const storage = await createStorage(config.storage as any);
|
|
const files = collectFiles(dir);
|
|
console.log(`Found ${files.length} files to mirror`);
|
|
|
|
if (dryRun) {
|
|
for (const f of files) { console.log(` Would upload: ${relative(dir, f)}`); }
|
|
console.log(`\nDry run: ${files.length} files would be uploaded.`);
|
|
return;
|
|
}
|
|
|
|
let uploaded = 0;
|
|
for (const filePath of files) {
|
|
const relPath = relative(dir, filePath);
|
|
const data = readFileSync(filePath);
|
|
const mime = getMimeType(filePath);
|
|
await storage.upload(relPath, data, mime || undefined);
|
|
uploaded++;
|
|
}
|
|
|
|
// Write .supabase marker
|
|
const marker = stringify({
|
|
synced_at: new Date().toISOString(),
|
|
bucket: config.storage.bucket || 'brain-files',
|
|
prefix: basename(dir) + '/',
|
|
file_count: uploaded,
|
|
});
|
|
writeFileSync(join(dir, '.supabase'), marker);
|
|
|
|
console.log(`Mirrored ${uploaded} files. Marker written to ${dir}/.supabase`);
|
|
}
|
|
|
|
async function unmirrorFiles(args: string[]) {
|
|
const dir = args.find(a => !a.startsWith('--'));
|
|
if (!dir) { console.error('Usage: gbrain files unmirror <dir>'); process.exit(1); }
|
|
|
|
const markerPath = join(dir, '.supabase');
|
|
if (existsSync(markerPath)) {
|
|
unlinkSync(markerPath);
|
|
console.log(`Removed mirror marker from ${dir}. Files remain in storage.`);
|
|
} else {
|
|
console.log(`No mirror marker found in ${dir}. Nothing to do.`);
|
|
}
|
|
}
|
|
|
|
async function redirectFiles(args: string[]) {
|
|
const dir = args.find(a => !a.startsWith('--'));
|
|
const dryRun = args.includes('--dry-run');
|
|
if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files redirect <dir> [--dry-run]'); process.exit(1); }
|
|
|
|
const markerPath = join(dir, '.supabase');
|
|
if (!existsSync(markerPath)) {
|
|
console.error('Directory must be mirrored first. Run: gbrain files mirror <dir>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const { parse: parseYaml, stringify } = await import('../core/yaml-lite.ts');
|
|
const marker = parseYaml(readFileSync(markerPath, 'utf-8'));
|
|
const files = collectFiles(dir);
|
|
|
|
if (dryRun) {
|
|
for (const f of files) { console.log(` Would redirect: ${relative(dir, f)}`); }
|
|
console.log(`\nDry run: ${files.length} files would be redirected.`);
|
|
return;
|
|
}
|
|
|
|
// Verify remote files exist before deleting locals
|
|
const { loadConfig } = await import('../core/config.ts');
|
|
const config = loadConfig();
|
|
let storage: any = null;
|
|
if (config?.storage) {
|
|
const { createStorage } = await import('../core/storage.ts');
|
|
storage = await createStorage(config.storage as any);
|
|
}
|
|
|
|
let redirected = 0;
|
|
let skippedMissing = 0;
|
|
for (const filePath of files) {
|
|
const relPath = relative(dir, filePath);
|
|
const hash = fileHash(filePath);
|
|
|
|
// Verify remote exists before deleting local
|
|
if (storage) {
|
|
const remoteExists = await storage.exists(relPath);
|
|
if (!remoteExists) {
|
|
console.error(` Skipping ${relPath}: not found in remote storage (would lose data)`);
|
|
skippedMissing++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const breadcrumb = stringify({
|
|
moved_to: 'storage',
|
|
bucket: marker.bucket || 'brain-files',
|
|
path: relPath,
|
|
moved_at: new Date().toISOString().split('T')[0],
|
|
original_hash: `sha256:${hash}`,
|
|
});
|
|
writeFileSync(filePath + '.redirect', breadcrumb);
|
|
unlinkSync(filePath);
|
|
redirected++;
|
|
}
|
|
|
|
console.log(`Redirected ${redirected} files. Originals removed, breadcrumbs created.`);
|
|
if (skippedMissing > 0) {
|
|
console.log(`Skipped ${skippedMissing} files (not found in remote storage — run 'gbrain files mirror' first).`);
|
|
}
|
|
console.log('To undo: gbrain files restore <dir>');
|
|
}
|
|
|
|
async function restoreFiles(args: string[]) {
|
|
const dir = args.find(a => !a.startsWith('--'));
|
|
if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files restore <dir>'); process.exit(1); }
|
|
|
|
const { createStorage } = await import('../core/storage.ts');
|
|
const { loadConfig } = await import('../core/config.ts');
|
|
const { parse: parseYaml } = await import('../core/yaml-lite.ts');
|
|
const config = loadConfig();
|
|
if (!config?.storage) { console.error('No storage backend configured.'); process.exit(1); }
|
|
|
|
const storage = await createStorage(config.storage as any);
|
|
const redirectFiles: string[] = [];
|
|
|
|
function findRedirects(d: string) {
|
|
for (const entry of readdirSync(d)) {
|
|
if (entry.startsWith('.')) continue;
|
|
const full = join(d, entry);
|
|
if (statSync(full).isDirectory()) findRedirects(full);
|
|
else if (entry.endsWith('.redirect')) redirectFiles.push(full);
|
|
}
|
|
}
|
|
findRedirects(dir);
|
|
|
|
let restored = 0;
|
|
let failed = 0;
|
|
for (const redirectPath of redirectFiles) {
|
|
const info = parseYaml(readFileSync(redirectPath, 'utf-8'));
|
|
const originalPath = redirectPath.replace(/\.redirect$/, '');
|
|
try {
|
|
const data = await storage.download(info.path);
|
|
writeFileSync(originalPath, data);
|
|
unlinkSync(redirectPath);
|
|
restored++;
|
|
} catch (e: unknown) {
|
|
const msg = e instanceof Error ? e.message : String(e);
|
|
console.error(` Failed to restore ${info.path}: ${msg}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
console.log(`Restored ${restored} files. ${failed > 0 ? `${failed} failed.` : ''}`);
|
|
}
|
|
|
|
async function cleanFiles(args: string[]) {
|
|
const dir = args.find(a => !a.startsWith('--'));
|
|
const confirmed = args.includes('--yes');
|
|
if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files clean <dir> [--yes]'); process.exit(1); }
|
|
|
|
if (!confirmed) {
|
|
console.error('WARNING: This permanently removes .redirect breadcrumbs.');
|
|
console.error('After this, files are only accessible from cloud storage.');
|
|
console.error('Git history still has the originals if you need them.');
|
|
console.error('Run with --yes to confirm.');
|
|
process.exit(1);
|
|
}
|
|
|
|
let cleaned = 0;
|
|
function findAndClean(d: string) {
|
|
for (const entry of readdirSync(d)) {
|
|
if (entry.startsWith('.')) continue;
|
|
const full = join(d, entry);
|
|
if (statSync(full).isDirectory()) findAndClean(full);
|
|
else if (entry.endsWith('.redirect')) { unlinkSync(full); cleaned++; }
|
|
}
|
|
}
|
|
findAndClean(dir);
|
|
|
|
console.log(`Cleaned ${cleaned} redirect breadcrumbs. Cloud storage is now the only source.`);
|
|
}
|
|
|
|
async function filesStatus(args: string[]) {
|
|
const dir = args[0] || '.';
|
|
|
|
let mirrored = 0, redirected = 0, local = 0;
|
|
|
|
function scan(d: string) {
|
|
for (const entry of readdirSync(d)) {
|
|
if (entry.startsWith('.') && entry !== '.supabase') continue;
|
|
const full = join(d, entry);
|
|
if (entry === '.supabase') { mirrored++; continue; }
|
|
if (statSync(full).isDirectory()) scan(full);
|
|
else if (entry.endsWith('.redirect')) redirected++;
|
|
else if (!entry.endsWith('.md')) local++;
|
|
}
|
|
}
|
|
scan(dir);
|
|
|
|
console.log('File migration status:');
|
|
console.log(` Mirrored directories: ${mirrored}`);
|
|
console.log(` Redirected files: ${redirected}`);
|
|
console.log(` Local binary files: ${local}`);
|
|
|
|
if (mirrored === 0 && redirected === 0 && local > 0) {
|
|
console.log(`\n${local} local files. Run: gbrain files mirror <dir> to start migration.`);
|
|
} else if (redirected > 0) {
|
|
console.log(`\n${redirected} files redirected to storage. Run: gbrain files clean <dir> --yes to remove breadcrumbs.`);
|
|
}
|
|
}
|
|
|
|
function collectFiles(dir: string): string[] {
|
|
const files: string[] = [];
|
|
|
|
function walk(d: string) {
|
|
for (const entry of readdirSync(d)) {
|
|
if (entry.startsWith('.')) continue;
|
|
|
|
const full = join(d, entry);
|
|
const stat = statSync(full);
|
|
|
|
if (stat.isDirectory()) {
|
|
walk(full);
|
|
} else if (!entry.endsWith('.md')) {
|
|
// Non-markdown files are candidates for storage
|
|
files.push(full);
|
|
}
|
|
}
|
|
}
|
|
|
|
walk(dir);
|
|
return files.sort();
|
|
}
|