diff --git a/src/commands/files.ts b/src/commands/files.ts index dc20857..445bbcd 100644 --- a/src/commands/files.ts +++ b/src/commands/files.ts @@ -1,8 +1,12 @@ -import { readFileSync, readdirSync, statSync, existsSync, writeFileSync, unlinkSync } from 'fs'; -import { join, relative, extname, basename } from 'path'; +import { readFileSync, readdirSync, statSync, existsSync, writeFileSync, unlinkSync, mkdirSync } from 'fs'; +import { join, relative, extname, basename, dirname } from 'path'; import { createHash } from 'crypto'; import type { BrainEngine } from '../core/engine.ts'; import * as db from '../core/db.ts'; +import { humanSize } from '../core/file-resolver.ts'; + +/** Size threshold: files >= 100 MB use TUS resumable upload */ +const SIZE_THRESHOLD = 100 * 1024 * 1024; interface FileRecord { id: number; @@ -67,20 +71,28 @@ export async function runFiles(engine: BrainEngine, args: string[]) { case 'clean': await cleanFiles(args.slice(1)); break; + case 'upload-raw': + await uploadRaw(args.slice(1)); + break; + case 'signed-url': + await signedUrl(args.slice(1)); + break; case 'status': await filesStatus(args.slice(1)); break; default: - console.error(`Usage: gbrain files [args]`); + console.error(`Usage: gbrain files [args]`); console.error(` list [slug] List files for a page (or all)`); console.error(` upload --page Upload file linked to page`); + console.error(` upload-raw --page [--type ] Smart upload with .redirect.yaml pointer`); + console.error(` signed-url Generate signed URL for stored file`); console.error(` sync Upload directory to storage`); console.error(` verify Verify all uploads match local`); console.error(` mirror [--dry-run] Mirror files to cloud storage`); console.error(` unmirror Remove mirror marker (files stay in storage)`); - console.error(` redirect [--dry-run] Replace files with .redirect breadcrumbs`); + console.error(` redirect [--dry-run] Replace files with .redirect.yaml pointers`); console.error(` restore Download from storage, recreate local files`); - console.error(` clean [--yes] Delete .redirect breadcrumbs (irreversible)`); + console.error(` clean [--yes] Delete redirect pointers (irreversible)`); console.error(` status Show migration status of directories`); process.exit(1); } @@ -138,6 +150,8 @@ async function uploadFile(args: string[]) { const { createStorage } = await import('../core/storage.ts'); const storage = await createStorage(config.storage as any); const content = readFileSync(filePath); + const method = content.length >= SIZE_THRESHOLD ? 'TUS resumable' : 'standard'; + console.log(`Uploading ${humanSize(stat.size)} via ${method}...`); await storage.upload(storagePath, content, mimeType || undefined); } @@ -150,7 +164,135 @@ async function uploadFile(args: string[]) { mime_type = EXCLUDED.mime_type `; - console.log(`Uploaded: ${storagePath} (${Math.round(stat.size / 1024)}KB)`); + console.log(`Uploaded: ${storagePath} (${humanSize(stat.size)})`); +} + +/** + * Smart upload with size routing and .redirect.yaml pointer creation. + * + * Size routing: + * < 100 MB text/PDF → stays in git (brain repo), no cloud upload + * >= 100 MB OR media → upload to cloud storage, create .redirect.yaml pointer + * + * The .redirect.yaml pointer stays in the brain repo so git tracks what was stored. + */ +async function uploadRaw(args: string[]) { + const filePath = args.find(a => !a.startsWith('--')); + const pageSlug = args.find((a, i) => args[i - 1] === '--page') || null; + const fileType = args.find((a, i) => args[i - 1] === '--type') || null; + const noPointer = args.includes('--no-pointer'); + + if (!filePath || !existsSync(filePath)) { + console.error('Usage: gbrain files upload-raw --page [--type ] [--no-pointer]'); + process.exit(1); + } + + const stat = statSync(filePath); + const filename = basename(filePath); + const mimeType = getMimeType(filePath); + const isMedia = mimeType?.startsWith('video/') || mimeType?.startsWith('audio/') || mimeType?.startsWith('image/'); + const needsCloud = stat.size >= SIZE_THRESHOLD || isMedia; + + if (!needsCloud) { + // Small text/PDF files stay in git + console.log(JSON.stringify({ + success: true, + storage: 'git', + path: filePath, + size: stat.size, + size_human: humanSize(stat.size), + })); + return; + } + + // Upload to cloud storage + const { loadConfig } = await import('../core/config.ts'); + const config = loadConfig(); + if (!config?.storage) { + console.error('No storage backend configured. Run gbrain init with storage settings.'); + console.error('Or use gbrain files upload for manual uploads.'); + process.exit(1); + } + + const { createStorage } = await import('../core/storage.ts'); + const storage = await createStorage(config.storage as any); + const content = readFileSync(filePath); + const hash = createHash('sha256').update(content).digest('hex'); + const storagePath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`; + const bucket = (config.storage as any).bucket || 'brain-files'; + + const method = content.length >= SIZE_THRESHOLD ? 'TUS resumable' : 'standard'; + console.error(`Uploading ${humanSize(stat.size)} via ${method}...`); + await storage.upload(storagePath, content, mimeType || undefined); + + // Create .redirect.yaml pointer in the brain repo + let pointerPath: string | null = null; + if (!noPointer && pageSlug) { + const { stringify } = await import('../core/yaml-lite.ts'); + const pointer = stringify({ + target: `supabase://${bucket}/${storagePath}`, + bucket, + storage_path: storagePath, + size: stat.size, + size_human: humanSize(stat.size), + hash: `sha256:${hash}`, + mime: mimeType || 'application/octet-stream', + uploaded: new Date().toISOString(), + ...(fileType ? { type: fileType } : {}), + }); + // Write pointer next to the page that references it + pointerPath = `${pageSlug}/${filename}.redirect.yaml`; + console.error(`Pointer: ${pointerPath}`); + // Note: the caller is responsible for writing the pointer file + // to the brain repo at the correct location + } + + // Record in DB + const sql = db.getConnection(); + await sql` + INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata) + VALUES (${pageSlug}, ${filename}, ${storagePath}, ${mimeType}, ${stat.size}, ${'sha256:' + hash}, + ${JSON.stringify({ type: fileType, upload_method: method })}::jsonb) + ON CONFLICT (storage_path) DO UPDATE SET + content_hash = EXCLUDED.content_hash, + size_bytes = EXCLUDED.size_bytes, + mime_type = EXCLUDED.mime_type + `; + + // Output JSON for scripting + console.log(JSON.stringify({ + success: true, + storage: 'supabase', + storagePath, + bucket, + reference: `supabase://${bucket}/${storagePath}`, + pointerPath, + size: stat.size, + size_human: humanSize(stat.size), + hash: `sha256:${hash}`, + upload_method: method, + })); +} + +/** Generate a signed URL for a stored file */ +async function signedUrl(args: string[]) { + const storagePath = args.find(a => !a.startsWith('--')); + if (!storagePath) { + console.error('Usage: gbrain files signed-url '); + process.exit(1); + } + + const { loadConfig } = await import('../core/config.ts'); + const config = loadConfig(); + if (!config?.storage) { + console.error('No storage backend configured.'); + process.exit(1); + } + + const { createStorage } = await import('../core/storage.ts'); + const storage = await createStorage(config.storage as any); + const url = await storage.getUrl(storagePath); + console.log(url); } async function syncFiles(dir?: string) { @@ -343,14 +485,20 @@ async function redirectFiles(args: string[]) { } } - const breadcrumb = stringify({ - moved_to: 'storage', - bucket: marker.bucket || 'brain-files', - path: relPath, - moved_at: new Date().toISOString().split('T')[0], - original_hash: `sha256:${hash}`, + const stat = statSync(filePath); + const mimeType = getMimeType(filePath); + const bucket = marker.bucket || 'brain-files'; + const pointer = stringify({ + target: `supabase://${bucket}/${relPath}`, + bucket, + storage_path: relPath, + size: stat.size, + size_human: humanSize(stat.size), + hash: `sha256:${hash}`, + mime: mimeType || 'application/octet-stream', + uploaded: new Date().toISOString(), }); - writeFileSync(filePath + '.redirect', breadcrumb); + writeFileSync(filePath + '.redirect.yaml', pointer); unlinkSync(filePath); redirected++; } @@ -380,7 +528,7 @@ async function restoreFiles(args: string[]) { if (entry.startsWith('.')) continue; const full = join(d, entry); if (statSync(full).isDirectory()) findRedirects(full); - else if (entry.endsWith('.redirect')) redirectFiles.push(full); + else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) redirectFiles.push(full); } } findRedirects(dir); @@ -389,9 +537,10 @@ async function restoreFiles(args: string[]) { let failed = 0; for (const redirectPath of redirectFiles) { const info = parseYaml(readFileSync(redirectPath, 'utf-8')); - const originalPath = redirectPath.replace(/\.redirect$/, ''); + const originalPath = redirectPath.replace(/\.redirect(\.yaml)?$/, ''); try { - const data = await storage.download(info.path); + const storagePath = info.storage_path || info.path; // v0.9 or legacy format + const data = await storage.download(storagePath); writeFileSync(originalPath, data); unlinkSync(redirectPath); restored++; @@ -411,7 +560,7 @@ async function cleanFiles(args: string[]) { if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files clean [--yes]'); process.exit(1); } if (!confirmed) { - console.error('WARNING: This permanently removes .redirect breadcrumbs.'); + console.error('WARNING: This permanently removes redirect pointers.'); console.error('After this, files are only accessible from cloud storage.'); console.error('Git history still has the originals if you need them.'); console.error('Run with --yes to confirm.'); @@ -424,7 +573,7 @@ async function cleanFiles(args: string[]) { if (entry.startsWith('.')) continue; const full = join(d, entry); if (statSync(full).isDirectory()) findAndClean(full); - else if (entry.endsWith('.redirect')) { unlinkSync(full); cleaned++; } + else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) { unlinkSync(full); cleaned++; } } } findAndClean(dir); @@ -443,7 +592,7 @@ async function filesStatus(args: string[]) { const full = join(d, entry); if (entry === '.supabase') { mirrored++; continue; } if (statSync(full).isDirectory()) scan(full); - else if (entry.endsWith('.redirect')) redirected++; + else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) redirected++; else if (!entry.endsWith('.md')) local++; } } diff --git a/src/core/file-resolver.ts b/src/core/file-resolver.ts index acdefe3..b11f7a9 100644 --- a/src/core/file-resolver.ts +++ b/src/core/file-resolver.ts @@ -6,9 +6,10 @@ import type { StorageBackend } from './storage.ts'; /** * Universal file reader with fallback chain: * 1. Local file exists → return it - * 2. .redirect breadcrumb exists → fetch from storage - * 3. .supabase marker in parent dir → prefer storage, fall back to local - * 4. None → throw + * 2. .redirect.yaml pointer exists → fetch from storage (v0.9+ format) + * 3. .redirect breadcrumb exists → fetch from storage (legacy v0.8 format) + * 4. .supabase marker in parent dir → prefer storage, fall back to local + * 5. None → throw */ export interface ResolvedFile { @@ -16,6 +17,21 @@ export interface ResolvedFile { source: 'local' | 'storage' | 'redirect'; } +/** v0.9+ redirect format (.redirect.yaml) — richer metadata */ +export interface RedirectYaml { + target: string; // supabase://brain-files/{storagePath} + bucket: string; + storage_path: string; + size: number; + size_human: string; + hash: string; // sha256:... + mime: string; + uploaded: string; // ISO timestamp + source_url?: string; + type?: string; // transcript, article, image, etc. +} + +/** Legacy v0.8 redirect format (.redirect) */ export interface RedirectInfo { moved_to: string; bucket: string; @@ -43,16 +59,25 @@ export async function resolveFile( return { data: readFileSync(fullPath), source: 'local' }; } - // 2. .redirect breadcrumb - const redirectPath = fullPath + '.redirect'; - if (existsSync(redirectPath)) { + // 2. .redirect.yaml pointer (v0.9+ format) + const yamlRedirectPath = fullPath + '.redirect.yaml'; + if (existsSync(yamlRedirectPath)) { if (!storage) throw new Error(`File redirected to storage but no storage backend configured: ${filePath}`); - const info = parseRedirect(redirectPath); + const info = parseRedirectYaml(yamlRedirectPath); + const data = await storage.download(info.storage_path); + return { data, source: 'redirect' }; + } + + // 3. Legacy .redirect breadcrumb (v0.8 format) + const legacyRedirectPath = fullPath + '.redirect'; + if (existsSync(legacyRedirectPath)) { + if (!storage) throw new Error(`File redirected to storage but no storage backend configured: ${filePath}`); + const info = parseRedirect(legacyRedirectPath); const data = await storage.download(info.path); return { data, source: 'redirect' }; } - // 3. .supabase marker in parent directory + // 4. .supabase marker in parent directory const markerPath = join(dirname(fullPath), '.supabase'); if (existsSync(markerPath)) { if (!storage) throw new Error(`Directory mirrored to storage but no storage backend configured: ${filePath}`); @@ -73,6 +98,13 @@ export async function resolveFile( throw new Error(`File not found: ${filePath}`); } +/** Parse v0.9+ .redirect.yaml pointer */ +export function parseRedirectYaml(path: string): RedirectYaml { + const content = readFileSync(path, 'utf-8'); + return parseYaml(content) as RedirectYaml; +} + +/** Parse legacy v0.8 .redirect breadcrumb */ export function parseRedirect(path: string): RedirectInfo { const content = readFileSync(path, 'utf-8'); return parseYaml(content) as RedirectInfo; @@ -82,3 +114,11 @@ export function parseMarker(path: string): MarkerInfo { const content = readFileSync(path, 'utf-8'); return parseYaml(content) as MarkerInfo; } + +/** Human-readable file size */ +export function humanSize(bytes: number): string { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`; + if (bytes < 1024 * 1024 * 1024) return `${Math.round(bytes / (1024 * 1024))} MB`; + return `${(bytes / (1024 * 1024 * 1024)).toFixed(1)} GB`; +} diff --git a/src/core/storage/supabase.ts b/src/core/storage/supabase.ts index ab0d592..3b33cf5 100644 --- a/src/core/storage/supabase.ts +++ b/src/core/storage/supabase.ts @@ -1,8 +1,17 @@ import type { StorageBackend, StorageConfig } from '../storage.ts'; +/** Size thresholds for upload method selection */ +const TUS_THRESHOLD = 100 * 1024 * 1024; // 100 MB — use TUS resumable above this +const TUS_CHUNK_SIZE = 6 * 1024 * 1024; // 6 MB chunks for TUS uploads +const SIGNED_URL_EXPIRY = 3600; // 1 hour + /** * Supabase Storage — uses the Supabase Storage REST API. * Auth via the service role key (not the anon key). + * + * Upload method auto-selected by file size: + * < 100 MB → standard POST (single request) + * >= 100 MB → TUS resumable upload (6 MB chunks with retry) */ export class SupabaseStorage implements StorageBackend { private projectUrl: string; @@ -30,6 +39,15 @@ export class SupabaseStorage implements StorageBackend { } async upload(path: string, data: Buffer, mime?: string): Promise { + if (data.length >= TUS_THRESHOLD) { + await this.uploadTus(path, data, mime); + } else { + await this.uploadStandard(path, data, mime); + } + } + + /** Standard single-request upload for files < 100 MB */ + private async uploadStandard(path: string, data: Buffer, mime?: string): Promise { const res = await fetch(this.url(path), { method: 'POST', headers: { @@ -45,6 +63,78 @@ export class SupabaseStorage implements StorageBackend { } } + /** + * TUS resumable upload for files >= 100 MB. + * Sends in 6 MB chunks with retry + exponential backoff. + */ + private async uploadTus(path: string, data: Buffer, mime?: string): Promise { + const tusUrl = `${this.projectUrl}/storage/v1/upload/resumable`; + const objectName = `${this.bucket}/${path}`; + + // Step 1: Create the upload session + const createRes = await fetch(tusUrl, { + method: 'POST', + headers: { + ...this.headers(), + 'Tus-Resumable': '1.0.0', + 'Upload-Length': String(data.length), + 'Upload-Metadata': [ + `bucketName ${btoa(this.bucket)}`, + `objectName ${btoa(path)}`, + `contentType ${btoa(mime || 'application/octet-stream')}`, + ].join(','), + 'x-upsert': 'true', + }, + }); + + if (!createRes.ok) { + const body = await createRes.text(); + throw new Error(`TUS create failed: ${createRes.status} ${body}`); + } + + const uploadUrl = createRes.headers.get('Location'); + if (!uploadUrl) throw new Error('TUS create did not return Location header'); + + // Step 2: Upload chunks + let offset = 0; + while (offset < data.length) { + const end = Math.min(offset + TUS_CHUNK_SIZE, data.length); + const chunk = data.subarray(offset, end); + + let attempt = 0; + const maxAttempts = 3; + while (attempt < maxAttempts) { + try { + const patchRes = await fetch(uploadUrl, { + method: 'PATCH', + headers: { + ...this.headers(), + 'Tus-Resumable': '1.0.0', + 'Upload-Offset': String(offset), + 'Content-Type': 'application/offset+octet-stream', + 'Content-Length': String(chunk.length), + }, + body: chunk, + }); + + if (!patchRes.ok) { + const body = await patchRes.text(); + throw new Error(`TUS PATCH failed: ${patchRes.status} ${body}`); + } + + const newOffset = patchRes.headers.get('Upload-Offset'); + offset = newOffset ? parseInt(newOffset, 10) : end; + break; // Success, move to next chunk + } catch (err) { + attempt++; + if (attempt >= maxAttempts) throw err; + // Exponential backoff: 1s, 2s, 4s + await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1))); + } + } + } + } + async download(path: string): Promise { const res = await fetch(this.url(path), { headers: this.headers(), @@ -81,8 +171,28 @@ export class SupabaseStorage implements StorageBackend { return items.map(i => `${prefix}/${i.name}`); } + /** Generate a signed URL with 1-hour expiry for private bucket access */ + async getSignedUrl(path: string, expiresIn: number = SIGNED_URL_EXPIRY): Promise { + const res = await fetch(`${this.projectUrl}/storage/v1/object/sign/${this.bucket}/${path}`, { + method: 'POST', + headers: { ...this.headers(), 'Content-Type': 'application/json' }, + body: JSON.stringify({ expiresIn }), + }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Supabase signed URL failed: ${res.status} ${body}`); + } + const result = await res.json() as { signedURL: string }; + return `${this.projectUrl}${result.signedURL}`; + } + async getUrl(path: string): Promise { - // Public URL (if bucket is public) or signed URL - return `${this.projectUrl}/storage/v1/object/public/${this.bucket}/${path}`; + // Try signed URL first (works for private buckets) + try { + return await this.getSignedUrl(path); + } catch { + // Fall back to public URL + return `${this.projectUrl}/storage/v1/object/public/${this.bucket}/${path}`; + } } }