diff --git a/scripts/migrate.js b/scripts/migrate.js new file mode 100644 index 0000000..f371735 --- /dev/null +++ b/scripts/migrate.js @@ -0,0 +1,630 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const sax = require('sax'); +const { PrismaClient } = require('@prisma/client'); + +const DATABASE_URL = + process.env.DATABASE_URL || + 'postgresql://everclone:everclone_secret_pw@127.0.0.1:5433/everclone?schema=public'; + +const ATTACHMENTS_DIR = + process.env.ATTACHMENTS_DIR || 'X:\\App Server Data\\everclone\\attachments'; + +const DEFAULT_SOURCE_DIR = + process.env.ENEX_SOURCE_DIR || + 'Y:\\Evernote Backup\\Migration\\source-exports\\all-years-20261022-125657-5486cfaa'; + +const prisma = new PrismaClient({ + datasources: { db: { url: DATABASE_URL } }, +}); + +const MIME_EXTENSION_MAP = { + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', + 'image/svg+xml': '.svg', + 'application/pdf': '.pdf', + 'audio/mpeg': '.mp3', + 'audio/mp3': '.mp3', + 'audio/wav': '.wav', + 'audio/ogg': '.ogg', + 'audio/webm': '.webm', + 'audio/mp4': '.m4a', + 'video/mp4': '.mp4', + 'text/plain': '.txt', + 'text/html': '.html', + 'text/markdown': '.md', +}; + +function guessExtensionFromMime(mime) { + if (!mime) return '.bin'; + return MIME_EXTENSION_MAP[mime.toLowerCase()] || '.bin'; +} + +function parseEvernoteTimestamp(ts) { + if (!ts) return null; + const trimmed = ts.trim(); + const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z?$/i.exec(trimmed); + if (m) { + const iso = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`; + const d = new Date(iso); + if (!isNaN(d.getTime())) return d; + } + const fallback = new Date(trimmed); + return isNaN(fallback.getTime()) ? null : fallback; +} + +function stripHtmlToText(html) { + if (!html) return ''; + return html + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'/gi, "'") + .replace(/'/gi, "'") + .replace(/\s+/g, ' ') + .trim(); +} + +function makeExcerpt(text, maxLength = 160) { + if (!text) return ''; + if (text.length <= maxLength) return text; + return text.slice(0, maxLength).trim() + '...'; +} + +function escapeHtml(str) { + return str + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +function convertEnmlToHtml(enml, mediaByHash) { + if (!enml) return ''; + + let html = enml + .replace(/<\?xml[^>]*\?>/gi, '') + .replace(/]*>/gi, '') + .replace(/]*>/gi, '
') + .replace(/<\/en-note>/gi, '
'); + + html = html.replace(/]*?)\/?>/gi, (_match, attrs) => { + const isChecked = /checked\s*=\s*["']true["']/i.test(attrs); + return ` `; + }); + + html = html.replace( + /]*?)(?:\/>|><\/en-media>)/gi, + (_match, attrs) => { + const hashMatch = /hash\s*=\s*["']([a-fA-F0-9]+)["']/i.exec(attrs); + const typeMatch = /type\s*=\s*["']([^"']+)["']/i.exec(attrs); + const hash = hashMatch ? hashMatch[1].toLowerCase() : ''; + const mime = typeMatch ? typeMatch[1].toLowerCase() : ''; + + const attachment = hash ? mediaByHash.get(hash) : undefined; + if (!attachment) { + return `[Missing media ${hash ? hash.slice(0, 8) : 'unknown'}]`; + } + + const url = `/api/attachments/${attachment.id}`; + const effectiveMime = attachment.mimeType || mime; + + if (effectiveMime.startsWith('image/')) { + return `${escapeHtml(attachment.filename)}`; + } + if (effectiveMime.startsWith('audio/')) { + return `

🎵 ${escapeHtml(attachment.filename)}

`; + } + if (effectiveMime === 'application/pdf') { + return `
📄 ${escapeHtml(attachment.filename)}
`; + } + return `
📎 ${escapeHtml(attachment.filename)}
`; + } + ); + + return html.trim(); +} + +async function saveAttachmentBuffer(buffer, originalFilename, mimeType) { + const md5Hash = crypto.createHash('md5').update(buffer).digest('hex'); + const ext = path.extname(originalFilename) || guessExtensionFromMime(mimeType); + const subDir = md5Hash.slice(0, 2); + const targetDir = path.join(ATTACHMENTS_DIR, subDir); + await fs.promises.mkdir(targetDir, { recursive: true }); + + const fileNameOnDisk = `${md5Hash}${ext}`; + const fullPath = path.join(targetDir, fileNameOnDisk); + + try { + await fs.promises.access(fullPath); + } catch { + await fs.promises.writeFile(fullPath, buffer); + } + + let extractedText = null; + if ( + mimeType.startsWith('text/') || + mimeType === 'application/json' || + mimeType === 'application/xml' || + originalFilename.endsWith('.md') || + originalFilename.endsWith('.txt') || + originalFilename.endsWith('.csv') + ) { + extractedText = buffer.toString('utf-8').slice(0, 50000); + } + + return { + md5Hash, + storagePath: `${subDir}/${fileNameOnDisk}`, + sizeBytes: buffer.length, + extractedText, + }; +} + +const tagCache = new Map(); +async function getOrCreateTagId(name, userId = 'default') { + const clean = name.trim(); + const cacheKey = `${userId}:${clean.toLowerCase()}`; + if (tagCache.has(cacheKey)) return tagCache.get(cacheKey); + + let tag = await prisma.tag.findFirst({ + where: { userId, name: { equals: clean, mode: 'insensitive' } }, + }); + if (!tag) { + tag = await prisma.tag.create({ + data: { userId, name: clean }, + }); + } + tagCache.set(cacheKey, tag.id); + return tag.id; +} + +const notebookCache = new Map(); +async function getOrCreateNotebookWithStack(notebookName, stackName, userId = 'default') { + const cleanNotebook = (notebookName || 'Imported Notes').trim(); + const cleanStack = stackName ? stackName.trim() : null; + const cacheKey = `${userId}:${cleanNotebook.toLowerCase()}:${cleanStack ? cleanStack.toLowerCase() : ''}`; + if (notebookCache.has(cacheKey)) return notebookCache.get(cacheKey); + + let stackId = null; + if (cleanStack) { + let stack = await prisma.notebookStack.findFirst({ + where: { userId, name: { equals: cleanStack, mode: 'insensitive' } }, + }); + if (!stack) { + stack = await prisma.notebookStack.create({ + data: { userId, name: cleanStack }, + }); + } + stackId = stack.id; + } + + let notebook = await prisma.notebook.findFirst({ + where: { userId, name: { equals: cleanNotebook, mode: 'insensitive' } }, + }); + + if (notebook) { + if (stackId && notebook.stackId !== stackId) { + notebook = await prisma.notebook.update({ + where: { id: notebook.id }, + data: { stackId }, + }); + } + notebookCache.set(cacheKey, notebook.id); + return notebook.id; + } + + notebook = await prisma.notebook.create({ + data: { + userId, + name: cleanNotebook, + stackId, + }, + }); + notebookCache.set(cacheKey, notebook.id); + return notebook.id; +} + +function parseSingleNoteXml(singleNoteXml) { + return new Promise((resolve) => { + const parser = sax.parser(false, { lowercase: true, trim: false }); + const note = { + title: 'Untitled Note', + contentEnml: '', + tags: [], + resources: [], + }; + let currentResource = null; + const tagStack = []; + let textBuffer = ''; + + parser.onopentag = (node) => { + tagStack.push(node.name); + textBuffer = ''; + if (node.name === 'resource') { + currentResource = { + base64Data: '', + mime: 'application/octet-stream', + }; + } + }; + + parser.ontext = (t) => { + textBuffer += t; + }; + + parser.oncdata = (c) => { + textBuffer += c; + }; + + parser.onclosetag = (tagName) => { + const parentTag = tagStack[tagStack.length - 2] || ''; + const val = textBuffer.trim(); + + if (currentResource) { + if (tagName === 'data') { + currentResource.base64Data = val; + } else if (tagName === 'mime') { + currentResource.mime = val; + } else if (tagName === 'width') { + const w = parseInt(val, 10); + if (!isNaN(w)) currentResource.width = w; + } else if (tagName === 'height') { + const h = parseInt(val, 10); + if (!isNaN(h)) currentResource.height = h; + } else if (tagName === 'file-name') { + currentResource.fileName = val; + } else if (tagName === 'resource') { + note.resources.push(currentResource); + currentResource = null; + } + } else { + if (tagName === 'title' && parentTag === 'note') { + note.title = val || 'Untitled Note'; + } else if (tagName === 'content' && parentTag === 'note') { + note.contentEnml = textBuffer; + } else if (tagName === 'created' && parentTag === 'note') { + note.created = val; + } else if (tagName === 'updated' && parentTag === 'note') { + note.updated = val; + } else if (tagName === 'tag' && parentTag === 'note') { + if (val) note.tags.push(val); + } else if (tagName === 'author' && parentTag === 'note-attributes') { + note.author = val; + } else if (tagName === 'source-url' && parentTag === 'note-attributes') { + note.sourceUrl = val; + } else if (tagName === 'latitude' && parentTag === 'note-attributes') { + const lat = parseFloat(val); + if (!isNaN(lat)) note.latitude = lat; + } else if (tagName === 'longitude' && parentTag === 'note-attributes') { + const lon = parseFloat(val); + if (!isNaN(lon)) note.longitude = lon; + } else if (tagName === 'reminder-order' && parentTag === 'note-attributes') { + note.reminderTime = val; + } + } + + tagStack.pop(); + }; + + parser.onerror = (e) => { + parser.error = null; + parser.resume(); + }; + + parser.onend = () => { + resolve(note); + }; + + parser.write(singleNoteXml).close(); + }); +} + +async function processNote(rawNote, notebookId, userId = 'default') { + const title = (rawNote.title || 'Untitled Note').trim(); + const createdAt = parseEvernoteTimestamp(rawNote.created) || new Date(); + const updatedAt = parseEvernoteTimestamp(rawNote.updated) || createdAt; + const reminderAt = parseEvernoteTimestamp(rawNote.reminderTime); + + // Check if identical note already imported (idempotent resume) + const existing = await prisma.note.findFirst({ + where: { + userId, + notebookId, + title, + createdAt, + }, + select: { id: true }, + }); + + if (existing) { + return { status: 'skipped', noteId: existing.id }; + } + + // Pre-resolve tags + const tagIds = []; + if (rawNote.tags && rawNote.tags.length > 0) { + for (const t of rawNote.tags) { + if (t && t.trim()) { + tagIds.push(await getOrCreateTagId(t.trim(), userId)); + } + } + } + + // Create initial Note record + const note = await prisma.note.create({ + data: { + userId, + notebookId, + title, + contentHtml: '', + contentText: '', + excerpt: '', + author: rawNote.author || null, + sourceUrl: rawNote.sourceUrl || null, + latitude: rawNote.latitude ?? null, + longitude: rawNote.longitude ?? null, + reminderAt: reminderAt ?? null, + createdAt, + updatedAt, + tags: { + create: tagIds.map((tagId) => ({ tagId })), + }, + }, + }); + + // Process attachments + const mediaByHash = new Map(); + if (rawNote.resources && rawNote.resources.length > 0) { + for (let i = 0; i < rawNote.resources.length; i++) { + const res = rawNote.resources[i]; + if (!res.base64Data) continue; + const cleanB64 = res.base64Data.replace(/\s+/g, ''); + const buffer = Buffer.from(cleanB64, 'base64'); + if (buffer.length === 0) continue; + + const mimeType = res.mime || 'application/octet-stream'; + const ext = guessExtensionFromMime(mimeType); + const filename = res.fileName?.trim() || `attachment-${i + 1}${ext}`; + + const savedBlob = await saveAttachmentBuffer(buffer, filename, mimeType); + const attachment = await prisma.attachment.create({ + data: { + noteId: note.id, + filename, + mimeType, + sizeBytes: savedBlob.sizeBytes, + md5Hash: savedBlob.md5Hash, + storagePath: savedBlob.storagePath, + width: res.width ?? null, + height: res.height ?? null, + extractedText: savedBlob.extractedText, + }, + }); + + mediaByHash.set(savedBlob.md5Hash.toLowerCase(), { + id: attachment.id, + filename: attachment.filename, + mimeType: attachment.mimeType, + sizeBytes: attachment.sizeBytes, + }); + } + } + + // Format HTML & text + const contentHtml = convertEnmlToHtml(rawNote.contentEnml, mediaByHash); + const contentText = stripHtmlToText(contentHtml); + const excerpt = makeExcerpt(contentText); + + await prisma.note.update({ + where: { id: note.id }, + data: { + contentHtml, + contentText, + excerpt, + updatedAt, + }, + }); + + return { status: 'created', noteId: note.id, attachments: mediaByHash.size }; +} + +/** + * Strictly sequential streaming .enex importer: + * Extracts one ... block at a time from the stream, + * parses and writes to PostgreSQL/NAS sequentially without concurrency overlap. + */ +async function importEnexFile(filePath, options = {}) { + const stat = await fs.promises.stat(filePath); + const sizeMb = (stat.size / (1024 * 1024)).toFixed(2); + + const parsedPath = path.parse(filePath); + const notebookName = options.notebookName || parsedPath.name; + const parentDir = path.basename(parsedPath.dir); + const stackName = options.stackName !== undefined + ? options.stackName + : (parentDir && parentDir !== path.basename(DEFAULT_SOURCE_DIR) ? parentDir : null); + + const notebookId = await getOrCreateNotebookWithStack(notebookName, stackName, options.userId || 'default'); + + console.log(`\n======================================================`); + console.log(`Importing: "${notebookName}" (Stack: ${stackName ? `"${stackName}"` : 'None'})`); + console.log(`File: ${filePath} (${sizeMb} MB)`); + console.log(`Notebook ID: ${notebookId}`); + console.log(`======================================================`); + + let totalParsed = 0; + let importedCount = 0; + let skippedCount = 0; + let errorCount = 0; + const startTime = Date.now(); + + const fileStream = fs.createReadStream(filePath, { encoding: 'utf8', highWaterMark: 1024 * 1024 }); + let carry = ''; + + for await (const chunk of fileStream) { + carry += chunk; + + while (true) { + const startIdx = carry.indexOf(''); + if (startIdx === -1) { + if (carry.length > 64) carry = carry.slice(-64); + break; + } + const closeIdx = carry.indexOf('', startIdx); + if (closeIdx === -1) { + if (startIdx > 0) carry = carry.slice(startIdx); + break; + } + + const singleNoteXml = carry.slice(startIdx, closeIdx + 7); + carry = carry.slice(closeIdx + 7); + totalParsed++; + + try { + const rawNote = await parseSingleNoteXml(singleNoteXml); + const res = await processNote(rawNote, notebookId, options.userId || 'default'); + if (res.status === 'created') { + importedCount++; + } else if (res.status === 'skipped') { + skippedCount++; + } + } catch (err) { + errorCount++; + console.error(`\nError saving note #${totalParsed}:`, err.message); + } + + if (totalParsed % 10 === 0 || totalParsed === 1) { + const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1); + const rate = (totalParsed / (elapsedSec || 1)).toFixed(1); + process.stdout.write( + `\r [${notebookName}] Notes: ${totalParsed} (${importedCount} new, ${skippedCount} skipped, ${errorCount} errors) - ${elapsedSec}s (${rate} notes/sec)` + ); + } + } + } + + const totalSec = ((Date.now() - startTime) / 1000).toFixed(1); + console.log( + `\nCompleted "${notebookName}": ${totalParsed} total notes (${importedCount} new, ${skippedCount} skipped, ${errorCount} errors) in ${totalSec}s.\n` + ); + + return { + notebookName, + stackName, + totalParsed, + importedCount, + skippedCount, + errorCount, + elapsedSeconds: parseFloat(totalSec), + }; +} + +function findEnexFiles(dir) { + const results = []; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + results.push(...findEnexFiles(fullPath)); + } else if (entry.isFile() && entry.name.endsWith('.enex')) { + const stat = fs.statSync(fullPath); + results.push({ path: fullPath, name: entry.name, size: stat.size }); + } + } + return results.sort((a, b) => a.size - b.size); +} + +// CLI entrypoint +async function main() { + const args = process.argv.slice(2); + const flag = args[0]; + + if (!flag) { + console.log(`Usage:`); + console.log(` node scripts/migrate.js --batch-a # Run 23 small notebooks (<10MB)`); + console.log(` node scripts/migrate.js --batch-b # Run 25 medium notebooks (10MB-500MB)`); + console.log(` node scripts/migrate.js --batch-c # Run 9 large notebooks (>=500MB)`); + console.log(` node scripts/migrate.js --all # Run all 57 notebooks sequentially`); + console.log(` node scripts/migrate.js # Run a specific .enex file`); + process.exit(0); + } + + const allFiles = findEnexFiles(DEFAULT_SOURCE_DIR); + + let targetFiles = []; + if (flag === '--batch-a') { + targetFiles = allFiles.filter((f) => f.size < 10 * 1024 * 1024); + } else if (flag === '--batch-b') { + targetFiles = allFiles.filter((f) => f.size >= 10 * 1024 * 1024 && f.size < 500 * 1024 * 1024); + } else if (flag === '--batch-c') { + targetFiles = allFiles.filter((f) => f.size >= 500 * 1024 * 1024); + } else if (flag === '--all') { + targetFiles = allFiles; + } else { + const fullPath = path.isAbsolute(flag) ? flag : path.join(DEFAULT_SOURCE_DIR, flag); + if (!fs.existsSync(fullPath)) { + console.error(`File not found: ${fullPath}`); + process.exit(1); + } + const stat = fs.statSync(fullPath); + targetFiles = [{ path: fullPath, name: path.basename(fullPath), size: stat.size }]; + } + + console.log(`\n======================================================`); + console.log(`Starting migration for ${targetFiles.length} file(s)...`); + console.log(`Total payload size: ${(targetFiles.reduce((acc, f) => acc + f.size, 0) / (1024 * 1024)).toFixed(1)} MB`); + console.log(`======================================================\n`); + + let grandTotalParsed = 0; + let grandTotalImported = 0; + let grandTotalSkipped = 0; + let grandTotalErrors = 0; + const grandStartTime = Date.now(); + + try { + for (let i = 0; i < targetFiles.length; i++) { + const f = targetFiles[i]; + console.log(`[Batch Progress: ${i + 1}/${targetFiles.length}] Processing ${f.name}...`); + const res = await importEnexFile(f.path); + grandTotalParsed += res.totalParsed; + grandTotalImported += res.importedCount; + grandTotalSkipped += res.skippedCount; + grandTotalErrors += res.errorCount; + } + + const totalMinutes = ((Date.now() - grandStartTime) / 60000).toFixed(2); + console.log(`\n======================================================`); + console.log(`Batch Summary:`); + console.log(` Files Processed: ${targetFiles.length}`); + console.log(` Total Notes: ${grandTotalParsed}`); + console.log(` New Notes: ${grandTotalImported}`); + console.log(` Skipped Notes: ${grandTotalSkipped}`); + console.log(` Errors: ${grandTotalErrors}`); + console.log(` Total Runtime: ${totalMinutes} minutes`); + console.log(`======================================================\n`); + } catch (e) { + console.error('Batch migration encountered fatal error:', e); + process.exit(1); + } finally { + await prisma.$disconnect(); + } +} + +if (require.main === module) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} + +module.exports = { importEnexFile, getOrCreateNotebookWithStack, processNote }; diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 1b6b502..26fe90e 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -54,7 +54,8 @@ export async function saveAttachmentBuffer( export function getFullAttachmentPath(storagePath: string): string { const baseDir = getAttachmentsDir(); - const resolved = path.resolve(baseDir, storagePath); + const normalizedStoragePath = storagePath.replace(/\\/g, '/'); + const resolved = path.resolve(baseDir, normalizedStoragePath); if (!resolved.startsWith(path.resolve(baseDir))) { throw new Error('Invalid storage path'); }