Add streaming bulk ENEX migration tool and normalize attachment paths
This commit is contained in:
chrisf committed
1 parent
31b674bf01
commit
e74d1ad57b
2 files changed
+632
-1
No files matched your search
@@ -0,0 +1,630 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const sax = require('sax');
|
||||
const { PrismaClient } = require('@prisma/client');
|
||||
|
||||
const DATABASE_URL =
|
||||
process.env.DATABASE_URL ||
|
||||
'postgresql://everclone:everclone_secret_pw@127.0.0.1:5433/everclone?schema=public';
|
||||
|
||||
const ATTACHMENTS_DIR =
|
||||
process.env.ATTACHMENTS_DIR || 'X:\\App Server Data\\everclone\\attachments';
|
||||
|
||||
const DEFAULT_SOURCE_DIR =
|
||||
process.env.ENEX_SOURCE_DIR ||
|
||||
'Y:\\Evernote Backup\\Migration\\source-exports\\all-years-20261022-125657-5486cfaa';
|
||||
|
||||
const prisma = new PrismaClient({
|
||||
datasources: { db: { url: DATABASE_URL } },
|
||||
});
|
||||
|
||||
const MIME_EXTENSION_MAP = {
|
||||
'image/jpeg': '.jpg',
|
||||
'image/jpg': '.jpg',
|
||||
'image/png': '.png',
|
||||
'image/gif': '.gif',
|
||||
'image/webp': '.webp',
|
||||
'image/svg+xml': '.svg',
|
||||
'application/pdf': '.pdf',
|
||||
'audio/mpeg': '.mp3',
|
||||
'audio/mp3': '.mp3',
|
||||
'audio/wav': '.wav',
|
||||
'audio/ogg': '.ogg',
|
||||
'audio/webm': '.webm',
|
||||
'audio/mp4': '.m4a',
|
||||
'video/mp4': '.mp4',
|
||||
'text/plain': '.txt',
|
||||
'text/html': '.html',
|
||||
'text/markdown': '.md',
|
||||
};
|
||||
|
||||
function guessExtensionFromMime(mime) {
|
||||
if (!mime) return '.bin';
|
||||
return MIME_EXTENSION_MAP[mime.toLowerCase()] || '.bin';
|
||||
}
|
||||
|
||||
function parseEvernoteTimestamp(ts) {
|
||||
if (!ts) return null;
|
||||
const trimmed = ts.trim();
|
||||
const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z?$/i.exec(trimmed);
|
||||
if (m) {
|
||||
const iso = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`;
|
||||
const d = new Date(iso);
|
||||
if (!isNaN(d.getTime())) return d;
|
||||
}
|
||||
const fallback = new Date(trimmed);
|
||||
return isNaN(fallback.getTime()) ? null : fallback;
|
||||
}
|
||||
|
||||
function stripHtmlToText(html) {
|
||||
if (!html) return '';
|
||||
return html
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/'/gi, "'")
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function makeExcerpt(text, maxLength = 160) {
|
||||
if (!text) return '';
|
||||
if (text.length <= maxLength) return text;
|
||||
return text.slice(0, maxLength).trim() + '...';
|
||||
}
|
||||
|
||||
function escapeHtml(str) {
|
||||
return str
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"');
|
||||
}
|
||||
|
||||
function convertEnmlToHtml(enml, mediaByHash) {
|
||||
if (!enml) return '';
|
||||
|
||||
let html = enml
|
||||
.replace(/<\?xml[^>]*\?>/gi, '')
|
||||
.replace(/<!DOCTYPE[^>]*>/gi, '')
|
||||
.replace(/<en-note[^>]*>/gi, '<div>')
|
||||
.replace(/<\/en-note>/gi, '</div>');
|
||||
|
||||
html = html.replace(/<en-todo\s+([^>]*?)\/?>/gi, (_match, attrs) => {
|
||||
const isChecked = /checked\s*=\s*["']true["']/i.test(attrs);
|
||||
return `<input type="checkbox" ${isChecked ? 'checked="checked"' : ''} data-en-todo="true" /> `;
|
||||
});
|
||||
|
||||
html = html.replace(
|
||||
/<en-media\b([^>]*?)(?:\/>|><\/en-media>)/gi,
|
||||
(_match, attrs) => {
|
||||
const hashMatch = /hash\s*=\s*["']([a-fA-F0-9]+)["']/i.exec(attrs);
|
||||
const typeMatch = /type\s*=\s*["']([^"']+)["']/i.exec(attrs);
|
||||
const hash = hashMatch ? hashMatch[1].toLowerCase() : '';
|
||||
const mime = typeMatch ? typeMatch[1].toLowerCase() : '';
|
||||
|
||||
const attachment = hash ? mediaByHash.get(hash) : undefined;
|
||||
if (!attachment) {
|
||||
return `<span class="text-xs text-gray-400">[Missing media ${hash ? hash.slice(0, 8) : 'unknown'}]</span>`;
|
||||
}
|
||||
|
||||
const url = `/api/attachments/${attachment.id}`;
|
||||
const effectiveMime = attachment.mimeType || mime;
|
||||
|
||||
if (effectiveMime.startsWith('image/')) {
|
||||
return `<img src="${url}" alt="${escapeHtml(attachment.filename)}" data-attachment-id="${attachment.id}" />`;
|
||||
}
|
||||
if (effectiveMime.startsWith('audio/')) {
|
||||
return `<div class="my-2 p-3 bg-gray-50 border rounded-lg"><p class="text-xs font-medium text-gray-600 mb-1">🎵 ${escapeHtml(attachment.filename)}</p><audio controls src="${url}" class="w-full"></audio></div>`;
|
||||
}
|
||||
if (effectiveMime === 'application/pdf') {
|
||||
return `<div class="my-2 p-3 bg-gray-50 border rounded-lg"><a href="${url}" target="_blank" rel="noopener noreferrer" class="text-evernote-green font-medium underline">📄 ${escapeHtml(attachment.filename)}</a></div>`;
|
||||
}
|
||||
return `<div class="my-2 p-2.5 bg-gray-50 border rounded-lg inline-block"><a href="${url}" target="_blank" rel="noopener noreferrer" class="text-evernote-green font-medium underline">📎 ${escapeHtml(attachment.filename)}</a></div>`;
|
||||
}
|
||||
);
|
||||
|
||||
return html.trim();
|
||||
}
|
||||
|
||||
async function saveAttachmentBuffer(buffer, originalFilename, mimeType) {
|
||||
const md5Hash = crypto.createHash('md5').update(buffer).digest('hex');
|
||||
const ext = path.extname(originalFilename) || guessExtensionFromMime(mimeType);
|
||||
const subDir = md5Hash.slice(0, 2);
|
||||
const targetDir = path.join(ATTACHMENTS_DIR, subDir);
|
||||
await fs.promises.mkdir(targetDir, { recursive: true });
|
||||
|
||||
const fileNameOnDisk = `${md5Hash}${ext}`;
|
||||
const fullPath = path.join(targetDir, fileNameOnDisk);
|
||||
|
||||
try {
|
||||
await fs.promises.access(fullPath);
|
||||
} catch {
|
||||
await fs.promises.writeFile(fullPath, buffer);
|
||||
}
|
||||
|
||||
let extractedText = null;
|
||||
if (
|
||||
mimeType.startsWith('text/') ||
|
||||
mimeType === 'application/json' ||
|
||||
mimeType === 'application/xml' ||
|
||||
originalFilename.endsWith('.md') ||
|
||||
originalFilename.endsWith('.txt') ||
|
||||
originalFilename.endsWith('.csv')
|
||||
) {
|
||||
extractedText = buffer.toString('utf-8').slice(0, 50000);
|
||||
}
|
||||
|
||||
return {
|
||||
md5Hash,
|
||||
storagePath: `${subDir}/${fileNameOnDisk}`,
|
||||
sizeBytes: buffer.length,
|
||||
extractedText,
|
||||
};
|
||||
}
|
||||
|
||||
const tagCache = new Map();
|
||||
async function getOrCreateTagId(name, userId = 'default') {
|
||||
const clean = name.trim();
|
||||
const cacheKey = `${userId}:${clean.toLowerCase()}`;
|
||||
if (tagCache.has(cacheKey)) return tagCache.get(cacheKey);
|
||||
|
||||
let tag = await prisma.tag.findFirst({
|
||||
where: { userId, name: { equals: clean, mode: 'insensitive' } },
|
||||
});
|
||||
if (!tag) {
|
||||
tag = await prisma.tag.create({
|
||||
data: { userId, name: clean },
|
||||
});
|
||||
}
|
||||
tagCache.set(cacheKey, tag.id);
|
||||
return tag.id;
|
||||
}
|
||||
|
||||
const notebookCache = new Map();
|
||||
async function getOrCreateNotebookWithStack(notebookName, stackName, userId = 'default') {
|
||||
const cleanNotebook = (notebookName || 'Imported Notes').trim();
|
||||
const cleanStack = stackName ? stackName.trim() : null;
|
||||
const cacheKey = `${userId}:${cleanNotebook.toLowerCase()}:${cleanStack ? cleanStack.toLowerCase() : ''}`;
|
||||
if (notebookCache.has(cacheKey)) return notebookCache.get(cacheKey);
|
||||
|
||||
let stackId = null;
|
||||
if (cleanStack) {
|
||||
let stack = await prisma.notebookStack.findFirst({
|
||||
where: { userId, name: { equals: cleanStack, mode: 'insensitive' } },
|
||||
});
|
||||
if (!stack) {
|
||||
stack = await prisma.notebookStack.create({
|
||||
data: { userId, name: cleanStack },
|
||||
});
|
||||
}
|
||||
stackId = stack.id;
|
||||
}
|
||||
|
||||
let notebook = await prisma.notebook.findFirst({
|
||||
where: { userId, name: { equals: cleanNotebook, mode: 'insensitive' } },
|
||||
});
|
||||
|
||||
if (notebook) {
|
||||
if (stackId && notebook.stackId !== stackId) {
|
||||
notebook = await prisma.notebook.update({
|
||||
where: { id: notebook.id },
|
||||
data: { stackId },
|
||||
});
|
||||
}
|
||||
notebookCache.set(cacheKey, notebook.id);
|
||||
return notebook.id;
|
||||
}
|
||||
|
||||
notebook = await prisma.notebook.create({
|
||||
data: {
|
||||
userId,
|
||||
name: cleanNotebook,
|
||||
stackId,
|
||||
},
|
||||
});
|
||||
notebookCache.set(cacheKey, notebook.id);
|
||||
return notebook.id;
|
||||
}
|
||||
|
||||
function parseSingleNoteXml(singleNoteXml) {
|
||||
return new Promise((resolve) => {
|
||||
const parser = sax.parser(false, { lowercase: true, trim: false });
|
||||
const note = {
|
||||
title: 'Untitled Note',
|
||||
contentEnml: '',
|
||||
tags: [],
|
||||
resources: [],
|
||||
};
|
||||
let currentResource = null;
|
||||
const tagStack = [];
|
||||
let textBuffer = '';
|
||||
|
||||
parser.onopentag = (node) => {
|
||||
tagStack.push(node.name);
|
||||
textBuffer = '';
|
||||
if (node.name === 'resource') {
|
||||
currentResource = {
|
||||
base64Data: '',
|
||||
mime: 'application/octet-stream',
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
parser.ontext = (t) => {
|
||||
textBuffer += t;
|
||||
};
|
||||
|
||||
parser.oncdata = (c) => {
|
||||
textBuffer += c;
|
||||
};
|
||||
|
||||
parser.onclosetag = (tagName) => {
|
||||
const parentTag = tagStack[tagStack.length - 2] || '';
|
||||
const val = textBuffer.trim();
|
||||
|
||||
if (currentResource) {
|
||||
if (tagName === 'data') {
|
||||
currentResource.base64Data = val;
|
||||
} else if (tagName === 'mime') {
|
||||
currentResource.mime = val;
|
||||
} else if (tagName === 'width') {
|
||||
const w = parseInt(val, 10);
|
||||
if (!isNaN(w)) currentResource.width = w;
|
||||
} else if (tagName === 'height') {
|
||||
const h = parseInt(val, 10);
|
||||
if (!isNaN(h)) currentResource.height = h;
|
||||
} else if (tagName === 'file-name') {
|
||||
currentResource.fileName = val;
|
||||
} else if (tagName === 'resource') {
|
||||
note.resources.push(currentResource);
|
||||
currentResource = null;
|
||||
}
|
||||
} else {
|
||||
if (tagName === 'title' && parentTag === 'note') {
|
||||
note.title = val || 'Untitled Note';
|
||||
} else if (tagName === 'content' && parentTag === 'note') {
|
||||
note.contentEnml = textBuffer;
|
||||
} else if (tagName === 'created' && parentTag === 'note') {
|
||||
note.created = val;
|
||||
} else if (tagName === 'updated' && parentTag === 'note') {
|
||||
note.updated = val;
|
||||
} else if (tagName === 'tag' && parentTag === 'note') {
|
||||
if (val) note.tags.push(val);
|
||||
} else if (tagName === 'author' && parentTag === 'note-attributes') {
|
||||
note.author = val;
|
||||
} else if (tagName === 'source-url' && parentTag === 'note-attributes') {
|
||||
note.sourceUrl = val;
|
||||
} else if (tagName === 'latitude' && parentTag === 'note-attributes') {
|
||||
const lat = parseFloat(val);
|
||||
if (!isNaN(lat)) note.latitude = lat;
|
||||
} else if (tagName === 'longitude' && parentTag === 'note-attributes') {
|
||||
const lon = parseFloat(val);
|
||||
if (!isNaN(lon)) note.longitude = lon;
|
||||
} else if (tagName === 'reminder-order' && parentTag === 'note-attributes') {
|
||||
note.reminderTime = val;
|
||||
}
|
||||
}
|
||||
|
||||
tagStack.pop();
|
||||
};
|
||||
|
||||
parser.onerror = (e) => {
|
||||
parser.error = null;
|
||||
parser.resume();
|
||||
};
|
||||
|
||||
parser.onend = () => {
|
||||
resolve(note);
|
||||
};
|
||||
|
||||
parser.write(singleNoteXml).close();
|
||||
});
|
||||
}
|
||||
|
||||
async function processNote(rawNote, notebookId, userId = 'default') {
|
||||
const title = (rawNote.title || 'Untitled Note').trim();
|
||||
const createdAt = parseEvernoteTimestamp(rawNote.created) || new Date();
|
||||
const updatedAt = parseEvernoteTimestamp(rawNote.updated) || createdAt;
|
||||
const reminderAt = parseEvernoteTimestamp(rawNote.reminderTime);
|
||||
|
||||
// Check if identical note already imported (idempotent resume)
|
||||
const existing = await prisma.note.findFirst({
|
||||
where: {
|
||||
userId,
|
||||
notebookId,
|
||||
title,
|
||||
createdAt,
|
||||
},
|
||||
select: { id: true },
|
||||
});
|
||||
|
||||
if (existing) {
|
||||
return { status: 'skipped', noteId: existing.id };
|
||||
}
|
||||
|
||||
// Pre-resolve tags
|
||||
const tagIds = [];
|
||||
if (rawNote.tags && rawNote.tags.length > 0) {
|
||||
for (const t of rawNote.tags) {
|
||||
if (t && t.trim()) {
|
||||
tagIds.push(await getOrCreateTagId(t.trim(), userId));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create initial Note record
|
||||
const note = await prisma.note.create({
|
||||
data: {
|
||||
userId,
|
||||
notebookId,
|
||||
title,
|
||||
contentHtml: '',
|
||||
contentText: '',
|
||||
excerpt: '',
|
||||
author: rawNote.author || null,
|
||||
sourceUrl: rawNote.sourceUrl || null,
|
||||
latitude: rawNote.latitude ?? null,
|
||||
longitude: rawNote.longitude ?? null,
|
||||
reminderAt: reminderAt ?? null,
|
||||
createdAt,
|
||||
updatedAt,
|
||||
tags: {
|
||||
create: tagIds.map((tagId) => ({ tagId })),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Process attachments
|
||||
const mediaByHash = new Map();
|
||||
if (rawNote.resources && rawNote.resources.length > 0) {
|
||||
for (let i = 0; i < rawNote.resources.length; i++) {
|
||||
const res = rawNote.resources[i];
|
||||
if (!res.base64Data) continue;
|
||||
const cleanB64 = res.base64Data.replace(/\s+/g, '');
|
||||
const buffer = Buffer.from(cleanB64, 'base64');
|
||||
if (buffer.length === 0) continue;
|
||||
|
||||
const mimeType = res.mime || 'application/octet-stream';
|
||||
const ext = guessExtensionFromMime(mimeType);
|
||||
const filename = res.fileName?.trim() || `attachment-${i + 1}${ext}`;
|
||||
|
||||
const savedBlob = await saveAttachmentBuffer(buffer, filename, mimeType);
|
||||
const attachment = await prisma.attachment.create({
|
||||
data: {
|
||||
noteId: note.id,
|
||||
filename,
|
||||
mimeType,
|
||||
sizeBytes: savedBlob.sizeBytes,
|
||||
md5Hash: savedBlob.md5Hash,
|
||||
storagePath: savedBlob.storagePath,
|
||||
width: res.width ?? null,
|
||||
height: res.height ?? null,
|
||||
extractedText: savedBlob.extractedText,
|
||||
},
|
||||
});
|
||||
|
||||
mediaByHash.set(savedBlob.md5Hash.toLowerCase(), {
|
||||
id: attachment.id,
|
||||
filename: attachment.filename,
|
||||
mimeType: attachment.mimeType,
|
||||
sizeBytes: attachment.sizeBytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Format HTML & text
|
||||
const contentHtml = convertEnmlToHtml(rawNote.contentEnml, mediaByHash);
|
||||
const contentText = stripHtmlToText(contentHtml);
|
||||
const excerpt = makeExcerpt(contentText);
|
||||
|
||||
await prisma.note.update({
|
||||
where: { id: note.id },
|
||||
data: {
|
||||
contentHtml,
|
||||
contentText,
|
||||
excerpt,
|
||||
updatedAt,
|
||||
},
|
||||
});
|
||||
|
||||
return { status: 'created', noteId: note.id, attachments: mediaByHash.size };
|
||||
}
|
||||
|
||||
/**
|
||||
* Strictly sequential streaming .enex importer:
|
||||
* Extracts one <note>...</note> block at a time from the stream,
|
||||
* parses and writes to PostgreSQL/NAS sequentially without concurrency overlap.
|
||||
*/
|
||||
async function importEnexFile(filePath, options = {}) {
|
||||
const stat = await fs.promises.stat(filePath);
|
||||
const sizeMb = (stat.size / (1024 * 1024)).toFixed(2);
|
||||
|
||||
const parsedPath = path.parse(filePath);
|
||||
const notebookName = options.notebookName || parsedPath.name;
|
||||
const parentDir = path.basename(parsedPath.dir);
|
||||
const stackName = options.stackName !== undefined
|
||||
? options.stackName
|
||||
: (parentDir && parentDir !== path.basename(DEFAULT_SOURCE_DIR) ? parentDir : null);
|
||||
|
||||
const notebookId = await getOrCreateNotebookWithStack(notebookName, stackName, options.userId || 'default');
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(`Importing: "${notebookName}" (Stack: ${stackName ? `"${stackName}"` : 'None'})`);
|
||||
console.log(`File: ${filePath} (${sizeMb} MB)`);
|
||||
console.log(`Notebook ID: ${notebookId}`);
|
||||
console.log(`======================================================`);
|
||||
|
||||
let totalParsed = 0;
|
||||
let importedCount = 0;
|
||||
let skippedCount = 0;
|
||||
let errorCount = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
const fileStream = fs.createReadStream(filePath, { encoding: 'utf8', highWaterMark: 1024 * 1024 });
|
||||
let carry = '';
|
||||
|
||||
for await (const chunk of fileStream) {
|
||||
carry += chunk;
|
||||
|
||||
while (true) {
|
||||
const startIdx = carry.indexOf('<note>');
|
||||
if (startIdx === -1) {
|
||||
if (carry.length > 64) carry = carry.slice(-64);
|
||||
break;
|
||||
}
|
||||
const closeIdx = carry.indexOf('</note>', startIdx);
|
||||
if (closeIdx === -1) {
|
||||
if (startIdx > 0) carry = carry.slice(startIdx);
|
||||
break;
|
||||
}
|
||||
|
||||
const singleNoteXml = carry.slice(startIdx, closeIdx + 7);
|
||||
carry = carry.slice(closeIdx + 7);
|
||||
totalParsed++;
|
||||
|
||||
try {
|
||||
const rawNote = await parseSingleNoteXml(singleNoteXml);
|
||||
const res = await processNote(rawNote, notebookId, options.userId || 'default');
|
||||
if (res.status === 'created') {
|
||||
importedCount++;
|
||||
} else if (res.status === 'skipped') {
|
||||
skippedCount++;
|
||||
}
|
||||
} catch (err) {
|
||||
errorCount++;
|
||||
console.error(`\nError saving note #${totalParsed}:`, err.message);
|
||||
}
|
||||
|
||||
if (totalParsed % 10 === 0 || totalParsed === 1) {
|
||||
const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
const rate = (totalParsed / (elapsedSec || 1)).toFixed(1);
|
||||
process.stdout.write(
|
||||
`\r [${notebookName}] Notes: ${totalParsed} (${importedCount} new, ${skippedCount} skipped, ${errorCount} errors) - ${elapsedSec}s (${rate} notes/sec)`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const totalSec = ((Date.now() - startTime) / 1000).toFixed(1);
|
||||
console.log(
|
||||
`\nCompleted "${notebookName}": ${totalParsed} total notes (${importedCount} new, ${skippedCount} skipped, ${errorCount} errors) in ${totalSec}s.\n`
|
||||
);
|
||||
|
||||
return {
|
||||
notebookName,
|
||||
stackName,
|
||||
totalParsed,
|
||||
importedCount,
|
||||
skippedCount,
|
||||
errorCount,
|
||||
elapsedSeconds: parseFloat(totalSec),
|
||||
};
|
||||
}
|
||||
|
||||
function findEnexFiles(dir) {
|
||||
const results = [];
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results.push(...findEnexFiles(fullPath));
|
||||
} else if (entry.isFile() && entry.name.endsWith('.enex')) {
|
||||
const stat = fs.statSync(fullPath);
|
||||
results.push({ path: fullPath, name: entry.name, size: stat.size });
|
||||
}
|
||||
}
|
||||
return results.sort((a, b) => a.size - b.size);
|
||||
}
|
||||
|
||||
// CLI entrypoint
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
const flag = args[0];
|
||||
|
||||
if (!flag) {
|
||||
console.log(`Usage:`);
|
||||
console.log(` node scripts/migrate.js --batch-a # Run 23 small notebooks (<10MB)`);
|
||||
console.log(` node scripts/migrate.js --batch-b # Run 25 medium notebooks (10MB-500MB)`);
|
||||
console.log(` node scripts/migrate.js --batch-c # Run 9 large notebooks (>=500MB)`);
|
||||
console.log(` node scripts/migrate.js --all # Run all 57 notebooks sequentially`);
|
||||
console.log(` node scripts/migrate.js <filename> # Run a specific .enex file`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const allFiles = findEnexFiles(DEFAULT_SOURCE_DIR);
|
||||
|
||||
let targetFiles = [];
|
||||
if (flag === '--batch-a') {
|
||||
targetFiles = allFiles.filter((f) => f.size < 10 * 1024 * 1024);
|
||||
} else if (flag === '--batch-b') {
|
||||
targetFiles = allFiles.filter((f) => f.size >= 10 * 1024 * 1024 && f.size < 500 * 1024 * 1024);
|
||||
} else if (flag === '--batch-c') {
|
||||
targetFiles = allFiles.filter((f) => f.size >= 500 * 1024 * 1024);
|
||||
} else if (flag === '--all') {
|
||||
targetFiles = allFiles;
|
||||
} else {
|
||||
const fullPath = path.isAbsolute(flag) ? flag : path.join(DEFAULT_SOURCE_DIR, flag);
|
||||
if (!fs.existsSync(fullPath)) {
|
||||
console.error(`File not found: ${fullPath}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const stat = fs.statSync(fullPath);
|
||||
targetFiles = [{ path: fullPath, name: path.basename(fullPath), size: stat.size }];
|
||||
}
|
||||
|
||||
console.log(`\n======================================================`);
|
||||
console.log(`Starting migration for ${targetFiles.length} file(s)...`);
|
||||
console.log(`Total payload size: ${(targetFiles.reduce((acc, f) => acc + f.size, 0) / (1024 * 1024)).toFixed(1)} MB`);
|
||||
console.log(`======================================================\n`);
|
||||
|
||||
let grandTotalParsed = 0;
|
||||
let grandTotalImported = 0;
|
||||
let grandTotalSkipped = 0;
|
||||
let grandTotalErrors = 0;
|
||||
const grandStartTime = Date.now();
|
||||
|
||||
try {
|
||||
for (let i = 0; i < targetFiles.length; i++) {
|
||||
const f = targetFiles[i];
|
||||
console.log(`[Batch Progress: ${i + 1}/${targetFiles.length}] Processing ${f.name}...`);
|
||||
const res = await importEnexFile(f.path);
|
||||
grandTotalParsed += res.totalParsed;
|
||||
grandTotalImported += res.importedCount;
|
||||
grandTotalSkipped += res.skippedCount;
|
||||
grandTotalErrors += res.errorCount;
|
||||
}
|
||||
|
||||
const totalMinutes = ((Date.now() - grandStartTime) / 60000).toFixed(2);
|
||||
console.log(`\n======================================================`);
|
||||
console.log(`Batch Summary:`);
|
||||
console.log(` Files Processed: ${targetFiles.length}`);
|
||||
console.log(` Total Notes: ${grandTotalParsed}`);
|
||||
console.log(` New Notes: ${grandTotalImported}`);
|
||||
console.log(` Skipped Notes: ${grandTotalSkipped}`);
|
||||
console.log(` Errors: ${grandTotalErrors}`);
|
||||
console.log(` Total Runtime: ${totalMinutes} minutes`);
|
||||
console.log(`======================================================\n`);
|
||||
} catch (e) {
|
||||
console.error('Batch migration encountered fatal error:', e);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await prisma.$disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { importEnexFile, getOrCreateNotebookWithStack, processNote };
|
||||
+2
-1
@@ -54,7 +54,8 @@ export async function saveAttachmentBuffer(
|
||||
|
||||
export function getFullAttachmentPath(storagePath: string): string {
|
||||
const baseDir = getAttachmentsDir();
|
||||
const resolved = path.resolve(baseDir, storagePath);
|
||||
const normalizedStoragePath = storagePath.replace(/\\/g, '/');
|
||||
const resolved = path.resolve(baseDir, normalizedStoragePath);
|
||||
if (!resolved.startsWith(path.resolve(baseDir))) {
|
||||
throw new Error('Invalid storage path');
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user