import sax from 'sax'; import { marked } from 'marked'; import { JSDOM } from 'jsdom'; import { Readability } from '@mozilla/readability'; import { prisma, getCurrentUserId, ensureDefaultNotebook } from './prisma'; import { saveAttachmentBuffer, stripHtmlToText, makeExcerpt, guessExtensionFromMime, } from './storage'; export function parseEvernoteTimestamp(ts?: string | null): Date | null { if (!ts) return null; const trimmed = ts.trim(); const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z?$/i.exec(trimmed); if (m) { const iso = `${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}Z`; const d = new Date(iso); if (!isNaN(d.getTime())) return d; } const fallback = new Date(trimmed); return isNaN(fallback.getTime()) ? null : fallback; } export function toEvernoteTimestamp(date: Date): string { return date.toISOString().replace(/[-:]/g, '').replace(/\.\d{3}Z$/, 'Z'); } interface ParsedResource { base64Data: string; mime: string; width?: number; height?: number; fileName?: string; } interface ParsedEnexNote { title: string; contentEnml: string; created?: string; updated?: string; tags: string[]; sourceUrl?: string; author?: string; latitude?: number; longitude?: number; reminderTime?: string; resources: ParsedResource[]; } export async function getOrCreateNotebookByName( name: string, explicitUserId?: string ): Promise { const userId = explicitUserId || (await getCurrentUserId()); const cleanName = (name || 'Imported Notes').trim(); const existing = await prisma.notebook.findFirst({ where: { userId, name: { equals: cleanName, mode: 'insensitive' } }, }); if (existing) return existing.id; const created = await prisma.notebook.create({ data: { userId, name: cleanName }, }); return created.id; } export async function getOrCreateTagIds( tagNames: string[], explicitUserId?: string ): Promise { const userId = explicitUserId || (await getCurrentUserId()); const uniqueNames = Array.from( new Set(tagNames.map((t) => t.trim()).filter((t) => t.length > 0)) ); const ids: string[] = []; for (const name of uniqueNames) { let tag = await prisma.tag.findFirst({ where: { userId, name: { equals: name, mode: 'insensitive' } }, }); if (!tag) { tag = await prisma.tag.create({ data: { userId, name }, }); } ids.push(tag.id); } return ids; } export function convertEnmlToHtml( enml: string, mediaByHash: Map< string, { id: string; filename: string; mimeType: string; sizeBytes: number } > ): string { if (!enml) return ''; let html = enml .replace(/<\?xml[^>]*\?>/gi, '') .replace(/]*>/gi, '') .replace(/]*>/gi, '
') .replace(/<\/en-note>/gi, '
'); html = html.replace(/]*?)\/?>/gi, (_match, attrs: string) => { const isChecked = /checked\s*=\s*["']true["']/i.test(attrs); return ` `; }); html = html.replace( /]*?)(?:\/>|><\/en-media>)/gi, (_match, attrs: string) => { const hashMatch = /hash\s*=\s*["']([a-fA-F0-9]+)["']/i.exec(attrs); const typeMatch = /type\s*=\s*["']([^"']+)["']/i.exec(attrs); const hash = hashMatch ? hashMatch[1].toLowerCase() : ''; const mime = typeMatch ? typeMatch[1].toLowerCase() : ''; const attachment = hash ? mediaByHash.get(hash) : undefined; if (!attachment) { return `[Missing media ${hash.slice(0, 8)}]`; } const url = `/api/attachments/${attachment.id}`; const effectiveMime = attachment.mimeType || mime; if (effectiveMime.startsWith('image/')) { return `${escapeHtml(attachment.filename)}`; } if (effectiveMime.startsWith('audio/')) { return `

🎵 ${escapeHtml(attachment.filename)}

`; } if (effectiveMime === 'application/pdf') { return ``; } return ``; } ); return html.trim(); } function escapeHtml(str: string): string { return str .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"'); } export async function saveParsedEnexNote( parsed: ParsedEnexNote, notebookId: string ): Promise { const userId = await getCurrentUserId(); const createdAt = parseEvernoteTimestamp(parsed.created) || new Date(); const updatedAt = parseEvernoteTimestamp(parsed.updated) || createdAt; const reminderAt = parseEvernoteTimestamp(parsed.reminderTime); const tagIds = await getOrCreateTagIds(parsed.tags, userId); const note = await prisma.note.create({ data: { userId, title: parsed.title?.trim() || 'Untitled Note', contentHtml: '', contentText: '', excerpt: '', notebookId, sourceUrl: parsed.sourceUrl || null, author: parsed.author || null, latitude: parsed.latitude ?? null, longitude: parsed.longitude ?? null, reminderAt: reminderAt ?? null, createdAt, updatedAt, tags: { create: tagIds.map((tagId) => ({ tagId })), }, }, }); const mediaByHash = new Map< string, { id: string; filename: string; mimeType: string; sizeBytes: number } >(); for (let i = 0; i < parsed.resources.length; i++) { const res = parsed.resources[i]; if (!res.base64Data) continue; const cleanB64 = res.base64Data.replace(/\s+/g, ''); const buffer = Buffer.from(cleanB64, 'base64'); if (buffer.length === 0) continue; const mimeType = res.mime || 'application/octet-stream'; const ext = guessExtensionFromMime(mimeType); const filename = res.fileName?.trim() || `attachment-${i + 1}${ext}`; const savedBlob = await saveAttachmentBuffer(buffer, filename, mimeType); const attachment = await prisma.attachment.create({ data: { noteId: note.id, filename, mimeType, sizeBytes: savedBlob.sizeBytes, md5Hash: savedBlob.md5Hash, storagePath: savedBlob.storagePath, width: res.width ?? null, height: res.height ?? null, extractedText: savedBlob.extractedText, }, }); mediaByHash.set(savedBlob.md5Hash.toLowerCase(), { id: attachment.id, filename: attachment.filename, mimeType: attachment.mimeType, sizeBytes: attachment.sizeBytes, }); } const contentHtml = convertEnmlToHtml(parsed.contentEnml, mediaByHash); const contentText = stripHtmlToText(contentHtml); const excerpt = makeExcerpt(contentText); await prisma.note.update({ where: { id: note.id }, data: { contentHtml, contentText, excerpt, updatedAt, }, }); return note.id; } export async function importEnexXml( xmlContent: string, options: { notebookId?: string; notebookName?: string } ): Promise<{ importedCount: number; errors: string[]; noteIds: string[] }> { const notebookId = options.notebookId || (options.notebookName ? await getOrCreateNotebookByName(options.notebookName) : (await ensureDefaultNotebook()).id); const parsedNotes: ParsedEnexNote[] = []; const errors: string[] = []; await new Promise((resolve) => { const parser = sax.parser(false, { lowercase: true, trim: false }); let currentNote: ParsedEnexNote | null = null; let currentResource: ParsedResource | null = null; const tagStack: string[] = []; let textBuffer = ''; parser.onopentag = (node) => { tagStack.push(node.name); textBuffer = ''; if (node.name === 'note') { currentNote = { title: 'Untitled Note', contentEnml: '', tags: [], resources: [], }; } else if (node.name === 'resource' && currentNote) { currentResource = { base64Data: '', mime: 'application/octet-stream', }; } }; parser.ontext = (t) => { textBuffer += t; }; parser.oncdata = (cdata) => { textBuffer += cdata; }; parser.onclosetag = (tagName) => { const parentTag = tagStack[tagStack.length - 2] || ''; const val = textBuffer.trim(); if (currentNote) { if (currentResource) { if (tagName === 'data') { currentResource.base64Data = val; } else if (tagName === 'mime') { currentResource.mime = val; } else if (tagName === 'width') { const w = parseInt(val, 10); if (!isNaN(w)) currentResource.width = w; } else if (tagName === 'height') { const h = parseInt(val, 10); if (!isNaN(h)) currentResource.height = h; } else if (tagName === 'file-name') { currentResource.fileName = val; } else if (tagName === 'resource') { currentNote.resources.push(currentResource); currentResource = null; } } else { if (tagName === 'title' && parentTag === 'note') { currentNote.title = val || 'Untitled Note'; } else if (tagName === 'content' && parentTag === 'note') { currentNote.contentEnml = textBuffer; } else if (tagName === 'created' && parentTag === 'note') { currentNote.created = val; } else if (tagName === 'updated' && parentTag === 'note') { currentNote.updated = val; } else if (tagName === 'tag' && parentTag === 'note') { if (val) currentNote.tags.push(val); } else if (tagName === 'source-url' && parentTag === 'note-attributes') { currentNote.sourceUrl = val; } else if (tagName === 'author' && parentTag === 'note-attributes') { currentNote.author = val; } else if (tagName === 'latitude' && parentTag === 'note-attributes') { const lat = parseFloat(val); if (!isNaN(lat)) currentNote.latitude = lat; } else if (tagName === 'longitude' && parentTag === 'note-attributes') { const lon = parseFloat(val); if (!isNaN(lon)) currentNote.longitude = lon; } else if (tagName === 'reminder-time' && parentTag === 'note-attributes') { currentNote.reminderTime = val; } else if (tagName === 'note') { parsedNotes.push(currentNote); currentNote = null; } } } tagStack.pop(); textBuffer = ''; }; parser.onerror = (err) => { errors.push(err.message); parser.resume(); }; parser.onend = () => { resolve(); }; parser.write(xmlContent).close(); }); const noteIds: string[] = []; for (const pNote of parsedNotes) { try { const id = await saveParsedEnexNote(pNote, notebookId); noteIds.push(id); } catch (e: unknown) { errors.push( `Failed to save "${pNote.title}": ${e instanceof Error ? e.message : String(e)}` ); } } return { importedCount: noteIds.length, errors, noteIds }; } export async function importMarkdownNote( mdContent: string, filename: string, notebookId: string ): Promise { let body = mdContent.replace(/\r\n/g, '\n'); let title = filename.replace(/\.md$/i, '').trim() || 'Untitled Note'; let createdAt: Date | null = null; let updatedAt: Date | null = null; let sourceUrl: string | null = null; const tags: string[] = []; if (body.startsWith('---\n')) { const endIdx = body.indexOf('\n---\n', 4); if (endIdx !== -1) { const fm = body.slice(4, endIdx); body = body.slice(endIdx + 5); for (const line of fm.split('\n')) { const m = /^([a-zA-Z0-9_-]+):\s*(.+)$/.exec(line.trim()); if (!m) continue; const key = m[1].toLowerCase(); const val = m[2].replace(/^["']|["']$/g, '').trim(); if (key === 'title' && val) title = val; if ((key === 'created' || key === 'date' || key === 'created_at') && val) { createdAt = parseEvernoteTimestamp(val); } if ((key === 'updated' || key === 'updated_at') && val) { updatedAt = parseEvernoteTimestamp(val); } if ((key === 'source' || key === 'source_url' || key === 'url') && val) { sourceUrl = val; } if (key === 'tags') { const clean = val.replace(/^\[|\]$/g, ''); for (const t of clean.split(',')) { const trimmed = t.replace(/^["']|["']$/g, '').trim(); if (trimmed) tags.push(trimmed); } } } } } const headingMatch = /^#\s+(.+)\n/.exec(body.trimStart()); if (headingMatch) { title = headingMatch[1].trim(); body = body.trimStart().slice(headingMatch[0].length); } const createdMatch = /\*\*Created:\*\*\s*([^\n]+)\n?/i.exec(body); if (createdMatch) { createdAt = parseEvernoteTimestamp(createdMatch[1].trim()) || createdAt; body = body.replace(createdMatch[0], ''); } const updatedMatch = /\*\*Updated:\*\*\s*([^\n]+)\n?/i.exec(body); if (updatedMatch) { updatedAt = parseEvernoteTimestamp(updatedMatch[1].trim()) || updatedAt; body = body.replace(updatedMatch[0], ''); } const tagsMatch = /\*\*Tags:\*\*\s*([^\n]+)\n?/i.exec(body); if (tagsMatch) { for (const t of tagsMatch[1].split(',')) { if (t.trim()) tags.push(t.trim()); } body = body.replace(tagsMatch[0], ''); } body = body.replace(/^\s*---\s*\n/, '').trim(); const userId = await getCurrentUserId(); const rawHtml = await marked.parse(body, { gfm: true, breaks: true }); const contentText = stripHtmlToText(rawHtml); const excerpt = makeExcerpt(contentText); const tagIds = await getOrCreateTagIds(tags, userId); const finalCreated = createdAt || new Date(); const finalUpdated = updatedAt || finalCreated; const note = await prisma.note.create({ data: { userId, title, contentHtml: rawHtml, contentText, excerpt, notebookId, sourceUrl, createdAt: finalCreated, updatedAt: finalUpdated, tags: { create: tagIds.map((tagId) => ({ tagId })), }, }, }); return note.id; } /** * Normalizes lazy-loaded images, tags, and image carousels/galleries * BEFORE Readability runs so Readability does not strip article graphics. */ export function prepareDomForReadability(doc: Document, baseUrl: string): { heroImgSrc: string | null } { // 1. Remove duplicate thumbnail/lightbox carousels & author avatars that clutter clips doc .querySelectorAll( '.gallery-thumbnails, .gallery-thumbnails-expanded, [id^="main-expanable-carousel"], [id^="gallery-thumbnails"], [id^="expanded-gallery-thumbnails"], .splide__arrows, .splide__pagination, .w-author, .author-bio, a[href*="/author/"] picture, a[href*="/author/"] img' ) .forEach((el) => el.remove()); const resolveUrl = (raw: string | null | undefined): string => { if (!raw) return ''; const cleaned = raw.replace(/&/g, '&').trim(); if (!cleaned || cleaned.startsWith('data:')) return ''; try { return new URL(cleaned, baseUrl).href; } catch { return cleaned; } }; // 2. Promote lazy-loaded attributes on all elements doc.querySelectorAll('img').forEach((img) => { const candidates = [ img.getAttribute('data-img-url'), img.getAttribute('data-src'), img.getAttribute('data-lazy-src'), img.getAttribute('data-original'), img.getAttribute('data-full-url'), img.getAttribute('src'), ]; for (const c of candidates) { const resolved = resolveUrl(c); if (resolved) { img.setAttribute('src', resolved); break; } } }); // 3. Unwrap elements into clean elements with best resolution source doc.querySelectorAll('picture').forEach((pic) => { const img = pic.querySelector('img') || doc.createElement('img'); if (!img.getAttribute('src')) { const sources = Array.from(pic.querySelectorAll('source')); for (const s of sources) { const srcset = s.getAttribute('srcset') || s.getAttribute('data-srcset'); if (srcset) { const firstUrl = srcset.split(',')[0].trim().split(/\s+/)[0]; const resolved = resolveUrl(firstUrl); if (resolved) { img.setAttribute('src', resolved); break; } } } } if (img.getAttribute('src')) { pic.replaceWith(img); } }); // 4. Rescue lead/hero image before Readability strips
const heroEl = doc.querySelector( '.heading_image img, header figure img, article header img, .post-thumbnail img, .featured-image img' ); const ogImage = doc .querySelector('meta[property="og:image"]') ?.getAttribute('content'); const heroImgSrc = resolveUrl(heroEl?.getAttribute('src')) || resolveUrl(ogImage) || null; // 5. Flatten Valnet / Splide / Swiper / WordPress galleries into a DocumentFragment of //
nodes (not wrapped in a textless
) // so Readability's cleanConditionally("div") never prunes them! doc .querySelectorAll( '.valnet-gallery, .article__gallery, .w-gallery-carousel, .splide-gallery, .custom-image-gallery, .wp-block-gallery, .swiper-wrapper' ) .forEach((gal) => { const frag = doc.createDocumentFragment(); const seenBaseUrls = new Set(); gal.querySelectorAll('img').forEach((img) => { const src = img.getAttribute('src') || ''; const baseKey = src.split('?')[0]; if (!src || seenBaseUrls.has(baseKey)) return; seenBaseUrls.add(baseKey); const fig = doc.createElement('figure'); const cleanImg = doc.createElement('img'); cleanImg.setAttribute('src', src); const alt = img.getAttribute('alt')?.trim() || ''; if (alt) cleanImg.setAttribute('alt', alt); fig.appendChild(cleanImg); if (alt) { const cap = doc.createElement('figcaption'); cap.textContent = alt; fig.appendChild(cap); } frag.appendChild(fig); }); if (seenBaseUrls.size > 0) { gal.replaceWith(frag); } }); return { heroImgSrc }; } /** * Downloads remote images in the clipped HTML, * stores them locally as Attachments on the Note, and rewrites the HTML * to point to `/api/attachments/`. */ export async function localizeClippedImages( noteId: string, contentHtml: string, sourceUrl?: string ): Promise { const dom = new JSDOM(contentHtml); const doc = dom.window.document; const imgs = Array.from(doc.querySelectorAll('img')); if (imgs.length === 0) return contentHtml; const urlToLocalSrc = new Map(); const uniqueUrls: string[] = []; for (const img of imgs) { const src = img.getAttribute('src')?.trim() || ''; if (src.startsWith('http://') || src.startsWith('https://')) { if (!uniqueUrls.includes(src) && uniqueUrls.length < 30) { uniqueUrls.push(src); } } } await Promise.all( uniqueUrls.map(async (remoteUrl, idx) => { try { const res = await fetch(remoteUrl, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', ...(sourceUrl ? { Referer: sourceUrl } : {}), }, signal: AbortSignal.timeout(10000), }); if (!res.ok) return; const arrayBuf = await res.arrayBuffer(); const buffer = Buffer.from(arrayBuf); if (buffer.length < 200) return; // skip tiny 1x1 tracking pixels const contentType = res.headers.get('content-type')?.split(';')[0].trim() || 'image/jpeg'; const ext = guessExtensionFromMime(contentType); let urlName = ''; try { const parsedUrl = new URL(remoteUrl); urlName = parsedUrl.pathname.split('/').pop() || ''; } catch { // ignore } const filename = urlName && /\.[a-zA-Z0-9]{3,4}$/.test(urlName) ? urlName : `clip-image-${idx + 1}${ext}`; const savedBlob = await saveAttachmentBuffer(buffer, filename, contentType); const att = await prisma.attachment.create({ data: { noteId, filename, mimeType: contentType, sizeBytes: savedBlob.sizeBytes, md5Hash: savedBlob.md5Hash, storagePath: savedBlob.storagePath, }, }); urlToLocalSrc.set(remoteUrl, `/api/attachments/${att.id}`); } catch { // Keep remote URL if download fails } }) ); for (const img of imgs) { const src = img.getAttribute('src')?.trim() || ''; const localSrc = urlToLocalSrc.get(src); if (localSrc) { img.setAttribute('src', localSrc); } } return doc.body.innerHTML; } export async function importHtmlNote( htmlString: string, fallbackTitle: string, notebookId: string, sourceUrl?: string ): Promise { const baseUrl = sourceUrl || 'https://example.com'; const dom = new JSDOM(htmlString, { url: baseUrl }); const doc = dom.window.document; const docTitle = doc.querySelector('title')?.textContent?.trim(); const { heroImgSrc } = prepareDomForReadability(doc, baseUrl); const reader = new Readability(doc); const parsed = reader.parse(); const title = parsed?.title?.trim() || docTitle || fallbackTitle.replace(/\.html?$/i, '').trim() || 'Web Clip'; let contentHtml = parsed?.content || doc.body?.innerHTML || htmlString; if (heroImgSrc && !contentHtml.includes(heroImgSrc)) { contentHtml = `

${escapeHtml(title)}

\n${contentHtml}`; } const userId = await getCurrentUserId(); const contentText = stripHtmlToText(contentHtml); const excerpt = makeExcerpt(parsed?.excerpt || contentText); const note = await prisma.note.create({ data: { userId, title, contentHtml, contentText, excerpt, notebookId, sourceUrl: sourceUrl || null, }, }); const localizedHtml = await localizeClippedImages(note.id, contentHtml, sourceUrl); if (localizedHtml !== contentHtml) { await prisma.note.update({ where: { id: note.id }, data: { contentHtml: localizedHtml }, }); } return note.id; } export async function clipUrlToNote( url: string, options: { title?: string; notebookId?: string; tags?: string[]; isShortcut?: boolean; } ): Promise { const notebookId = options.notebookId || (await ensureDefaultNotebook()).id; let html = ''; try { const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', }, signal: AbortSignal.timeout(15000), }); html = await res.text(); } catch { html = `

Saved link: ${escapeHtml(url)}

`; } const dom = new JSDOM(html, { url }); const doc = dom.window.document; const docTitle = doc.title?.trim(); const { heroImgSrc } = prepareDomForReadability(doc, url); const reader = new Readability(doc); const article = reader.parse(); const title = options.title?.trim() || article?.title?.trim() || docTitle || url; let contentHtml = article?.content || `

Clipped from ${escapeHtml(url)}

${doc.body?.innerHTML || ''}`; if (heroImgSrc && !contentHtml.includes(heroImgSrc)) { contentHtml = `

${escapeHtml(title)}

\n${contentHtml}`; } const userId = await getCurrentUserId(); const contentText = stripHtmlToText(contentHtml); const excerpt = makeExcerpt(article?.excerpt || contentText); const tagIds = await getOrCreateTagIds(options.tags || ['web-clip'], userId); const note = await prisma.note.create({ data: { userId, title, contentHtml, contentText, excerpt, notebookId, sourceUrl: url, isShortcut: options.isShortcut ?? false, tags: { create: tagIds.map((tagId) => ({ tagId })), }, }, }); // Download all article images and store them locally as attachments on the note const localizedHtml = await localizeClippedImages(note.id, contentHtml, url); if (localizedHtml !== contentHtml) { await prisma.note.update({ where: { id: note.id }, data: { contentHtml: localizedHtml }, }); } // Auto-sort triage for newly clipped web notes try { const { applyAutoSortToNote } = await import('./autoSort'); await applyAutoSortToNote(note.id, userId); } catch (err) { console.error('Auto-sort for web clip failed (non-fatal):', err); } return note.id; } export function exportNotesToEnex( notes: Array<{ title: string; contentHtml: string; createdAt: Date; updatedAt: Date; sourceUrl?: string | null; author?: string | null; tags: Array<{ tag: { name: string } }>; }> ): string { const exportedDate = toEvernoteTimestamp(new Date()); const noteBlocks = notes .map((n) => { const tagsXml = n.tags .map((nt) => ` ${escapeHtml(nt.tag.name)}`) .join('\n'); const attrsXml: string[] = []; if (n.sourceUrl) { attrsXml.push(` ${escapeHtml(n.sourceUrl)}`); } if (n.author) { attrsXml.push(` ${escapeHtml(n.author)}`); } return ` ${escapeHtml(n.title)} ${n.contentHtml}]]> ${toEvernoteTimestamp(n.createdAt)} ${toEvernoteTimestamp(n.updatedAt)} ${tagsXml ? tagsXml + '\n' : ''}${ attrsXml.length > 0 ? ` \n${attrsXml.join('\n')}\n \n` : '' } `; }) .join('\n'); return ` ${noteBlocks} `; }