From e1a861faea0266d52222e1c6070a7c7124a6e993 Mon Sep 17 00:00:00 2001 From: Chris Fuller Date: Thu, 22 Oct 2026 17:25:23 -0600 Subject: [PATCH] Add high-scale optimizations: pg_trgm GIN indexes, infinite scroll pagination, virtualized card grid, fast groupBy sidebar counts --- entrypoint.sh | 21 ++ src/app/api/sidebar/route.ts | 48 ++- src/components/EvercloneWorkspace.tsx | 503 +++++++++++++++----------- 3 files changed, 350 insertions(+), 222 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index a9db91d..21b56f2 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,5 +4,26 @@ set -e echo "Applying Prisma schema to PostgreSQL..." node ./node_modules/prisma/build/index.js db push --skip-generate --accept-data-loss +echo "Ensuring high-scale PostgreSQL pg_trgm GIN indexes..." +node -e ' +const { PrismaClient } = require("@prisma/client"); +const prisma = new PrismaClient(); +async function main() { + try { + await prisma.$executeRawUnsafe(`CREATE EXTENSION IF NOT EXISTS pg_trgm;`); + await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS note_title_trgm_idx ON "Note" USING gin ("title" gin_trgm_ops);`); + await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS note_content_text_trgm_idx ON "Note" USING gin ("contentText" gin_trgm_ops);`); + await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS attachment_extracted_text_trgm_idx ON "Attachment" USING gin ("extractedText" gin_trgm_ops);`); + await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS tag_name_trgm_idx ON "Tag" USING gin ("name" gin_trgm_ops);`); + console.log("pg_trgm GIN indexes verified successfully."); + } catch (err) { + console.error("Warning ensuring pg_trgm indexes:", err.message); + } finally { + await prisma.$disconnect(); + } +} +main(); +' + echo "Starting Everclone server on port 3000..." exec node server.js diff --git a/src/app/api/sidebar/route.ts b/src/app/api/sidebar/route.ts index ccd3a1c..f1de9e5 100644 --- a/src/app/api/sidebar/route.ts +++ b/src/app/api/sidebar/route.ts @@ -16,6 +16,8 @@ export async function GET() { shortcutNotes, allNotesCount, trashCount, + notebookCountsRaw, + tagCountsRaw, ] = await Promise.all([ prisma.user.findMany({ orderBy: { createdAt: 'asc' }, @@ -27,24 +29,10 @@ export async function GET() { prisma.notebook.findMany({ where: { userId }, orderBy: [{ isDefault: 'desc' }, { sortOrder: 'asc' }, { name: 'asc' }], - include: { - _count: { - select: { - notes: { where: { isTrashed: false } }, - }, - }, - }, }), prisma.tag.findMany({ where: { userId }, orderBy: { name: 'asc' }, - include: { - _count: { - select: { - notes: { where: { note: { isTrashed: false, userId } } }, - }, - }, - }, }), prisma.savedSearch.findMany({ where: { userId }, @@ -57,25 +45,49 @@ export async function GET() { }), prisma.note.count({ where: { userId, isTrashed: false } }), prisma.note.count({ where: { userId, isTrashed: true } }), + // Fast single GROUP BY for notebook note counts + prisma.note.groupBy({ + by: ['notebookId'], + where: { userId, isTrashed: false }, + _count: { _all: true }, + }), + // Fast single raw GROUP BY query for tag note counts + prisma.$queryRaw>` + SELECT nt."tagId", COUNT(nt."noteId") as count + FROM "NoteTag" nt + JOIN "Note" n ON n.id = nt."noteId" + WHERE n."userId" = ${userId} AND n."isTrashed" = false + GROUP BY nt."tagId" + `, ]); + const notebookCountMap = new Map(); + for (const item of notebookCountsRaw) { + notebookCountMap.set(item.notebookId, item._count._all); + } + + const tagCountMap = new Map(); + for (const item of tagCountsRaw) { + tagCountMap.set(item.tagId, Number(item.count)); + } + return NextResponse.json({ currentUser, users, stacks, notebooks: notebooks.map((nb) => ({ ...nb, - noteCount: nb._count.notes, + noteCount: notebookCountMap.get(nb.id) || 0, })), tags: tags.map((t) => ({ ...t, - noteCount: t._count.notes, + noteCount: tagCountMap.get(t.id) || 0, })), savedSearches, shortcuts: { notes: shortcutNotes, - notebooks: notebooks.filter((n) => n.isShortcut), - tags: tags.filter((t) => t.isShortcut), + notebooks: notebooks.filter((n) => n.isShortcut).map(nb => ({ ...nb, noteCount: notebookCountMap.get(nb.id) || 0 })), + tags: tags.filter((t) => t.isShortcut).map(t => ({ ...t, noteCount: tagCountMap.get(t.id) || 0 })), savedSearches: savedSearches.filter((s) => s.isShortcut), }, counts: { diff --git a/src/components/EvercloneWorkspace.tsx b/src/components/EvercloneWorkspace.tsx index deca4b5..d17f953 100644 --- a/src/components/EvercloneWorkspace.tsx +++ b/src/components/EvercloneWorkspace.tsx @@ -142,6 +142,8 @@ export default function EvercloneWorkspace() { const [notes, setNotes] = useState([]); const [totalNotes, setTotalNotes] = useState(0); const [loadingList, setLoadingList] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const [hasMoreNotes, setHasMoreNotes] = useState(true); const [selectedNoteId, setSelectedNoteId] = useState(''); const [activeNote, setActiveNote] = useState(null); const [saveState, setSaveState] = useState<'saved' | 'saving' | 'dirty'>('saved'); @@ -189,64 +191,41 @@ export default function EvercloneWorkspace() { } }, []); - const fetchNotesList = useCallback( - async (preferSelectNoteId?: string) => { - setLoadingList(true); - try { - const sp = new URLSearchParams(); - if (debouncedSearch.trim()) sp.set('q', debouncedSearch.trim()); - if (activeView === 'notebook' && selectedNotebookId) { - sp.set('notebookId', selectedNotebookId); - } else if (filterNotebookId) { - sp.set('notebookId', filterNotebookId); - } - if (activeView === 'stack' && selectedStackId) { - sp.set('stackId', selectedStackId); - } - if (activeView === 'tag' && selectedTagId) { - sp.set('tagId', selectedTagId); - } - if (filterTagIds.length > 0) { - sp.set('tagIds', filterTagIds.join(',')); - } - if (filterUntagged) sp.set('untagged', 'true'); - if (filterHasImages) sp.set('hasImages', 'true'); - if (filterHasAttachments) sp.set('hasAttachments', 'true'); - if (filterHasTodo) sp.set('hasTodo', 'true'); - if (filterHasWebClip) sp.set('hasWebClip', 'true'); - if (filterOnlyPinned) sp.set('onlyPinned', 'true'); - if (filterUpdatedDays > 0) sp.set('updatedSinceDays', String(filterUpdatedDays)); - if (activeView === 'trash') { - sp.set('trashed', 'true'); - } else if (activeView === 'shortcuts') { - sp.set('shortcut', 'true'); - } - sp.set('sortBy', sortBy); - sp.set('sortDir', sortDir); - sp.set('limit', '1000'); - - const res = await fetch(`/api/notes?${sp.toString()}`); - if (!res.ok) return; - const data = await res.json(); - const loadedNotes: NoteSummary[] = data.notes || []; - setNotes(loadedNotes); - setTotalNotes(data.total || 0); - - if (preferSelectNoteId) { - setSelectedNoteId(preferSelectNoteId); - } else if (loadedNotes.length > 0) { - setSelectedNoteId((prev) => - prev && loadedNotes.some((n) => n.id === prev) ? prev : loadedNotes[0].id - ); - } else { - setSelectedNoteId(''); - setActiveNote(null); - } - } catch (e) { - console.error('Failed to fetch notes:', e); - } finally { - setLoadingList(false); + const buildNoteQueryParams = useCallback( + (offset = 0, limit = 150) => { + const sp = new URLSearchParams(); + if (debouncedSearch.trim()) sp.set('q', debouncedSearch.trim()); + if (activeView === 'notebook' && selectedNotebookId) { + sp.set('notebookId', selectedNotebookId); + } else if (filterNotebookId) { + sp.set('notebookId', filterNotebookId); } + if (activeView === 'stack' && selectedStackId) { + sp.set('stackId', selectedStackId); + } + if (activeView === 'tag' && selectedTagId) { + sp.set('tagId', selectedTagId); + } + if (filterTagIds.length > 0) { + sp.set('tagIds', filterTagIds.join(',')); + } + if (filterUntagged) sp.set('untagged', 'true'); + if (filterHasImages) sp.set('hasImages', 'true'); + if (filterHasAttachments) sp.set('hasAttachments', 'true'); + if (filterHasTodo) sp.set('hasTodo', 'true'); + if (filterHasWebClip) sp.set('hasWebClip', 'true'); + if (filterOnlyPinned) sp.set('onlyPinned', 'true'); + if (filterUpdatedDays > 0) sp.set('updatedSinceDays', String(filterUpdatedDays)); + if (activeView === 'trash') { + sp.set('trashed', 'true'); + } else if (activeView === 'shortcuts') { + sp.set('shortcut', 'true'); + } + sp.set('sortBy', sortBy); + sp.set('sortDir', sortDir); + sp.set('limit', String(limit)); + sp.set('offset', String(offset)); + return sp; }, [ debouncedSearch, @@ -268,6 +247,65 @@ export default function EvercloneWorkspace() { ] ); + const fetchNotesList = useCallback( + async (preferSelectNoteId?: string) => { + setLoadingList(true); + try { + const sp = buildNoteQueryParams(0, 150); + const res = await fetch(`/api/notes?${sp.toString()}`); + if (!res.ok) return; + const data = await res.json(); + const loadedNotes: NoteSummary[] = data.notes || []; + setNotes(loadedNotes); + setTotalNotes(data.total || 0); + setHasMoreNotes(loadedNotes.length < (data.total || 0)); + + if (preferSelectNoteId) { + setSelectedNoteId(preferSelectNoteId); + } else if (loadedNotes.length > 0) { + setSelectedNoteId((prev) => + prev && loadedNotes.some((n) => n.id === prev) ? prev : loadedNotes[0].id + ); + } else { + setSelectedNoteId(''); + setActiveNote(null); + } + } catch (e) { + console.error('Failed to fetch notes:', e); + } finally { + setLoadingList(false); + } + }, + [buildNoteQueryParams] + ); + + const fetchMoreNotes = useCallback(async () => { + if (loadingMore || !hasMoreNotes || loadingList) return; + setLoadingMore(true); + try { + const sp = buildNoteQueryParams(notes.length, 150); + const res = await fetch(`/api/notes?${sp.toString()}`); + if (!res.ok) return; + const data = await res.json(); + const nextBatch: NoteSummary[] = data.notes || []; + if (nextBatch.length === 0) { + setHasMoreNotes(false); + } else { + setNotes((prev) => { + const existingIds = new Set(prev.map((n) => n.id)); + const unique = nextBatch.filter((n) => !existingIds.has(n.id)); + const combined = [...prev, ...unique]; + setHasMoreNotes(combined.length < (data.total || 0)); + return combined; + }); + } + } catch (e) { + console.error('Failed to fetch next page of notes:', e); + } finally { + setLoadingMore(false); + } + }, [loadingMore, hasMoreNotes, loadingList, buildNoteQueryParams, notes.length]); + useEffect(() => { fetchSidebar(); }, [fetchSidebar]); @@ -628,15 +666,38 @@ export default function EvercloneWorkspace() { } }; - // Virtualizer for middle note list pane + // Card grid rows: 2 cards per row + const cardGridRowCount = Math.ceil(notes.length / 2); + + // Virtualizer for middle note list pane (handles Snippets, Compact list, and 2-column Card grid rows) const rowVirtualizer = useVirtualizer({ - count: notes.length, + count: listMode === 'cards' ? cardGridRowCount : notes.length, getScrollElement: () => listParentRef.current, estimateSize: () => - listMode === 'compact' ? 52 : listMode === 'cards' ? 148 : 112, - overscan: 10, + listMode === 'compact' ? 52 : listMode === 'cards' ? 240 : 112, + overscan: 8, }); + // Infinite scroll trigger: when scrolling near bottom, fetch next page of notes + const virtualItems = rowVirtualizer.getVirtualItems(); + useEffect(() => { + if (!virtualItems.length || !hasMoreNotes || loadingMore || loadingList) return; + const lastItem = virtualItems[virtualItems.length - 1]; + const totalVirtual = listMode === 'cards' ? cardGridRowCount : notes.length; + if (lastItem && lastItem.index >= totalVirtual - 6) { + fetchMoreNotes(); + } + }, [ + virtualItems, + hasMoreNotes, + loadingMore, + loadingList, + listMode, + cardGridRowCount, + notes.length, + fetchMoreNotes, + ]); + // Compute current view title const currentViewTitle = (() => { if (activeView === 'trash') return 'Trash'; @@ -1584,6 +1645,13 @@ export default function EvercloneWorkspace() { + {/* Loading indicator for next page */} + {loadingMore && ( +
+ Loading more notes… +
+ )} + {/* Collapsible Filter Panel */} {showFilterPanel && (
@@ -1984,159 +2052,186 @@ export default function EvercloneWorkspace() { )}
) : listMode === 'cards' ? ( - /* 2-COLUMN VISUAL CARD GRID VIEW */ -
- {notes.map((n) => { - const isSelected = n.id === selectedNoteId; - const firstImg = n.attachments.find((a) => - a.mimeType.startsWith('image/') - ); - const firstPdf = n.attachments.find( - (a) => - a.mimeType === 'application/pdf' || - a.filename.toLowerCase().endsWith('.pdf') - ); - const effectiveThumbUrl = - n.thumbnailUrl === 'none' - ? null - : n.thumbnailUrl || (firstImg ? `/api/attachments/${firstImg.id}` : null); + /* VIRTUALIZED 2-COLUMN VISUAL CARD GRID VIEW */ +
+ {rowVirtualizer.getVirtualItems().map((virtualRow) => { + const startIndex = virtualRow.index * 2; + const rowNotes = notes.slice(startIndex, startIndex + 2); return (
{ - e.dataTransfer.setData('text/everclone-note-id', n.id); + key={`card-row-${virtualRow.index}`} + data-index={virtualRow.index} + ref={rowVirtualizer.measureElement} + style={{ + position: 'absolute', + top: 0, + left: 0, + width: '100%', + transform: `translateY(${virtualRow.start}px)`, }} - onClick={() => openNoteInEditor(n.id)} - className={`group/card cursor-pointer rounded-xl border transition flex flex-col overflow-hidden shadow-xs ${ - isSelected - ? 'bg-emerald-50/60 border-evernote-green ring-2 ring-evernote-green/30' - : 'bg-white border-gray-200/90 hover:border-gray-300 hover:shadow-md' - }`} + className="grid grid-cols-1 sm:grid-cols-2 gap-3 p-3" > - {/* Top Hero Cover Image Banner (if present) */} - {effectiveThumbUrl ? ( -
- - {firstPdf && ( - - PDF - - )} - -
- ) : ( - n.thumbnailUrl !== 'none' && - firstPdf && ( - { - setSelectedNoteId(n.id); - setThumbnailInitialUrl(dataUrl); - setThumbnailEditorOpen(true); - }} - onCachedThumbnail={(nid, dataUrl) => { - setNotes((prev) => - prev.map((item) => - item.id === nid ? { ...item, thumbnailUrl: dataUrl } : item - ) - ); - }} - /> - ) - )} + {rowNotes.map((n) => { + const isSelected = n.id === selectedNoteId; + const firstImg = n.attachments.find((a) => + a.mimeType.startsWith('image/') + ); + const firstPdf = n.attachments.find( + (a) => + a.mimeType === 'application/pdf' || + a.filename.toLowerCase().endsWith('.pdf') + ); + const effectiveThumbUrl = + n.thumbnailUrl === 'none' + ? null + : n.thumbnailUrl || (firstImg ? `/api/attachments/${firstImg.id}` : null); - {/* Card Content */} -
-
- {/* Notebook Mover & Pin/Star Row */} -
-
e.stopPropagation()} - className="inline-flex items-center gap-1 min-w-0 max-w-[140px] px-1.5 py-0.5 rounded bg-gray-100 hover:bg-emerald-50 text-gray-600 hover:text-evernote-green transition" - title="Move note to another notebook" - > - - -
-
- {n.isPinned && ( - - )} - {n.isShortcut && ( - - )} -
-
- -

- {n.title || 'Untitled Note'} -

- -

{ + e.dataTransfer.setData('text/everclone-note-id', n.id); + }} + onClick={() => openNoteInEditor(n.id)} + className={`group/card cursor-pointer rounded-xl border transition flex flex-col overflow-hidden shadow-xs ${ + isSelected + ? 'bg-emerald-50/60 border-evernote-green ring-2 ring-evernote-green/30' + : 'bg-white border-gray-200/90 hover:border-gray-300 hover:shadow-md' }`} > - {n.excerpt || 'No additional text'} -

-
- - {/* Card Footer */} -
- - {format(new Date(n.updatedAt), 'MMM d')} - - -
- {n.attachments.length > 0 && ( - - - {n.attachments.length} - + {/* Top Hero Cover Image Banner (if present) */} + {effectiveThumbUrl ? ( +
+ + {firstPdf && ( + + PDF + + )} + +
+ ) : ( + n.thumbnailUrl !== 'none' && + firstPdf && ( + { + setSelectedNoteId(n.id); + setThumbnailInitialUrl(dataUrl); + setThumbnailEditorOpen(true); + }} + onCachedThumbnail={(nid, dataUrl) => { + setNotes((prev) => + prev.map((item) => + item.id === nid ? { ...item, thumbnailUrl: dataUrl } : item + ) + ); + }} + /> + ) )} - {n.tags.slice(0, 1).map((t) => ( - - #{t.name} - - ))} + + {/* Card Content */} +
+
+ {/* Notebook Mover & Pin/Star Row */} +
+
e.stopPropagation()} + className="inline-flex items-center gap-1 min-w-0 max-w-[140px] px-1.5 py-0.5 rounded bg-gray-100 hover:bg-emerald-50 text-gray-600 hover:text-evernote-green transition" + title="Move note to another notebook" + > + + +
+
+ {n.isPinned && ( + + )} + {n.isShortcut && ( + + )} +
+
+ +

+ {n.title || 'Untitled Note'} +

+ +

+ {n.excerpt || 'No additional text'} +

+
+ + {/* Card Footer */} +
+ + {format(new Date(n.updatedAt), 'MMM d')} + + +
+ {n.attachments.length > 0 && ( + + + {n.attachments.length} + + )} + {n.tags.slice(0, 1).map((t) => ( + + #{t.name} + + ))} +
+
+
-
-
+ ); + })}
); })}