Add high-scale optimizations: pg_trgm GIN indexes, infinite scroll pagination, virtualized card grid, fast groupBy sidebar counts

entrypoint.sh

@@ -4,5 +4,26 @@ set -e
 echo "Applying Prisma schema to PostgreSQL..."
 node ./node_modules/prisma/build/index.js db push --skip-generate --accept-data-loss
 
+echo "Ensuring high-scale PostgreSQL pg_trgm GIN indexes..."
+node -e '
+const { PrismaClient } = require("@prisma/client");
+const prisma = new PrismaClient();
+async function main() {
+  try {
+    await prisma.$executeRawUnsafe(`CREATE EXTENSION IF NOT EXISTS pg_trgm;`);
+    await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS note_title_trgm_idx ON "Note" USING gin ("title" gin_trgm_ops);`);
+    await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS note_content_text_trgm_idx ON "Note" USING gin ("contentText" gin_trgm_ops);`);
+    await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS attachment_extracted_text_trgm_idx ON "Attachment" USING gin ("extractedText" gin_trgm_ops);`);
+    await prisma.$executeRawUnsafe(`CREATE INDEX IF NOT EXISTS tag_name_trgm_idx ON "Tag" USING gin ("name" gin_trgm_ops);`);
+    console.log("pg_trgm GIN indexes verified successfully.");
+  } catch (err) {
+    console.error("Warning ensuring pg_trgm indexes:", err.message);
+  } finally {
+    await prisma.$disconnect();
+  }
+}
+main();
+'
+
 echo "Starting Everclone server on port 3000..."
 exec node server.js

src/app/api/sidebar/route.ts

@@ -16,6 +16,8 @@ export async function GET() {
       shortcutNotes,
       allNotesCount,
       trashCount,
+      notebookCountsRaw,
+      tagCountsRaw,
     ] = await Promise.all([
       prisma.user.findMany({
         orderBy: { createdAt: 'asc' },
@@ -27,24 +29,10 @@ export async function GET() {
       prisma.notebook.findMany({
         where: { userId },
         orderBy: [{ isDefault: 'desc' }, { sortOrder: 'asc' }, { name: 'asc' }],
-        include: {
-          _count: {
-            select: {
-              notes: { where: { isTrashed: false } },
-            },
-          },
-        },
       }),
       prisma.tag.findMany({
         where: { userId },
         orderBy: { name: 'asc' },
-        include: {
-          _count: {
-            select: {
-              notes: { where: { note: { isTrashed: false, userId } } },
-            },
-          },
-        },
       }),
       prisma.savedSearch.findMany({
         where: { userId },
@@ -57,25 +45,49 @@ export async function GET() {
       }),
       prisma.note.count({ where: { userId, isTrashed: false } }),
       prisma.note.count({ where: { userId, isTrashed: true } }),
+      // Fast single GROUP BY for notebook note counts
+      prisma.note.groupBy({
+        by: ['notebookId'],
+        where: { userId, isTrashed: false },
+        _count: { _all: true },
+      }),
+      // Fast single raw GROUP BY query for tag note counts
+      prisma.$queryRaw<Array<{ tagId: string; count: bigint }>>`
+        SELECT nt."tagId", COUNT(nt."noteId") as count
+        FROM "NoteTag" nt
+        JOIN "Note" n ON n.id = nt."noteId"
+        WHERE n."userId" = ${userId} AND n."isTrashed" = false
+        GROUP BY nt."tagId"
+      `,
     ]);
 
+    const notebookCountMap = new Map<string, number>();
+    for (const item of notebookCountsRaw) {
+      notebookCountMap.set(item.notebookId, item._count._all);
+    }
+
+    const tagCountMap = new Map<string, number>();
+    for (const item of tagCountsRaw) {
+      tagCountMap.set(item.tagId, Number(item.count));
+    }
+
     return NextResponse.json({
       currentUser,
       users,
       stacks,
       notebooks: notebooks.map((nb) => ({
         ...nb,
-        noteCount: nb._count.notes,
+        noteCount: notebookCountMap.get(nb.id) || 0,
       })),
       tags: tags.map((t) => ({
         ...t,
-        noteCount: t._count.notes,
+        noteCount: tagCountMap.get(t.id) || 0,
       })),
       savedSearches,
       shortcuts: {
         notes: shortcutNotes,
-        notebooks: notebooks.filter((n) => n.isShortcut),
+        notebooks: notebooks.filter((n) => n.isShortcut).map(nb => ({ ...nb, noteCount: notebookCountMap.get(nb.id) || 0 })),
-        tags: tags.filter((t) => t.isShortcut),
+        tags: tags.filter((t) => t.isShortcut).map(t => ({ ...t, noteCount: tagCountMap.get(t.id) || 0 })),
         savedSearches: savedSearches.filter((s) => s.isShortcut),
       },
       counts: {

src/components/EvercloneWorkspace.tsx

@@ -142,6 +142,8 @@ export default function EvercloneWorkspace() {
   const [notes, setNotes] = useState<NoteSummary[]>([]);
   const [totalNotes, setTotalNotes] = useState<number>(0);
   const [loadingList, setLoadingList] = useState<boolean>(true);
+  const [loadingMore, setLoadingMore] = useState<boolean>(false);
+  const [hasMoreNotes, setHasMoreNotes] = useState<boolean>(true);
   const [selectedNoteId, setSelectedNoteId] = useState<string>('');
   const [activeNote, setActiveNote] = useState<NoteDetail | null>(null);
   const [saveState, setSaveState] = useState<'saved' | 'saving' | 'dirty'>('saved');
@@ -189,10 +191,8 @@ export default function EvercloneWorkspace() {
     }
   }, []);
 
-  const fetchNotesList = useCallback(
+  const buildNoteQueryParams = useCallback(
-    async (preferSelectNoteId?: string) => {
+    (offset = 0, limit = 150) => {
-      setLoadingList(true);
-      try {
       const sp = new URLSearchParams();
       if (debouncedSearch.trim()) sp.set('q', debouncedSearch.trim());
       if (activeView === 'notebook' && selectedNotebookId) {
@@ -223,30 +223,9 @@ export default function EvercloneWorkspace() {
       }
       sp.set('sortBy', sortBy);
       sp.set('sortDir', sortDir);
-        sp.set('limit', '1000');
+      sp.set('limit', String(limit));
-
+      sp.set('offset', String(offset));
-        const res = await fetch(`/api/notes?${sp.toString()}`);
+      return sp;
-        if (!res.ok) return;
-        const data = await res.json();
-        const loadedNotes: NoteSummary[] = data.notes || [];
-        setNotes(loadedNotes);
-        setTotalNotes(data.total || 0);
-
-        if (preferSelectNoteId) {
-          setSelectedNoteId(preferSelectNoteId);
-        } else if (loadedNotes.length > 0) {
-          setSelectedNoteId((prev) =>
-            prev && loadedNotes.some((n) => n.id === prev) ? prev : loadedNotes[0].id
-          );
-        } else {
-          setSelectedNoteId('');
-          setActiveNote(null);
-        }
-      } catch (e) {
-        console.error('Failed to fetch notes:', e);
-      } finally {
-        setLoadingList(false);
-      }
     },
     [
       debouncedSearch,
@@ -268,6 +247,65 @@ export default function EvercloneWorkspace() {
     ]
   );
 
+  const fetchNotesList = useCallback(
+    async (preferSelectNoteId?: string) => {
+      setLoadingList(true);
+      try {
+        const sp = buildNoteQueryParams(0, 150);
+        const res = await fetch(`/api/notes?${sp.toString()}`);
+        if (!res.ok) return;
+        const data = await res.json();
+        const loadedNotes: NoteSummary[] = data.notes || [];
+        setNotes(loadedNotes);
+        setTotalNotes(data.total || 0);
+        setHasMoreNotes(loadedNotes.length < (data.total || 0));
+
+        if (preferSelectNoteId) {
+          setSelectedNoteId(preferSelectNoteId);
+        } else if (loadedNotes.length > 0) {
+          setSelectedNoteId((prev) =>
+            prev && loadedNotes.some((n) => n.id === prev) ? prev : loadedNotes[0].id
+          );
+        } else {
+          setSelectedNoteId('');
+          setActiveNote(null);
+        }
+      } catch (e) {
+        console.error('Failed to fetch notes:', e);
+      } finally {
+        setLoadingList(false);
+      }
+    },
+    [buildNoteQueryParams]
+  );
+
+  const fetchMoreNotes = useCallback(async () => {
+    if (loadingMore || !hasMoreNotes || loadingList) return;
+    setLoadingMore(true);
+    try {
+      const sp = buildNoteQueryParams(notes.length, 150);
+      const res = await fetch(`/api/notes?${sp.toString()}`);
+      if (!res.ok) return;
+      const data = await res.json();
+      const nextBatch: NoteSummary[] = data.notes || [];
+      if (nextBatch.length === 0) {
+        setHasMoreNotes(false);
+      } else {
+        setNotes((prev) => {
+          const existingIds = new Set(prev.map((n) => n.id));
+          const unique = nextBatch.filter((n) => !existingIds.has(n.id));
+          const combined = [...prev, ...unique];
+          setHasMoreNotes(combined.length < (data.total || 0));
+          return combined;
+        });
+      }
+    } catch (e) {
+      console.error('Failed to fetch next page of notes:', e);
+    } finally {
+      setLoadingMore(false);
+    }
+  }, [loadingMore, hasMoreNotes, loadingList, buildNoteQueryParams, notes.length]);
+
   useEffect(() => {
     fetchSidebar();
   }, [fetchSidebar]);
@@ -628,15 +666,38 @@ export default function EvercloneWorkspace() {
     }
   };
 
-  // Virtualizer for middle note list pane
+  // Card grid rows: 2 cards per row
+  const cardGridRowCount = Math.ceil(notes.length / 2);
+
+  // Virtualizer for middle note list pane (handles Snippets, Compact list, and 2-column Card grid rows)
   const rowVirtualizer = useVirtualizer({
-    count: notes.length,
+    count: listMode === 'cards' ? cardGridRowCount : notes.length,
     getScrollElement: () => listParentRef.current,
     estimateSize: () =>
-      listMode === 'compact' ? 52 : listMode === 'cards' ? 148 : 112,
+      listMode === 'compact' ? 52 : listMode === 'cards' ? 240 : 112,
-    overscan: 10,
+    overscan: 8,
   });
 
+  // Infinite scroll trigger: when scrolling near bottom, fetch next page of notes
+  const virtualItems = rowVirtualizer.getVirtualItems();
+  useEffect(() => {
+    if (!virtualItems.length || !hasMoreNotes || loadingMore || loadingList) return;
+    const lastItem = virtualItems[virtualItems.length - 1];
+    const totalVirtual = listMode === 'cards' ? cardGridRowCount : notes.length;
+    if (lastItem && lastItem.index >= totalVirtual - 6) {
+      fetchMoreNotes();
+    }
+  }, [
+    virtualItems,
+    hasMoreNotes,
+    loadingMore,
+    loadingList,
+    listMode,
+    cardGridRowCount,
+    notes.length,
+    fetchMoreNotes,
+  ]);
+
   // Compute current view title
   const currentViewTitle = (() => {
     if (activeView === 'trash') return 'Trash';
@@ -1584,6 +1645,13 @@ export default function EvercloneWorkspace() {
             </div>
           </div>
 
+          {/* Loading indicator for next page */}
+          {loadingMore && (
+            <div className="py-1 px-3 bg-emerald-50 text-center text-[10px] text-evernote-green font-medium animate-pulse rounded">
+              Loading more notes…
+            </div>
+          )}
+
           {/* Collapsible Filter Panel */}
           {showFilterPanel && (
             <div className="pt-2.5 border-t border-gray-200 space-y-3 text-xs">
@@ -1984,9 +2052,33 @@ export default function EvercloneWorkspace() {
               )}
             </div>
           ) : listMode === 'cards' ? (
-            /* 2-COLUMN VISUAL CARD GRID VIEW */
+            /* VIRTUALIZED 2-COLUMN VISUAL CARD GRID VIEW */
-            <div className="grid grid-cols-1 sm:grid-cols-2 gap-3 p-3">
+            <div
-              {notes.map((n) => {
+              style={{
+                height: `${rowVirtualizer.getTotalSize()}px`,
+                width: '100%',
+                position: 'relative',
+              }}
+            >
+              {rowVirtualizer.getVirtualItems().map((virtualRow) => {
+                const startIndex = virtualRow.index * 2;
+                const rowNotes = notes.slice(startIndex, startIndex + 2);
+
+                return (
+                  <div
+                    key={`card-row-${virtualRow.index}`}
+                    data-index={virtualRow.index}
+                    ref={rowVirtualizer.measureElement}
+                    style={{
+                      position: 'absolute',
+                      top: 0,
+                      left: 0,
+                      width: '100%',
+                      transform: `translateY(${virtualRow.start}px)`,
+                    }}
+                    className="grid grid-cols-1 sm:grid-cols-2 gap-3 p-3"
+                  >
+                    {rowNotes.map((n) => {
                       const isSelected = n.id === selectedNoteId;
                       const firstImg = n.attachments.find((a) =>
                         a.mimeType.startsWith('image/')
@@ -2141,6 +2233,9 @@ export default function EvercloneWorkspace() {
                       );
                     })}
                   </div>
+                );
+              })}
+            </div>
           ) : (
             /* VIRTUALIZED SNIPPET & COMPACT LIST VIEWS */
             <div