Fix duplicate first image in desktop extension clipper

src/app/api/hooks/note/route.ts

@@ -10,6 +10,7 @@ import {
   getOrCreateTagIds,
   prepareDomForReadability,
   localizeClippedImages,
+  shouldPrependHeroImage,
 } from '@/lib/importers';
 import {
   saveAttachmentBuffer,
@@ -213,7 +214,7 @@ export async function POST(req: NextRequest) {
         'Web Clip';
 
       let extractedHtml = article?.content || doc.body?.innerHTML || pageHtml;
-      if (heroImgSrc && !extractedHtml.includes(heroImgSrc)) {
+      if (shouldPrependHeroImage(extractedHtml, heroImgSrc)) {
         extractedHtml = `<p><img src="${heroImgSrc}" alt="" /></p>\n${extractedHtml}`;
       }
       if (comment.trim()) {

src/lib/importers.ts

@@ -592,6 +592,22 @@ export function prepareDomForReadability(doc: Document, baseUrl: string): { hero
   return { heroImgSrc };
 }
 
+export function normalizeImageBasename(urlOrSrc: string): { dir: string; stem: string } {
+  try {
+    const clean = urlOrSrc.split('?')[0].split('#')[0].trim().toLowerCase();
+    const lastSlash = clean.lastIndexOf('/');
+    const dir = lastSlash >= 0 ? clean.slice(0, lastSlash) : '';
+    let filename = lastSlash >= 0 ? clean.slice(lastSlash + 1) : clean;
+    // Remove extension
+    filename = filename.replace(/\.(jpe?g|png|webp|avif|gif|svg)$/i, '');
+    // Strip leading sizing/gallery prefixes like l-, m-, s-, intro-, etc.
+    const stem = filename.replace(/^(?:(?:l|m|s|xl|xs|large|medium|small|thumb|intro)[_-])+/i, '');
+    return { dir, stem };
+  } catch {
+    return { dir: '', stem: '' };
+  }
+}
+
 /**
  * Determines whether a rescued hero/lead image should be prepended to the article HTML.
  * Prevents duplicating the first image when the article body already contains the lead image
@@ -613,6 +629,10 @@ export function shouldPrependHeroImage(contentHtml: string, heroImgSrc: string |
   if (heroFilename.length > 5 && contentHtml.toLowerCase().includes(heroFilename)) {
     return false;
   }
+  const heroNorm = normalizeImageBasename(heroClean);
+  if (heroNorm.stem.length > 5 && contentHtml.toLowerCase().includes(heroNorm.stem)) {
+    return false;
+  }
 
   // 4. Inspect actual <img> tags inside contentHtml
   try {
@@ -629,25 +649,51 @@ export function shouldPrependHeroImage(contentHtml: string, heroImgSrc: string |
         ''
       ).split('?')[0].split('#')[0].trim().toLowerCase();
 
-      if (src && (src === heroClean || heroClean.endsWith(src) || src.endsWith(heroClean))) {
-        return false;
-      }
-      if (dataUrl && (dataUrl === heroClean || heroClean.endsWith(dataUrl) || dataUrl.endsWith(heroClean))) {
-        return false;
-      }
-      const srcFilename = src.split('/').pop() || '';
-      if (heroFilename.length > 5 && srcFilename.length > 5 && srcFilename === heroFilename) {
-        return false;
+      for (const target of [src, dataUrl]) {
+        if (!target) continue;
+        if (target === heroClean || heroClean.endsWith(target) || target.endsWith(heroClean)) {
+          return false;
+        }
+        const targetFilename = target.split('/').pop() || '';
+        if (heroFilename.length > 5 && targetFilename.length > 5 && targetFilename === heroFilename) {
+          return false;
+        }
+        const targetNorm = normalizeImageBasename(target);
+        if (heroNorm.stem.length > 5 && targetNorm.stem.length > 5 && heroNorm.stem === targetNorm.stem) {
+          if (!heroNorm.dir || !targetNorm.dir || heroNorm.dir === targetNorm.dir) {
+            return false;
+          }
+        }
       }
     }
 
-    // 5. If the article already opens with a lead image (within first 200 chars of text),
+    // 5. If the article already opens with a lead image (within first 150 chars of visible text),
     // do not prepend an extra hero image over it
     if (imgs.length > 0) {
-      const firstImg = imgs[0];
-      const htmlBefore = contentHtml.slice(0, contentHtml.indexOf(firstImg.outerHTML));
-      const textBefore = stripHtmlToText(htmlBefore).trim();
-      if (textBefore.length < 150) {
+      const doc = dom.window.document;
+      const root = doc.body || doc.documentElement;
+      let textLengthBeforeFirstImg = 0;
+      let foundImg = false;
+      const walker = doc.createTreeWalker(
+        root,
+        1 | 4 /* NodeFilter.SHOW_ELEMENT | NodeFilter.SHOW_TEXT */
+      );
+      let currentNode = walker.nextNode();
+      while (currentNode) {
+        if (currentNode.nodeType === 1 /* ELEMENT_NODE */) {
+          if ((currentNode as Element).tagName.toLowerCase() === 'img') {
+            foundImg = true;
+            break;
+          }
+        } else if (currentNode.nodeType === 3 /* TEXT_NODE */) {
+          textLengthBeforeFirstImg += (currentNode.textContent || '').trim().length;
+          if (textLengthBeforeFirstImg >= 150) {
+            break;
+          }
+        }
+        currentNode = walker.nextNode();
+      }
+      if (foundImg && textLengthBeforeFirstImg < 150) {
         return false;
       }
     }
@@ -670,8 +716,33 @@ export async function localizeClippedImages(
 ): Promise<string> {
   const dom = new JSDOM(contentHtml);
   const doc = dom.window.document;
+
+  // Deduplicate leading identical/variant images at the top of the document before downloading
+  const initialImgs = Array.from(doc.querySelectorAll('img'));
+  if (initialImgs.length >= 2) {
+    const s0 = initialImgs[0].getAttribute('src')?.trim() || '';
+    const s1 = initialImgs[1].getAttribute('src')?.trim() || '';
+    if (s0 && s1) {
+      const norm0 = normalizeImageBasename(s0);
+      const norm1 = normalizeImageBasename(s1);
+      const isDup =
+        s0 === s1 ||
+        (norm0.stem.length > 5 &&
+          norm0.stem === norm1.stem &&
+          (!norm0.dir || !norm1.dir || norm0.dir === norm1.dir));
+      if (isDup) {
+        const parentP = initialImgs[0].closest('p');
+        if (parentP && (!parentP.textContent || parentP.textContent.trim() === '')) {
+          parentP.remove();
+        } else {
+          initialImgs[0].remove();
+        }
+      }
+    }
+  }
+
   const imgs = Array.from(doc.querySelectorAll('img'));
-  if (imgs.length === 0) return contentHtml;
+  if (imgs.length === 0) return doc.body.innerHTML;
 
   const urlToLocalSrc = new Map<string, string>();
   const uniqueUrls: string[] = [];
@@ -749,9 +820,16 @@ export async function localizeClippedImages(
   if (remainingImgs.length >= 2) {
     const firstSrc = remainingImgs[0].getAttribute('src')?.trim();
     const secondSrc = remainingImgs[1].getAttribute('src')?.trim();
-    if (firstSrc && firstSrc === secondSrc) {
+    const norm0 = firstSrc ? normalizeImageBasename(firstSrc) : { dir: '', stem: '' };
+    const norm1 = secondSrc ? normalizeImageBasename(secondSrc) : { dir: '', stem: '' };
+    const isDup =
+      firstSrc === secondSrc ||
+      (norm0.stem.length > 5 &&
+        norm0.stem === norm1.stem &&
+        (!norm0.dir || !norm1.dir || norm0.dir === norm1.dir));
+    if (isDup) {
       const firstP = remainingImgs[0].closest('p');
-      if (firstP && firstP.textContent?.trim() === '') {
+      if (firstP && (!firstP.textContent || firstP.textContent.trim() === '')) {
         firstP.remove();
       } else {
         remainingImgs[0].remove();