fix: add browser headers and clean fallback for bot-protected clipping

src/lib/importers.ts

@@ -738,6 +738,16 @@ export async function clipUrlToNote(
       headers: {
         'User-Agent':
           'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Sec-Ch-Ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+        'Sec-Ch-Ua-Mobile': '?0',
+        'Sec-Ch-Ua-Platform': '"Windows"',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Upgrade-Insecure-Requests': '1',
       },
       signal: AbortSignal.timeout(15000),
     });
@@ -754,7 +764,7 @@ export async function clipUrlToNote(
   const reader = new Readability(doc);
   const article = reader.parse();
 
-  const title =
+  let title =
     options.title?.trim() ||
     article?.title?.trim() ||
     docTitle ||
@@ -764,7 +774,20 @@ export async function clipUrlToNote(
     article?.content ||
     `<p>Clipped from <a href="${escapeHtml(url)}" target="_blank">${escapeHtml(url)}</a></p>${doc.body?.innerHTML || ''}`;
 
-  if (heroImgSrc && !contentHtml.includes(heroImgSrc)) {
+  const isBotChallenge =
+    contentHtml.includes('Please enable JS') ||
+    contentHtml.includes('Just a moment...') ||
+    contentHtml.includes('captcha-delivery.com') ||
+    contentHtml.includes('challenge-platform') ||
+    title === 'Just a moment...' ||
+    (contentHtml.length < 300 && contentHtml.includes('enable JS'));
+
+  if (isBotChallenge) {
+    if (title === 'Just a moment...' || title.includes('.com') || title.includes('enable JS')) {
+      title = options.title?.trim() || url;
+    }
+    contentHtml = `<p><strong><a href="${escapeHtml(url)}" target="_blank" rel="noopener noreferrer">${escapeHtml(title)}</a></strong></p>\n<p><em>(This site is protected by bot verification. To clip full article text and graphics, use the Everclone Web Clipper extension directly from your browser tab.)</em></p>\n<p>Source: <a href="${escapeHtml(url)}" target="_blank" rel="noopener noreferrer">${escapeHtml(url)}</a></p>`;
+  } else if (heroImgSrc && !contentHtml.includes(heroImgSrc)) {
     contentHtml = `<p><img src="${escapeHtml(heroImgSrc)}" alt="${escapeHtml(title)}" /></p>\n${contentHtml}`;
   }