Initial commit: Spectra YouTube Suggest Engine

server/youtube.js

@@ -0,0 +1,170 @@
+// YouTube Fetcher & Parser for Spectra
+// Handles live YouTube search scraping, RSS channel feeds, and fallback caching
+
+export function parseRelativeTimeToDate(relativeText) {
+  const now = new Date();
+  if (!relativeText) return now.toISOString();
+
+  const text = relativeText.toLowerCase().trim();
+  let msAgo = 0;
+
+  const numMatch = text.match(/(\d+)/);
+  const num = numMatch ? parseInt(numMatch[1], 10) : 1;
+
+  if (text.includes("second")) {
+    msAgo = num * 1000;
+  } else if (text.includes("minute")) {
+    msAgo = num * 60 * 1000;
+  } else if (text.includes("hour")) {
+    msAgo = num * 60 * 60 * 1000;
+  } else if (text.includes("day")) {
+    msAgo = num * 24 * 60 * 60 * 1000;
+  } else if (text.includes("week")) {
+    msAgo = num * 7 * 24 * 60 * 60 * 1000;
+  } else if (text.includes("month")) {
+    msAgo = num * 30 * 24 * 60 * 60 * 1000;
+  } else if (text.includes("year")) {
+    msAgo = num * 365 * 24 * 60 * 60 * 1000;
+  } else if (text.includes("yesterday")) {
+    msAgo = 24 * 60 * 60 * 1000;
+  } else {
+    // Default to roughly recent
+    msAgo = 3 * 24 * 60 * 60 * 1000;
+  }
+
+  return new Date(now.getTime() - msAgo).toISOString();
+}
+
+export async function searchYouTube(query, options = {}) {
+  const { sort = "upload_date", timeout = 6000 } = options;
+  // sp=CAISAhAB sorts by upload date (newest first)
+  const spParam = sort === "upload_date" ? "&sp=CAISAhAB" : "";
+  const url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query)}${spParam}`;
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const res = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        "User-Agent":
+          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+        "Accept-Language": "en-US,en;q=0.9",
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+      }
+    });
+
+    clearTimeout(timer);
+    if (!res.ok) {
+      return [];
+    }
+
+    const html = await res.text();
+    const match =
+      html.match(/var ytInitialData = ({.*?});<\/script>/s) ||
+      html.match(/ytInitialData\s*=\s*({.+?});/);
+
+    if (!match) return [];
+
+    const data = JSON.parse(match[1]);
+    const items = [];
+    const contents =
+      data?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];
+
+    for (const section of contents) {
+      const renderers = section?.itemSectionRenderer?.contents || [];
+      for (const item of renderers) {
+        if (item.videoRenderer) {
+          const vr = item.videoRenderer;
+          if (!vr.videoId) continue;
+
+          const title = vr.title?.runs?.[0]?.text || "Untitled";
+          const channel = vr.ownerText?.runs?.[0]?.text || "Creator";
+          const channelId = vr.ownerText?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId || "";
+          const views = vr.viewCountText?.simpleText || vr.shortViewCountText?.simpleText || "";
+          const relativeTime = vr.publishedTimeText?.simpleText || "Recent";
+          const duration = vr.lengthText?.simpleText || "";
+          const description = vr.detailedMetadataSnippets?.[0]?.snippetText?.runs?.map(r => r.text).join("") || "";
+
+          items.push({
+            id: vr.videoId,
+            title,
+            channel,
+            channel_id: channelId,
+            thumbnail: `https://i.ytimg.com/vi/${vr.videoId}/hqdefault.jpg`,
+            duration,
+            views,
+            published_relative: relativeTime,
+            published_at: parseRelativeTimeToDate(relativeTime),
+            description,
+            url: `https://www.youtube.com/watch?v=${vr.videoId}`
+          });
+        }
+      }
+    }
+
+    return items;
+  } catch (err) {
+    clearTimeout(timer);
+    // Ignore network timeouts / errors and return empty list for fallback handling
+    return [];
+  }
+}
+
+export async function fetchChannelRss(channelId, timeout = 5000) {
+  if (!channelId) return [];
+  const url = `https://www.youtube.com/feeds/videos.xml?channel_id=${channelId}`;
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    clearTimeout(timer);
+    if (!res.ok) return [];
+
+    const xml = await res.text();
+    const entries = [];
+    const entryMatches = xml.matchAll(/<entry>([\s\S]*?)<\/entry>/g);
+
+    for (const match of entryMatches) {
+      const block = match[1];
+      const videoId = (block.match(/<yt:videoId>([^<]+)<\/yt:videoId>/) || [])[1];
+      const title = (block.match(/<title>([^<]+)<\/title>/) || [])[1];
+      const author = (block.match(/<name>([^<]+)<\/name>/) || [])[1];
+      const published = (block.match(/<published>([^<]+)<\/published>/) || [])[1];
+      const desc = (block.match(/<media:description>([\s\S]*?)<\/media:description>/) || [])[1];
+
+      if (videoId && title) {
+        entries.push({
+          id: videoId,
+          title: decodeXmlEntities(title),
+          channel: decodeXmlEntities(author || ""),
+          channel_id: channelId,
+          thumbnail: `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
+          duration: "",
+          views: "",
+          published_at: published || new Date().toISOString(),
+          published_relative: "Recent",
+          description: decodeXmlEntities(desc || "").slice(0, 200),
+          url: `https://www.youtube.com/watch?v=${videoId}`
+        });
+      }
+    }
+
+    return entries;
+  } catch (err) {
+    clearTimeout(timer);
+    return [];
+  }
+}
+
+function decodeXmlEntities(str) {
+  if (!str) return "";
+  return str
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+}