spectra/server/youtube.js

// YouTube Fetcher & Parser for Spectra
// Handles live YouTube search scraping, RSS channel feeds, and fallback caching

export function parseRelativeTimeToDate(relativeText) {
  const now = new Date();
  if (!relativeText) return now.toISOString();

  const text = relativeText.toLowerCase().trim();
  let msAgo = 0;

  const numMatch = text.match(/(\d+)/);
  const num = numMatch ? parseInt(numMatch[1], 10) : 1;

  if (text.includes("second")) {
    msAgo = num * 1000;
  } else if (text.includes("minute")) {
    msAgo = num * 60 * 1000;
  } else if (text.includes("hour")) {
    msAgo = num * 60 * 60 * 1000;
  } else if (text.includes("day")) {
    msAgo = num * 24 * 60 * 60 * 1000;
  } else if (text.includes("week")) {
    msAgo = num * 7 * 24 * 60 * 60 * 1000;
  } else if (text.includes("month")) {
    msAgo = num * 30 * 24 * 60 * 60 * 1000;
  } else if (text.includes("year")) {
    msAgo = num * 365 * 24 * 60 * 60 * 1000;
  } else if (text.includes("yesterday")) {
    msAgo = 24 * 60 * 60 * 1000;
  } else {
    // Default to roughly recent
    msAgo = 3 * 24 * 60 * 60 * 1000;
  }

  return new Date(now.getTime() - msAgo).toISOString();
}

export async function searchYouTube(query, options = {}) {
  const { sort = "upload_date", timeout = 6000 } = options;
  // sp=CAISAhAB sorts by upload date (newest first)
  const spParam = sort === "upload_date" ? "&sp=CAISAhAB" : "";
  const url = `https://www.youtube.com/results?search_query=${encodeURIComponent(query)}${spParam}`;

  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeout);

  try {
    const res = await fetch(url, {
      signal: controller.signal,
      headers: {
        "User-Agent":
          "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
      }
    });

    clearTimeout(timer);
    if (!res.ok) {
      return [];
    }

    const html = await res.text();
    const match =
      html.match(/var ytInitialData = ({.*?});<\/script>/s) ||
      html.match(/ytInitialData\s*=\s*({.+?});/);

    if (!match) return [];

    const data = JSON.parse(match[1]);
    const items = [];
    const contents =
      data?.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer?.contents || [];

    for (const section of contents) {
      const renderers = section?.itemSectionRenderer?.contents || [];
      for (const item of renderers) {
        if (item.videoRenderer) {
          const vr = item.videoRenderer;
          if (!vr.videoId) continue;

          const title = vr.title?.runs?.[0]?.text || "Untitled";
          const channel = vr.ownerText?.runs?.[0]?.text || "Creator";
          const channelId = vr.ownerText?.runs?.[0]?.navigationEndpoint?.browseEndpoint?.browseId || "";
          const views = vr.viewCountText?.simpleText || vr.shortViewCountText?.simpleText || "";
          const relativeTime = vr.publishedTimeText?.simpleText || "Recent";
          const duration = vr.lengthText?.simpleText || "";
          const description = vr.detailedMetadataSnippets?.[0]?.snippetText?.runs?.map(r => r.text).join("") || "";

          items.push({
            id: vr.videoId,
            title,
            channel,
            channel_id: channelId,
            thumbnail: `https://i.ytimg.com/vi/${vr.videoId}/hqdefault.jpg`,
            duration,
            views,
            published_relative: relativeTime,
            published_at: parseRelativeTimeToDate(relativeTime),
            description,
            url: `https://www.youtube.com/watch?v=${vr.videoId}`
          });
        }
      }
    }

    return items;
  } catch (err) {
    clearTimeout(timer);
    // Ignore network timeouts / errors and return empty list for fallback handling
    return [];
  }
}

export async function fetchChannelRss(channelId, timeout = 5000) {
  if (!channelId) return [];
  const url = `https://www.youtube.com/feeds/videos.xml?channel_id=${channelId}`;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeout);

  try {
    const res = await fetch(url, { signal: controller.signal });
    clearTimeout(timer);
    if (!res.ok) return [];

    const xml = await res.text();
    const entries = [];
    const entryMatches = xml.matchAll(/<entry>([\s\S]*?)<\/entry>/g);

    for (const match of entryMatches) {
      const block = match[1];
      const videoId = (block.match(/<yt:videoId>([^<]+)<\/yt:videoId>/) || [])[1];
      const title = (block.match(/<title>([^<]+)<\/title>/) || [])[1];
      const author = (block.match(/<name>([^<]+)<\/name>/) || [])[1];
      const published = (block.match(/<published>([^<]+)<\/published>/) || [])[1];
      const desc = (block.match(/<media:description>([\s\S]*?)<\/media:description>/) || [])[1];

      if (videoId && title) {
        entries.push({
          id: videoId,
          title: decodeXmlEntities(title),
          channel: decodeXmlEntities(author || ""),
          channel_id: channelId,
          thumbnail: `https://i.ytimg.com/vi/${videoId}/hqdefault.jpg`,
          duration: "",
          views: "",
          published_at: published || new Date().toISOString(),
          published_relative: "Recent",
          description: decodeXmlEntities(desc || "").slice(0, 200),
          url: `https://www.youtube.com/watch?v=${videoId}`
        });
      }
    }

    return entries;
  } catch (err) {
    clearTimeout(timer);
    return [];
  }
}

function decodeXmlEntities(str) {
  if (!str) return "";
  return str
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'");
}