Initial commit: Spectra YouTube Suggest Engine

server/engine.js

@@ -0,0 +1,225 @@
+// Spectra Recommendation Engine
+// Steerable suggestions, recency biasing, topic diversity balancing, and strict no-repeats guarantee
+
+import { searchYouTube } from "./youtube.js";
+
+export function calculateRecencyScore(publishedAt, recencyBias = "recent") {
+  const now = Date.now();
+  const pubTime = new Date(publishedAt).getTime();
+  const ageDays = Math.max(0, (now - pubTime) / (1000 * 60 * 60 * 24));
+
+  let baseMultiplier = 1.0;
+  if (ageDays <= 2) {
+    baseMultiplier = 2.5;
+  } else if (ageDays <= 7) {
+    baseMultiplier = 2.0;
+  } else if (ageDays <= 30) {
+    baseMultiplier = 1.5;
+  } else if (ageDays <= 90) {
+    baseMultiplier = 1.1;
+  } else if (ageDays <= 365) {
+    baseMultiplier = 0.8;
+  } else {
+    baseMultiplier = 0.5;
+  }
+
+  // Bias tuning
+  if (recencyBias === "ultra_fresh") {
+    if (ageDays > 3) return 0.1;
+    return baseMultiplier * 2.0;
+  } else if (recencyBias === "fresh") {
+    if (ageDays > 14) return 0.2;
+    return baseMultiplier * 1.5;
+  } else if (recencyBias === "broad") {
+    return Math.max(0.7, baseMultiplier);
+  } else if (recencyBias === "any") {
+    return 1.0;
+  }
+
+  return baseMultiplier;
+}
+
+export function generateRecommendationReason(video, topic, feedbackForChannel) {
+  if (feedbackForChannel === "thumbs_up") {
+    return `⭐ Highly rated creator: ${video.channel}`;
+  }
+  if (topic && topic.type === "person") {
+    return `👤 Featured creator: ${topic.name}`;
+  }
+  const now = Date.now();
+  const pubTime = new Date(video.published_at).getTime();
+  const ageDays = (now - pubTime) / (1000 * 60 * 60 * 24);
+
+  if (ageDays <= 3) {
+    return `🔥 Fresh release in ${topic ? topic.name : "your interests"}`;
+  }
+  if (topic && topic.weight > 1.2) {
+    return `🎯 Steered by your upvotes on ${topic.name}`;
+  }
+  return `✨ Curated for ${topic ? topic.name : "you"}`;
+}
+
+export async function generateSuggestions(db, options = {}) {
+  const {
+    limit = 12,
+    topicFilter = null,
+    fetchLive = true,
+    forceFresh = false
+  } = options;
+
+  const settings = db.getSettings();
+  const recencyBias = settings.recency_bias || "recent";
+
+  // 1. Get active topics
+  let activeTopics = db.getActiveTopics();
+  if (topicFilter) {
+    activeTopics = activeTopics.filter(t => t.id === topicFilter);
+  }
+  if (activeTopics.length === 0) {
+    activeTopics = db.getTopics().slice(0, 4);
+  }
+
+  // 2. Load seen video IDs (to strictly enforce NO REPEATS)
+  const seenIds = db.getSeenVideoIds();
+
+  // 3. Channel feedback affinity map
+  const allFeedback = db.getFeedback();
+  const channelFeedbackMap = {};
+  for (const fb of allFeedback) {
+    if (fb.channel) {
+      // most recent feedback wins
+      if (!channelFeedbackMap[fb.channel.toLowerCase()]) {
+        channelFeedbackMap[fb.channel.toLowerCase()] = fb.action;
+      }
+    }
+  }
+
+  // 4. Optionally fetch live YouTube results to supplement local catalog
+  if (fetchLive) {
+    // Pick 2-3 topics with highest weights to query live
+    const queryTopics = [...activeTopics]
+      .sort((a, b) => b.weight - a.weight)
+      .slice(0, 3);
+
+    for (const qt of queryTopics) {
+      try {
+        const liveItems = await searchYouTube(qt.query, {
+          sort: "upload_date",
+          timeout: 4000
+        });
+
+        for (const item of liveItems) {
+          item.topic_id = qt.id;
+          db.upsertVideo(item);
+        }
+      } catch (err) {
+        // Silently continue if YouTube search encounters a glitch
+      }
+    }
+  }
+
+  // 5. Query all candidates from database
+  const allDbVideos = db.getAllVideos();
+  const candidates = [];
+
+  for (const v of allDbVideos) {
+    // A. Check if Big Noped
+    if (db.isNoped(v.id, v.channel, v.title)) {
+      continue;
+    }
+
+    // B. Check if already seen (NO REPEATS)
+    if (!forceFresh && seenIds.has(v.id)) {
+      continue;
+    }
+
+    // C. Check if topic is active
+    let matchedTopic = activeTopics.find(t => t.id === v.topic_id);
+    if (!matchedTopic && !topicFilter) {
+      // Fallback matching by keyword
+      matchedTopic = activeTopics[0];
+    }
+    if (!matchedTopic && topicFilter) {
+      continue;
+    }
+
+    // Calculate score
+    const topicWeight = matchedTopic ? matchedTopic.weight : 1.0;
+    const recencyMult = calculateRecencyScore(v.published_at, recencyBias);
+
+    let channelBonus = 0;
+    const chFeedback = channelFeedbackMap[v.channel?.toLowerCase()];
+    if (chFeedback === "thumbs_up") channelBonus = 3.0;
+    if (chFeedback === "thumbs_down") channelBonus = -2.0;
+
+    // Small random jitter (0 to 0.4) to provide variation among ties
+    const jitter = Math.random() * 0.4;
+
+    const totalScore = (topicWeight * 2.0 + channelBonus + jitter) * recencyMult;
+
+    candidates.push({
+      ...v,
+      topic_name: matchedTopic ? matchedTopic.name : "General",
+      topic_id: matchedTopic ? matchedTopic.id : v.topic_id,
+      score: totalScore,
+      why_suggested: generateRecommendationReason(v, matchedTopic, chFeedback)
+    });
+  }
+
+  // 6. Balance across topics to ensure a diverse spread over interesting topics & people
+  // Sort candidates by score descending
+  candidates.sort((a, b) => b.score - a.score);
+
+  const selected = [];
+  const selectedIds = new Set();
+  const topicCounts = {};
+
+  // First pass: pick best video per topic (max 2 per topic)
+  for (const item of candidates) {
+    if (selected.length >= limit) break;
+    const count = topicCounts[item.topic_id] || 0;
+    if (count < 2) {
+      selected.push(item);
+      selectedIds.add(item.id);
+      topicCounts[item.topic_id] = count + 1;
+    }
+  }
+
+  // Second pass: fill remainder with highest scoring remaining videos
+  if (selected.length < limit) {
+    for (const item of candidates) {
+      if (selected.length >= limit) break;
+      if (!selectedIds.has(item.id)) {
+        selected.push(item);
+        selectedIds.add(item.id);
+      }
+    }
+  }
+
+  // If still fewer than limit (e.g. all seen or small seed), fallback to top candidates regardless of seen if forceFresh
+  if (selected.length === 0 && candidates.length === 0 && seenIds.size > 0) {
+    // If every video has been seen, return a notification or top un-noped items
+    const allFallback = allDbVideos.filter(v => !db.isNoped(v.id, v.channel, v.title));
+    for (const item of allFallback.slice(0, limit)) {
+      selected.push({
+        ...item,
+        topic_name: "Catalog",
+        score: 1.0,
+        why_suggested: "Restarted cycle from catalog"
+      });
+      selectedIds.add(item.id);
+    }
+  }
+
+  // 7. Record selected videos as SEEN so they will never repeat
+  const batchId = "batch-" + Date.now();
+  db.recordSeenVideos([...selectedIds], batchId);
+
+  return {
+    batchId,
+    recencyBias,
+    totalSeen: db.getSeenCount(),
+    count: selected.length,
+    suggestions: selected
+  };
+}