Compare commits

...
6 Commits
12 changed files with 219 additions and 74 deletions

No files matched your search

+9 -2
View File
@@ -5,16 +5,23 @@ services:
dockerfile: Dockerfile dockerfile: Dockerfile
container_name: thread-ripper container_name: thread-ripper
restart: unless-stopped restart: unless-stopped
networks:
- t3_proxy
ports: ports:
- "5000:5000" - "8475:5000"
environment: environment:
- PORT=5000 - PORT=5000
- DATA_DIR=/app/data - DATA_DIR=/app/data
volumes: volumes:
- ./data:/app/data - /home/chrisf/docker/appdata/thread-ripper:/app/data
- /home/chrisf/docker/thread-ripper/thread_ripper:/app/thread_ripper:ro
healthcheck: healthcheck:
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:5000/api/health\")'"] test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:5000/api/health\")'"]
interval: 30s interval: 30s
timeout: 10s timeout: 10s
retries: 3 retries: 3
start_period: 15s start_period: 15s
networks:
t3_proxy:
external: true
+78
View File
@@ -0,0 +1,78 @@
import math
from PIL import Image, ImageDraw
def create_icons(output_dir):
size = 256
img = Image.new("RGBA", (size, size), (0, 0, 0, 0))
# Draw linear gradient on rounded rectangle
# Gradient: (31, 111, 235) to (137, 87, 229)
mask = Image.new("L", (size, size), 0)
draw_mask = ImageDraw.Draw(mask)
radius = int(size * 0.24)
draw_mask.rounded_rectangle([(0, 0), (size - 1, size - 1)], radius=radius, fill=255)
grad = Image.new("RGBA", (size, size), (0, 0, 0, 0))
draw_grad = ImageDraw.Draw(grad)
for y in range(size):
for x in range(size):
t = (x + y) / (2 * size)
r = int(31 * (1 - t) + 137 * t)
g = int(111 * (1 - t) + 87 * t)
b = int(235 * (1 - t) + 229 * t)
draw_grad.point((x, y), fill=(r, g, b, 255))
img = Image.composite(grad, img, mask)
# Draw the white link chain icon in center
# Scale from 24x24 viewBox to fit nicely (about 130x130)
draw = ImageDraw.Draw(img)
stroke_w = int(size * 0.08)
# To draw smooth link curves, we draw at 4x and downsample
hi_size = size * 4
hi_img = Image.new("RGBA", (hi_size, hi_size), (0, 0, 0, 0))
hi_draw = ImageDraw.Draw(hi_img)
# Center = (hi_size/2, hi_size/2)
# The SVG path:
# Top-right link loop: center around (hi_size*0.58, hi_size*0.42)
# Bottom-left link loop: center around (hi_size*0.42, hi_size*0.58)
# We can draw the rounded pills rotated by 45 degrees
# Create rotated pill for upper right link
link_w = int(hi_size * 0.28)
link_h = int(hi_size * 0.52)
pill = Image.new("RGBA", (link_w, link_h), (0, 0, 0, 0))
pdraw = ImageDraw.Draw(pill)
pw = int(hi_size * 0.075)
pdraw.rounded_rectangle([(0, 0), (link_w - 1, link_h - 1)], radius=link_w // 2, outline=(255, 255, 255, 255), width=pw)
# Rotate pill 45 deg
rot_pill = pill.rotate(45, resample=Image.Resampling.BICUBIC, expand=True)
# Paste upper right link
offset_x1 = int(hi_size * 0.38)
offset_y1 = int(hi_size * 0.16)
hi_img.alpha_composite(rot_pill, (offset_x1, offset_y1))
# Paste bottom left link
offset_x2 = int(hi_size * 0.16)
offset_y2 = int(hi_size * 0.38)
hi_img.alpha_composite(rot_pill, (offset_x2, offset_y2))
# Downscale hi_img with anti-aliasing
icon_layer = hi_img.resize((size, size), Image.Resampling.LANCZOS)
img.alpha_composite(icon_layer)
# Save outputs
# 1. High-res PNG (256x256)
img.save(f"{output_dir}/favicon.png", "PNG")
img.save(f"{output_dir}/apple-touch-icon.png", "PNG")
# 2. Multi-size ICO (16, 32, 48, 64)
img.save(f"{output_dir}/favicon.ico", format="ICO", sizes=[(16, 16), (32, 32), (48, 48), (64, 64)])
print("Generated favicon.png, apple-touch-icon.png, favicon.ico successfully!")
if __name__ == "__main__":
create_icons(r"C:\Users\Chris\.gemini\antigravity\scratch\thread-ripper\thread_ripper\static")
+2 -1
View File
@@ -39,7 +39,8 @@ class RippedThread(BaseModel):
class RipRequest(BaseModel): class RipRequest(BaseModel):
url: str url: str
mode: str = "author_only" # 'author_only' or 'all_replies' mode: str = "author_only" # 'author_only' or 'all_replies'
max_scrolls: int = 6 include_images: bool = False
max_scrolls: int = 4
class HistorySummary(BaseModel): class HistorySummary(BaseModel):
id: str id: str
+46 -29
View File
@@ -11,8 +11,10 @@ class ThreadScraper:
self.headless = headless self.headless = headless
self._browser: Optional[Browser] = None self._browser: Optional[Browser] = None
self._playwright = None self._playwright = None
self._lock = asyncio.Lock()
async def get_browser(self) -> Browser: async def get_browser(self) -> Browser:
async with self._lock:
if self._browser is None or not self._browser.is_connected(): if self._browser is None or not self._browser.is_connected():
if self._playwright is None: if self._playwright is None:
self._playwright = await async_playwright().start() self._playwright = await async_playwright().start()
@@ -23,11 +25,16 @@ class ThreadScraper:
"--disable-setuid-sandbox", "--disable-setuid-sandbox",
"--disable-dev-shm-usage", "--disable-dev-shm-usage",
"--disable-gpu", "--disable-gpu",
"--disable-extensions",
"--disable-component-update",
"--disable-background-networking",
"--mute-audio"
] ]
) )
return self._browser return self._browser
async def close(self): async def close(self):
async with self._lock:
if self._browser: if self._browser:
await self._browser.close() await self._browser.close()
self._browser = None self._browser = None
@@ -35,8 +42,8 @@ class ThreadScraper:
await self._playwright.stop() await self._playwright.stop()
self._playwright = None self._playwright = None
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 5) -> RippedThread: async def rip(self, url: str, mode: str = "author_only", include_images: bool = False, max_scrolls: int = 4) -> RippedThread:
"""Scrapes and rips a Threads post and its chained continuation posts.""" """Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
browser = await self.get_browser() browser = await self.get_browser()
context: BrowserContext = await browser.new_context( context: BrowserContext = await browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -45,14 +52,26 @@ class ThreadScraper:
page: Page = await context.new_page() page: Page = await context.new_page()
try: try:
# Navigate and follow redirects # 1. Performance optimization: Block unnecessary binary media downloads
await page.goto(url, wait_until="networkidle", timeout=30000) # We only need the DOM attributes (src, alt, text) which are in the HTML payload
await page.route(
"**/*",
lambda route: route.abort()
if route.request.resource_type in ["image", "media", "font"]
else route.continue_(),
)
# 2. Fast navigation using domcontentloaded instead of networkidle
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
canonical_url = page.url canonical_url = page.url
# Small delay for hydration # 3. Wait for the primary post container to render
await page.wait_for_timeout(2500) try:
await page.wait_for_selector("div[data-pressable-container='true']", timeout=7000)
except Exception:
pass # Fall back to evaluating whatever rendered
# Extract Open Graph metadata # 4. Extract Open Graph metadata
og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''") og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''")
og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''") og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''")
og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''") og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''")
@@ -61,12 +80,10 @@ class ThreadScraper:
author_handle = "" author_handle = ""
author_name = "" author_name = ""
# Check canonical URL for /@username
match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url) match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url)
if match_handle: if match_handle:
author_handle = match_handle.group(1) author_handle = match_handle.group(1)
# Check og:title e.g. "Beth Parent McMillian (@bethmcmilliandesigns) on Threads"
if og_title: if og_title:
m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title) m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title)
if m_title: if m_title:
@@ -77,13 +94,19 @@ class ThreadScraper:
if not author_name: if not author_name:
author_name = author_handle or "Threads Author" author_name = author_handle or "Threads Author"
# Author profile avatar
author_avatar = og_image if og_image else None author_avatar = og_image if og_image else None
# Progressively scroll to load dynamic replies in the thread # 5. Smart adaptive scrolling for thread continuation replies
prev_containers_count = 0
for _ in range(max_scrolls): for _ in range(max_scrolls):
await page.evaluate("window.scrollBy(0, 1200)") containers = await page.query_selector_all("div[data-pressable-container='true']")
await page.wait_for_timeout(1000) curr_count = len(containers)
# If we already have multiple containers and count didn't change on last scroll, settle early
if curr_count > 1 and curr_count == prev_containers_count:
break
prev_containers_count = curr_count
await page.evaluate("window.scrollBy(0, 1400)")
await page.wait_for_timeout(350)
# Query all pressable post containers # Query all pressable post containers
containers = await page.query_selector_all("div[data-pressable-container='true']") containers = await page.query_selector_all("div[data-pressable-container='true']")
@@ -92,7 +115,6 @@ class ThreadScraper:
seen_texts = set() seen_texts = set()
for idx, c in enumerate(containers): for idx, c in enumerate(containers):
# 1. Identify post author handle
user_links = await c.query_selector_all("a[href*='/@']") user_links = await c.query_selector_all("a[href*='/@']")
post_handle = None post_handle = None
post_url = None post_url = None
@@ -113,7 +135,7 @@ class ThreadScraper:
if badge: if badge:
is_author_badge = True is_author_badge = True
# Determine if this segment belongs to the original author # Determine if this segment belongs to original author
is_author_post = False is_author_post = False
if idx == 0: if idx == 0:
is_author_post = True is_author_post = True
@@ -124,21 +146,18 @@ class ThreadScraper:
elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()): elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()):
is_author_post = True is_author_post = True
# If author_only mode is on and this is not by the author, skip
if mode == "author_only" and not is_author_post: if mode == "author_only" and not is_author_post:
continue continue
# 2. Extract text with dir='auto' # Extract text lines
dir_els = await c.query_selector_all("[dir='auto']") dir_els = await c.query_selector_all("[dir='auto']")
segment_text_parts = [] segment_text_parts = []
for del_el in dir_els: for del_el in dir_els:
txt = (await del_el.inner_text()).strip() txt = (await del_el.inner_text()).strip()
if not txt: if not txt:
continue continue
# Ignore handle, author badge, bullet dots, stats
if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle: if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle:
continue continue
# Ignore pure stat numbers (e.g., '16.2K', '5d', '2K', '711')
if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt): if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt):
continue continue
if txt not in segment_text_parts: if txt not in segment_text_parts:
@@ -148,21 +167,20 @@ class ThreadScraper:
if not combined_text and not is_author_post: if not combined_text and not is_author_post:
continue continue
# Deduplicate identical consecutive segments
text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest() text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest()
if text_hash in seen_texts and combined_text: if text_hash in seen_texts and combined_text:
continue continue
seen_texts.add(text_hash) seen_texts.add(text_hash)
# 3. Extract media images inside post (exclude profile pictures) # Extract post images (only real content photos, never profile avatars)
imgs = await c.query_selector_all("img")
post_images: List[MediaItem] = [] post_images: List[MediaItem] = []
if include_images:
imgs = await c.query_selector_all("img")
for img in imgs: for img in imgs:
src = await img.get_attribute("src") src = await img.get_attribute("src")
alt = await img.get_attribute("alt") or "" alt = await img.get_attribute("alt") or ""
if src: if src:
# Exclude small avatars or generic SVG icons if "profile_pic" in src or "profile picture" in alt.lower() or ("scontent" in src and "206x206" in src):
if "profile_pic" in src or "scontent" in src and "206x206" in src:
continue continue
post_images.append(MediaItem(type="image", url=src, alt_text=alt)) post_images.append(MediaItem(type="image", url=src, alt_text=alt))
@@ -179,7 +197,7 @@ class ThreadScraper:
) )
) )
# If no segments could be extracted from containers, fallback to OG description # Fallback to OG description if empty
if not raw_segments and og_desc: if not raw_segments and og_desc:
raw_segments.append( raw_segments.append(
PostSegment( PostSegment(
@@ -194,12 +212,11 @@ class ThreadScraper:
) )
) )
# Construct stitched outputs # Build stitched outputs without graphics by default
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments) markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments) plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments) html_out = build_standalone_html(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
# Generate unique ID for this ripped thread
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12] thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}" title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
if len(title) > 90: if len(title) > 90:
+17 -1
View File
@@ -1,4 +1,5 @@
import os import os
import asyncio
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Response from fastapi import FastAPI, HTTPException, Response
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
@@ -14,6 +15,8 @@ scraper = ThreadScraper(headless=True)
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
# Pre-warm Chromium in background so initial user requests are instant
asyncio.create_task(scraper.get_browser())
yield yield
await scraper.close() await scraper.close()
@@ -39,7 +42,12 @@ async def rip_endpoint(req: RipRequest):
raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.") raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.")
try: try:
ripped = await scraper.rip(url=req.url, mode=req.mode, max_scrolls=req.max_scrolls) ripped = await scraper.rip(
url=req.url,
mode=req.mode,
include_images=req.include_images,
max_scrolls=req.max_scrolls
)
storage.save(ripped) storage.save(ripped)
return ripped return ripped
except Exception as e: except Exception as e:
@@ -107,6 +115,14 @@ def download_thread(thread_id: str, format: str):
def health_check(): def health_check():
return {"status": "ok", "app": "Thread Ripper", "version": "1.0.0"} return {"status": "ok", "app": "Thread Ripper", "version": "1.0.0"}
# Direct favicon endpoint for browsers requesting /favicon.ico
@app.get("/favicon.ico", include_in_schema=False)
def get_favicon():
fav_file = os.path.join(static_dir, "favicon.ico")
if os.path.exists(fav_file):
return FileResponse(fav_file)
return Response(status_code=404)
# Mount static frontend directory # Mount static frontend directory
static_dir = os.path.join(os.path.dirname(__file__), "static") static_dir = os.path.join(os.path.dirname(__file__), "static")
if os.path.exists(static_dir): if os.path.exists(static_dir):
+32 -10
View File
@@ -43,6 +43,7 @@ document.addEventListener("DOMContentLoaded", () => {
let currentThread = null; let currentThread = null;
let allHistory = []; let allHistory = [];
let statusInterval = null;
// Initialize history // Initialize history
fetchHistory(); fetchHistory();
@@ -86,14 +87,14 @@ document.addEventListener("DOMContentLoaded", () => {
const mode = document.querySelector('input[name="rip-mode"]:checked').value; const mode = document.querySelector('input[name="rip-mode"]:checked').value;
setLoading(true); setLoading(true);
statusTitle.textContent = "Connecting to Threads..."; startProgressAnimation();
statusDesc.textContent = "Launching headless browser and resolving URL...";
try { try {
const startTime = performance.now();
const response = await fetch("/api/rip", { const response = await fetch("/api/rip", {
method: "POST", method: "POST",
headers: { "Content-Type": "application/json" }, headers: { "Content-Type": "application/json" },
body: JSON.stringify({ url, mode, max_scrolls: 5 }) body: JSON.stringify({ url, mode, include_images: false, max_scrolls: 4 })
}); });
if (!response.ok) { if (!response.ok) {
@@ -102,18 +103,44 @@ document.addEventListener("DOMContentLoaded", () => {
} }
const data = await response.json(); const data = await response.json();
const elapsed = ((performance.now() - startTime) / 1000).toFixed(1);
currentThread = data; currentThread = data;
renderThread(data); renderThread(data);
fetchHistory(); fetchHistory();
showToast("Thread successfully ripped & saved!"); showToast(`Thread ripped in ${elapsed}s!`);
} catch (err) { } catch (err) {
console.error(err); console.error(err);
showToast(`Error: ${err.message}`, true); showToast(`Error: ${err.message}`, true);
} finally { } finally {
stopProgressAnimation();
setLoading(false); setLoading(false);
} }
} }
function startProgressAnimation() {
const steps = [
{ title: "Connecting to Threads...", desc: "Bypassing heavy assets and resolving redirects..." },
{ title: "Extracting continuation posts...", desc: "Tracking author replies and thread structure..." },
{ title: "Stitching post segments...", desc: "Generating Markdown, HTML, and Plain Text formats..." }
];
let stepIdx = 0;
statusTitle.textContent = steps[0].title;
statusDesc.textContent = steps[0].desc;
clearInterval(statusInterval);
statusInterval = setInterval(() => {
stepIdx++;
if (stepIdx < steps.length) {
statusTitle.textContent = steps[stepIdx].title;
statusDesc.textContent = steps[stepIdx].desc;
}
}, 1200);
}
function stopProgressAnimation() {
clearInterval(statusInterval);
}
function setLoading(isLoading) { function setLoading(isLoading) {
const btnText = ripBtn.querySelector(".btn-text"); const btnText = ripBtn.querySelector(".btn-text");
const spinner = ripBtn.querySelector(".btn-spinner"); const spinner = ripBtn.querySelector(".btn-spinner");
@@ -134,16 +161,11 @@ document.addEventListener("DOMContentLoaded", () => {
function renderThread(thread) { function renderThread(thread) {
resultSection.classList.remove("hidden"); resultSection.classList.remove("hidden");
// Header info // Header info (clean text header, no graphic avatar)
resAuthorName.textContent = thread.author_name; resAuthorName.textContent = thread.author_name;
resAuthorLink.textContent = `@${thread.author_handle}`; resAuthorLink.textContent = `@${thread.author_handle}`;
resAuthorLink.href = thread.canonical_url; resAuthorLink.href = thread.canonical_url;
if (thread.author_avatar) {
resAvatar.src = thread.author_avatar;
resAvatar.classList.remove("hidden");
} else {
resAvatar.classList.add("hidden"); resAvatar.classList.add("hidden");
}
resSegmentCount.textContent = thread.segment_count; resSegmentCount.textContent = thread.segment_count;
resWordCount.textContent = thread.total_words; resWordCount.textContent = thread.total_words;
Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

+15
View File
@@ -0,0 +1,15 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" width="64" height="64">
<defs>
<linearGradient id="logo-grad" x1="0%" y1="0%" x2="100%" y2="100%">
<stop offset="0%" stop-color="#1f6feb" />
<stop offset="100%" stop-color="#8957e5" />
</linearGradient>
</defs>
<!-- Rounded badge background -->
<rect width="64" height="64" rx="16" fill="url(#logo-grad)" />
<!-- Centered Chain/Link Icon -->
<g transform="translate(8, 8) scale(2)" fill="none" stroke="#ffffff" stroke-width="2.3" stroke-linecap="round" stroke-linejoin="round">
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71" />
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71" />
</g>
</svg>

After

Width:  |  Height:  |  Size: 739 B

+5
View File
@@ -4,6 +4,11 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Thread Ripper - Stitch Threads into Single Posts</title> <title>Thread Ripper - Stitch Threads into Single Posts</title>
<!-- Favicon and Shortcut Icons matching the app logo -->
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
<link rel="alternate icon" type="image/png" href="/favicon.png">
<link rel="shortcut icon" href="/favicon.ico">
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
<link rel="stylesheet" href="style.css"> <link rel="stylesheet" href="style.css">
<link rel="preconnect" href="https://fonts.googleapis.com"> <link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+15 -31
View File
@@ -5,7 +5,6 @@ from .models import PostSegment
def clean_segment_text(text: str) -> str: def clean_segment_text(text: str) -> str:
"""Clean up whitespace and trailing artifacts from text.""" """Clean up whitespace and trailing artifacts from text."""
lines = [l.strip() for l in text.split("\n")] lines = [l.strip() for l in text.split("\n")]
# Remove empty lines while preserving deliberate paragraph breaks
cleaned_lines = [] cleaned_lines = []
prev_blank = False prev_blank = False
for line in lines: for line in lines:
@@ -17,8 +16,8 @@ def clean_segment_text(text: str) -> str:
prev_blank = True prev_blank = True
return "\n".join(cleaned_lines).strip() return "\n".join(cleaned_lines).strip()
def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment]) -> str: def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
"""Stitches segments into a single cohesive Markdown document.""" """Stitches segments into a single cohesive Markdown document without graphics."""
lines = [] lines = []
title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post" title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post"
if len(title) > 80: if len(title) > 80:
@@ -35,16 +34,15 @@ def build_markdown(author_name: str, author_handle: str, canonical_url: str, seg
for idx, seg in enumerate(segments): for idx, seg in enumerate(segments):
cleaned_text = clean_segment_text(seg.text) cleaned_text = clean_segment_text(seg.text)
if not cleaned_text and not seg.images: if not cleaned_text:
continue continue
if cleaned_text:
lines.append(cleaned_text) lines.append(cleaned_text)
lines.append("") lines.append("")
# Include media images if any if include_images:
for img in seg.images: for img in seg.images:
alt = img.alt_text or f"Thread Image {idx + 1}" alt = img.alt_text or f"Image {idx + 1}"
lines.append(f"![{alt}]({img.url})") lines.append(f"![{alt}]({img.url})")
lines.append("") lines.append("")
@@ -69,8 +67,8 @@ def build_plain_text(author_name: str, author_handle: str, canonical_url: str, s
output.append("=" * 60) output.append("=" * 60)
return "\n".join(output).strip() return "\n".join(output).strip()
def build_standalone_html(author_name: str, author_handle: str, author_avatar: str, canonical_url: str, segments: List[PostSegment]) -> str: def build_standalone_html(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
"""Generates an elegant, responsive standalone HTML file.""" """Generates an elegant, text-focused standalone HTML document without clutter."""
title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}" title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}"
if len(title) > 80: if len(title) > 80:
title = title[:77] + "..." title = title[:77] + "..."
@@ -78,10 +76,11 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
content_sections = [] content_sections = []
for idx, seg in enumerate(segments): for idx, seg in enumerate(segments):
cleaned_text = clean_segment_text(seg.text) cleaned_text = clean_segment_text(seg.text)
if not cleaned_text:
continue
paragraphs = cleaned_text.split("\n") paragraphs = cleaned_text.split("\n")
section_html = "<div class='thread-segment'>" section_html = "<div class='thread-segment'>"
# Render bullet points or paragraphs cleanly
in_list = False in_list = False
for p in paragraphs: for p in paragraphs:
p = p.strip() p = p.strip()
@@ -100,7 +99,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
if in_list: if in_list:
section_html += "</ul>" section_html += "</ul>"
# Attached images if include_images:
for img in seg.images: for img in seg.images:
section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>" section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>"
@@ -108,7 +107,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
content_sections.append(section_html) content_sections.append(section_html)
body_content = "\n".join(content_sections) body_content = "\n".join(content_sections)
avatar_tag = f"<img class='avatar' src='{author_avatar}' alt='{author_handle}' />" if author_avatar else ""
html = f"""<!DOCTYPE html> html = f"""<!DOCTYPE html>
<html lang="en"> <html lang="en">
@@ -139,7 +137,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
background: var(--bg); background: var(--bg);
color: var(--text); color: var(--text);
line-height: 1.65; line-height: 1.7;
margin: 0; margin: 0;
padding: 32px 16px; padding: 32px 16px;
display: flex; display: flex;
@@ -155,22 +153,14 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
box-shadow: 0 8px 30px rgba(0,0,0,0.12); box-shadow: 0 8px 30px rgba(0,0,0,0.12);
}} }}
.header {{ .header {{
display: flex;
align-items: center;
gap: 16px;
margin-bottom: 28px; margin-bottom: 28px;
padding-bottom: 20px; padding-bottom: 20px;
border-bottom: 1px solid var(--border); border-bottom: 1px solid var(--border);
}} }}
.avatar {{
width: 54px;
height: 54px;
border-radius: 50%;
object-fit: cover;
}}
.author-info h2 {{ .author-info h2 {{
margin: 0; margin: 0 0 6px 0;
font-size: 1.25rem; font-size: 1.35rem;
font-weight: 700;
}} }}
.author-info a {{ .author-info a {{
color: var(--accent); color: var(--accent);
@@ -178,7 +168,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
font-size: 0.95rem; font-size: 0.95rem;
}} }}
.thread-segment {{ .thread-segment {{
margin-bottom: 24px; margin-bottom: 20px;
font-size: 1.1rem; font-size: 1.1rem;
}} }}
.thread-segment p {{ .thread-segment p {{
@@ -191,11 +181,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
.thread-segment li {{ .thread-segment li {{
margin-bottom: 8px; margin-bottom: 8px;
}} }}
.segment-image img {{
max-width: 100%;
border-radius: 12px;
margin-top: 12px;
}}
.footer {{ .footer {{
margin-top: 36px; margin-top: 36px;
padding-top: 20px; padding-top: 20px;
@@ -214,7 +199,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
<body> <body>
<article class="article"> <article class="article">
<div class="header"> <div class="header">
{avatar_tag}
<div class="author-info"> <div class="author-info">
<h2>{author_name}</h2> <h2>{author_name}</h2>
<a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a> <a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a>