Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
801008adfe | ||
|
|
8a91a05c2a | ||
|
|
7609d70dd1 | ||
|
|
42f050ae9a | ||
|
|
56d9aa4d30 | ||
|
|
c1129017d0 |
No files matched your search
+9
-2
@@ -5,16 +5,23 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
container_name: thread-ripper
|
container_name: thread-ripper
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- t3_proxy
|
||||||
ports:
|
ports:
|
||||||
- "5000:5000"
|
- "8475:5000"
|
||||||
environment:
|
environment:
|
||||||
- PORT=5000
|
- PORT=5000
|
||||||
- DATA_DIR=/app/data
|
- DATA_DIR=/app/data
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- /home/chrisf/docker/appdata/thread-ripper:/app/data
|
||||||
|
- /home/chrisf/docker/thread-ripper/thread_ripper:/app/thread_ripper:ro
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:5000/api/health\")'"]
|
test: ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:5000/api/health\")'"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
start_period: 15s
|
start_period: 15s
|
||||||
|
|
||||||
|
networks:
|
||||||
|
t3_proxy:
|
||||||
|
external: true
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
import math
|
||||||
|
from PIL import Image, ImageDraw
|
||||||
|
|
||||||
|
def create_icons(output_dir):
|
||||||
|
size = 256
|
||||||
|
img = Image.new("RGBA", (size, size), (0, 0, 0, 0))
|
||||||
|
|
||||||
|
# Draw linear gradient on rounded rectangle
|
||||||
|
# Gradient: (31, 111, 235) to (137, 87, 229)
|
||||||
|
mask = Image.new("L", (size, size), 0)
|
||||||
|
draw_mask = ImageDraw.Draw(mask)
|
||||||
|
radius = int(size * 0.24)
|
||||||
|
draw_mask.rounded_rectangle([(0, 0), (size - 1, size - 1)], radius=radius, fill=255)
|
||||||
|
|
||||||
|
grad = Image.new("RGBA", (size, size), (0, 0, 0, 0))
|
||||||
|
draw_grad = ImageDraw.Draw(grad)
|
||||||
|
for y in range(size):
|
||||||
|
for x in range(size):
|
||||||
|
t = (x + y) / (2 * size)
|
||||||
|
r = int(31 * (1 - t) + 137 * t)
|
||||||
|
g = int(111 * (1 - t) + 87 * t)
|
||||||
|
b = int(235 * (1 - t) + 229 * t)
|
||||||
|
draw_grad.point((x, y), fill=(r, g, b, 255))
|
||||||
|
|
||||||
|
img = Image.composite(grad, img, mask)
|
||||||
|
|
||||||
|
# Draw the white link chain icon in center
|
||||||
|
# Scale from 24x24 viewBox to fit nicely (about 130x130)
|
||||||
|
draw = ImageDraw.Draw(img)
|
||||||
|
stroke_w = int(size * 0.08)
|
||||||
|
|
||||||
|
# To draw smooth link curves, we draw at 4x and downsample
|
||||||
|
hi_size = size * 4
|
||||||
|
hi_img = Image.new("RGBA", (hi_size, hi_size), (0, 0, 0, 0))
|
||||||
|
hi_draw = ImageDraw.Draw(hi_img)
|
||||||
|
|
||||||
|
# Center = (hi_size/2, hi_size/2)
|
||||||
|
# The SVG path:
|
||||||
|
# Top-right link loop: center around (hi_size*0.58, hi_size*0.42)
|
||||||
|
# Bottom-left link loop: center around (hi_size*0.42, hi_size*0.58)
|
||||||
|
# We can draw the rounded pills rotated by 45 degrees
|
||||||
|
|
||||||
|
# Create rotated pill for upper right link
|
||||||
|
link_w = int(hi_size * 0.28)
|
||||||
|
link_h = int(hi_size * 0.52)
|
||||||
|
pill = Image.new("RGBA", (link_w, link_h), (0, 0, 0, 0))
|
||||||
|
pdraw = ImageDraw.Draw(pill)
|
||||||
|
pw = int(hi_size * 0.075)
|
||||||
|
pdraw.rounded_rectangle([(0, 0), (link_w - 1, link_h - 1)], radius=link_w // 2, outline=(255, 255, 255, 255), width=pw)
|
||||||
|
|
||||||
|
# Rotate pill 45 deg
|
||||||
|
rot_pill = pill.rotate(45, resample=Image.Resampling.BICUBIC, expand=True)
|
||||||
|
|
||||||
|
# Paste upper right link
|
||||||
|
offset_x1 = int(hi_size * 0.38)
|
||||||
|
offset_y1 = int(hi_size * 0.16)
|
||||||
|
hi_img.alpha_composite(rot_pill, (offset_x1, offset_y1))
|
||||||
|
|
||||||
|
# Paste bottom left link
|
||||||
|
offset_x2 = int(hi_size * 0.16)
|
||||||
|
offset_y2 = int(hi_size * 0.38)
|
||||||
|
hi_img.alpha_composite(rot_pill, (offset_x2, offset_y2))
|
||||||
|
|
||||||
|
# Downscale hi_img with anti-aliasing
|
||||||
|
icon_layer = hi_img.resize((size, size), Image.Resampling.LANCZOS)
|
||||||
|
img.alpha_composite(icon_layer)
|
||||||
|
|
||||||
|
# Save outputs
|
||||||
|
# 1. High-res PNG (256x256)
|
||||||
|
img.save(f"{output_dir}/favicon.png", "PNG")
|
||||||
|
img.save(f"{output_dir}/apple-touch-icon.png", "PNG")
|
||||||
|
|
||||||
|
# 2. Multi-size ICO (16, 32, 48, 64)
|
||||||
|
img.save(f"{output_dir}/favicon.ico", format="ICO", sizes=[(16, 16), (32, 32), (48, 48), (64, 64)])
|
||||||
|
print("Generated favicon.png, apple-touch-icon.png, favicon.ico successfully!")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
create_icons(r"C:\Users\Chris\.gemini\antigravity\scratch\thread-ripper\thread_ripper\static")
|
||||||
@@ -39,7 +39,8 @@ class RippedThread(BaseModel):
|
|||||||
class RipRequest(BaseModel):
|
class RipRequest(BaseModel):
|
||||||
url: str
|
url: str
|
||||||
mode: str = "author_only" # 'author_only' or 'all_replies'
|
mode: str = "author_only" # 'author_only' or 'all_replies'
|
||||||
max_scrolls: int = 6
|
include_images: bool = False
|
||||||
|
max_scrolls: int = 4
|
||||||
|
|
||||||
class HistorySummary(BaseModel):
|
class HistorySummary(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
|
|||||||
+46
-29
@@ -11,8 +11,10 @@ class ThreadScraper:
|
|||||||
self.headless = headless
|
self.headless = headless
|
||||||
self._browser: Optional[Browser] = None
|
self._browser: Optional[Browser] = None
|
||||||
self._playwright = None
|
self._playwright = None
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
async def get_browser(self) -> Browser:
|
async def get_browser(self) -> Browser:
|
||||||
|
async with self._lock:
|
||||||
if self._browser is None or not self._browser.is_connected():
|
if self._browser is None or not self._browser.is_connected():
|
||||||
if self._playwright is None:
|
if self._playwright is None:
|
||||||
self._playwright = await async_playwright().start()
|
self._playwright = await async_playwright().start()
|
||||||
@@ -23,11 +25,16 @@ class ThreadScraper:
|
|||||||
"--disable-setuid-sandbox",
|
"--disable-setuid-sandbox",
|
||||||
"--disable-dev-shm-usage",
|
"--disable-dev-shm-usage",
|
||||||
"--disable-gpu",
|
"--disable-gpu",
|
||||||
|
"--disable-extensions",
|
||||||
|
"--disable-component-update",
|
||||||
|
"--disable-background-networking",
|
||||||
|
"--mute-audio"
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
return self._browser
|
return self._browser
|
||||||
|
|
||||||
async def close(self):
|
async def close(self):
|
||||||
|
async with self._lock:
|
||||||
if self._browser:
|
if self._browser:
|
||||||
await self._browser.close()
|
await self._browser.close()
|
||||||
self._browser = None
|
self._browser = None
|
||||||
@@ -35,8 +42,8 @@ class ThreadScraper:
|
|||||||
await self._playwright.stop()
|
await self._playwright.stop()
|
||||||
self._playwright = None
|
self._playwright = None
|
||||||
|
|
||||||
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 5) -> RippedThread:
|
async def rip(self, url: str, mode: str = "author_only", include_images: bool = False, max_scrolls: int = 4) -> RippedThread:
|
||||||
"""Scrapes and rips a Threads post and its chained continuation posts."""
|
"""Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
|
||||||
browser = await self.get_browser()
|
browser = await self.get_browser()
|
||||||
context: BrowserContext = await browser.new_context(
|
context: BrowserContext = await browser.new_context(
|
||||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
@@ -45,14 +52,26 @@ class ThreadScraper:
|
|||||||
page: Page = await context.new_page()
|
page: Page = await context.new_page()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Navigate and follow redirects
|
# 1. Performance optimization: Block unnecessary binary media downloads
|
||||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
# We only need the DOM attributes (src, alt, text) which are in the HTML payload
|
||||||
|
await page.route(
|
||||||
|
"**/*",
|
||||||
|
lambda route: route.abort()
|
||||||
|
if route.request.resource_type in ["image", "media", "font"]
|
||||||
|
else route.continue_(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Fast navigation using domcontentloaded instead of networkidle
|
||||||
|
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
|
||||||
canonical_url = page.url
|
canonical_url = page.url
|
||||||
|
|
||||||
# Small delay for hydration
|
# 3. Wait for the primary post container to render
|
||||||
await page.wait_for_timeout(2500)
|
try:
|
||||||
|
await page.wait_for_selector("div[data-pressable-container='true']", timeout=7000)
|
||||||
|
except Exception:
|
||||||
|
pass # Fall back to evaluating whatever rendered
|
||||||
|
|
||||||
# Extract Open Graph metadata
|
# 4. Extract Open Graph metadata
|
||||||
og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''")
|
og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''")
|
||||||
og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''")
|
og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''")
|
||||||
og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''")
|
og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''")
|
||||||
@@ -61,12 +80,10 @@ class ThreadScraper:
|
|||||||
author_handle = ""
|
author_handle = ""
|
||||||
author_name = ""
|
author_name = ""
|
||||||
|
|
||||||
# Check canonical URL for /@username
|
|
||||||
match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url)
|
match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url)
|
||||||
if match_handle:
|
if match_handle:
|
||||||
author_handle = match_handle.group(1)
|
author_handle = match_handle.group(1)
|
||||||
|
|
||||||
# Check og:title e.g. "Beth Parent McMillian (@bethmcmilliandesigns) on Threads"
|
|
||||||
if og_title:
|
if og_title:
|
||||||
m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title)
|
m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title)
|
||||||
if m_title:
|
if m_title:
|
||||||
@@ -77,13 +94,19 @@ class ThreadScraper:
|
|||||||
if not author_name:
|
if not author_name:
|
||||||
author_name = author_handle or "Threads Author"
|
author_name = author_handle or "Threads Author"
|
||||||
|
|
||||||
# Author profile avatar
|
|
||||||
author_avatar = og_image if og_image else None
|
author_avatar = og_image if og_image else None
|
||||||
|
|
||||||
# Progressively scroll to load dynamic replies in the thread
|
# 5. Smart adaptive scrolling for thread continuation replies
|
||||||
|
prev_containers_count = 0
|
||||||
for _ in range(max_scrolls):
|
for _ in range(max_scrolls):
|
||||||
await page.evaluate("window.scrollBy(0, 1200)")
|
containers = await page.query_selector_all("div[data-pressable-container='true']")
|
||||||
await page.wait_for_timeout(1000)
|
curr_count = len(containers)
|
||||||
|
# If we already have multiple containers and count didn't change on last scroll, settle early
|
||||||
|
if curr_count > 1 and curr_count == prev_containers_count:
|
||||||
|
break
|
||||||
|
prev_containers_count = curr_count
|
||||||
|
await page.evaluate("window.scrollBy(0, 1400)")
|
||||||
|
await page.wait_for_timeout(350)
|
||||||
|
|
||||||
# Query all pressable post containers
|
# Query all pressable post containers
|
||||||
containers = await page.query_selector_all("div[data-pressable-container='true']")
|
containers = await page.query_selector_all("div[data-pressable-container='true']")
|
||||||
@@ -92,7 +115,6 @@ class ThreadScraper:
|
|||||||
seen_texts = set()
|
seen_texts = set()
|
||||||
|
|
||||||
for idx, c in enumerate(containers):
|
for idx, c in enumerate(containers):
|
||||||
# 1. Identify post author handle
|
|
||||||
user_links = await c.query_selector_all("a[href*='/@']")
|
user_links = await c.query_selector_all("a[href*='/@']")
|
||||||
post_handle = None
|
post_handle = None
|
||||||
post_url = None
|
post_url = None
|
||||||
@@ -113,7 +135,7 @@ class ThreadScraper:
|
|||||||
if badge:
|
if badge:
|
||||||
is_author_badge = True
|
is_author_badge = True
|
||||||
|
|
||||||
# Determine if this segment belongs to the original author
|
# Determine if this segment belongs to original author
|
||||||
is_author_post = False
|
is_author_post = False
|
||||||
if idx == 0:
|
if idx == 0:
|
||||||
is_author_post = True
|
is_author_post = True
|
||||||
@@ -124,21 +146,18 @@ class ThreadScraper:
|
|||||||
elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()):
|
elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()):
|
||||||
is_author_post = True
|
is_author_post = True
|
||||||
|
|
||||||
# If author_only mode is on and this is not by the author, skip
|
|
||||||
if mode == "author_only" and not is_author_post:
|
if mode == "author_only" and not is_author_post:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# 2. Extract text with dir='auto'
|
# Extract text lines
|
||||||
dir_els = await c.query_selector_all("[dir='auto']")
|
dir_els = await c.query_selector_all("[dir='auto']")
|
||||||
segment_text_parts = []
|
segment_text_parts = []
|
||||||
for del_el in dir_els:
|
for del_el in dir_els:
|
||||||
txt = (await del_el.inner_text()).strip()
|
txt = (await del_el.inner_text()).strip()
|
||||||
if not txt:
|
if not txt:
|
||||||
continue
|
continue
|
||||||
# Ignore handle, author badge, bullet dots, stats
|
|
||||||
if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle:
|
if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle:
|
||||||
continue
|
continue
|
||||||
# Ignore pure stat numbers (e.g., '16.2K', '5d', '2K', '711')
|
|
||||||
if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt):
|
if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt):
|
||||||
continue
|
continue
|
||||||
if txt not in segment_text_parts:
|
if txt not in segment_text_parts:
|
||||||
@@ -148,21 +167,20 @@ class ThreadScraper:
|
|||||||
if not combined_text and not is_author_post:
|
if not combined_text and not is_author_post:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Deduplicate identical consecutive segments
|
|
||||||
text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest()
|
text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest()
|
||||||
if text_hash in seen_texts and combined_text:
|
if text_hash in seen_texts and combined_text:
|
||||||
continue
|
continue
|
||||||
seen_texts.add(text_hash)
|
seen_texts.add(text_hash)
|
||||||
|
|
||||||
# 3. Extract media images inside post (exclude profile pictures)
|
# Extract post images (only real content photos, never profile avatars)
|
||||||
imgs = await c.query_selector_all("img")
|
|
||||||
post_images: List[MediaItem] = []
|
post_images: List[MediaItem] = []
|
||||||
|
if include_images:
|
||||||
|
imgs = await c.query_selector_all("img")
|
||||||
for img in imgs:
|
for img in imgs:
|
||||||
src = await img.get_attribute("src")
|
src = await img.get_attribute("src")
|
||||||
alt = await img.get_attribute("alt") or ""
|
alt = await img.get_attribute("alt") or ""
|
||||||
if src:
|
if src:
|
||||||
# Exclude small avatars or generic SVG icons
|
if "profile_pic" in src or "profile picture" in alt.lower() or ("scontent" in src and "206x206" in src):
|
||||||
if "profile_pic" in src or "scontent" in src and "206x206" in src:
|
|
||||||
continue
|
continue
|
||||||
post_images.append(MediaItem(type="image", url=src, alt_text=alt))
|
post_images.append(MediaItem(type="image", url=src, alt_text=alt))
|
||||||
|
|
||||||
@@ -179,7 +197,7 @@ class ThreadScraper:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# If no segments could be extracted from containers, fallback to OG description
|
# Fallback to OG description if empty
|
||||||
if not raw_segments and og_desc:
|
if not raw_segments and og_desc:
|
||||||
raw_segments.append(
|
raw_segments.append(
|
||||||
PostSegment(
|
PostSegment(
|
||||||
@@ -194,12 +212,11 @@ class ThreadScraper:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Construct stitched outputs
|
# Build stitched outputs without graphics by default
|
||||||
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments)
|
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
|
||||||
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
|
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
|
||||||
html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments)
|
html_out = build_standalone_html(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
|
||||||
|
|
||||||
# Generate unique ID for this ripped thread
|
|
||||||
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
|
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
|
||||||
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
|
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
|
||||||
if len(title) > 90:
|
if len(title) > 90:
|
||||||
|
|||||||
+17
-1
@@ -1,4 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
|
import asyncio
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from fastapi import FastAPI, HTTPException, Response
|
from fastapi import FastAPI, HTTPException, Response
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
@@ -14,6 +15,8 @@ scraper = ThreadScraper(headless=True)
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
|
# Pre-warm Chromium in background so initial user requests are instant
|
||||||
|
asyncio.create_task(scraper.get_browser())
|
||||||
yield
|
yield
|
||||||
await scraper.close()
|
await scraper.close()
|
||||||
|
|
||||||
@@ -39,7 +42,12 @@ async def rip_endpoint(req: RipRequest):
|
|||||||
raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.")
|
raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ripped = await scraper.rip(url=req.url, mode=req.mode, max_scrolls=req.max_scrolls)
|
ripped = await scraper.rip(
|
||||||
|
url=req.url,
|
||||||
|
mode=req.mode,
|
||||||
|
include_images=req.include_images,
|
||||||
|
max_scrolls=req.max_scrolls
|
||||||
|
)
|
||||||
storage.save(ripped)
|
storage.save(ripped)
|
||||||
return ripped
|
return ripped
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -107,6 +115,14 @@ def download_thread(thread_id: str, format: str):
|
|||||||
def health_check():
|
def health_check():
|
||||||
return {"status": "ok", "app": "Thread Ripper", "version": "1.0.0"}
|
return {"status": "ok", "app": "Thread Ripper", "version": "1.0.0"}
|
||||||
|
|
||||||
|
# Direct favicon endpoint for browsers requesting /favicon.ico
|
||||||
|
@app.get("/favicon.ico", include_in_schema=False)
|
||||||
|
def get_favicon():
|
||||||
|
fav_file = os.path.join(static_dir, "favicon.ico")
|
||||||
|
if os.path.exists(fav_file):
|
||||||
|
return FileResponse(fav_file)
|
||||||
|
return Response(status_code=404)
|
||||||
|
|
||||||
# Mount static frontend directory
|
# Mount static frontend directory
|
||||||
static_dir = os.path.join(os.path.dirname(__file__), "static")
|
static_dir = os.path.join(os.path.dirname(__file__), "static")
|
||||||
if os.path.exists(static_dir):
|
if os.path.exists(static_dir):
|
||||||
|
|||||||
+32
-10
@@ -43,6 +43,7 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
|
|
||||||
let currentThread = null;
|
let currentThread = null;
|
||||||
let allHistory = [];
|
let allHistory = [];
|
||||||
|
let statusInterval = null;
|
||||||
|
|
||||||
// Initialize history
|
// Initialize history
|
||||||
fetchHistory();
|
fetchHistory();
|
||||||
@@ -86,14 +87,14 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
const mode = document.querySelector('input[name="rip-mode"]:checked').value;
|
const mode = document.querySelector('input[name="rip-mode"]:checked').value;
|
||||||
|
|
||||||
setLoading(true);
|
setLoading(true);
|
||||||
statusTitle.textContent = "Connecting to Threads...";
|
startProgressAnimation();
|
||||||
statusDesc.textContent = "Launching headless browser and resolving URL...";
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const startTime = performance.now();
|
||||||
const response = await fetch("/api/rip", {
|
const response = await fetch("/api/rip", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ url, mode, max_scrolls: 5 })
|
body: JSON.stringify({ url, mode, include_images: false, max_scrolls: 4 })
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
@@ -102,18 +103,44 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
const elapsed = ((performance.now() - startTime) / 1000).toFixed(1);
|
||||||
currentThread = data;
|
currentThread = data;
|
||||||
renderThread(data);
|
renderThread(data);
|
||||||
fetchHistory();
|
fetchHistory();
|
||||||
showToast("Thread successfully ripped & saved!");
|
showToast(`Thread ripped in ${elapsed}s!`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
showToast(`Error: ${err.message}`, true);
|
showToast(`Error: ${err.message}`, true);
|
||||||
} finally {
|
} finally {
|
||||||
|
stopProgressAnimation();
|
||||||
setLoading(false);
|
setLoading(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function startProgressAnimation() {
|
||||||
|
const steps = [
|
||||||
|
{ title: "Connecting to Threads...", desc: "Bypassing heavy assets and resolving redirects..." },
|
||||||
|
{ title: "Extracting continuation posts...", desc: "Tracking author replies and thread structure..." },
|
||||||
|
{ title: "Stitching post segments...", desc: "Generating Markdown, HTML, and Plain Text formats..." }
|
||||||
|
];
|
||||||
|
let stepIdx = 0;
|
||||||
|
statusTitle.textContent = steps[0].title;
|
||||||
|
statusDesc.textContent = steps[0].desc;
|
||||||
|
|
||||||
|
clearInterval(statusInterval);
|
||||||
|
statusInterval = setInterval(() => {
|
||||||
|
stepIdx++;
|
||||||
|
if (stepIdx < steps.length) {
|
||||||
|
statusTitle.textContent = steps[stepIdx].title;
|
||||||
|
statusDesc.textContent = steps[stepIdx].desc;
|
||||||
|
}
|
||||||
|
}, 1200);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopProgressAnimation() {
|
||||||
|
clearInterval(statusInterval);
|
||||||
|
}
|
||||||
|
|
||||||
function setLoading(isLoading) {
|
function setLoading(isLoading) {
|
||||||
const btnText = ripBtn.querySelector(".btn-text");
|
const btnText = ripBtn.querySelector(".btn-text");
|
||||||
const spinner = ripBtn.querySelector(".btn-spinner");
|
const spinner = ripBtn.querySelector(".btn-spinner");
|
||||||
@@ -134,16 +161,11 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
function renderThread(thread) {
|
function renderThread(thread) {
|
||||||
resultSection.classList.remove("hidden");
|
resultSection.classList.remove("hidden");
|
||||||
|
|
||||||
// Header info
|
// Header info (clean text header, no graphic avatar)
|
||||||
resAuthorName.textContent = thread.author_name;
|
resAuthorName.textContent = thread.author_name;
|
||||||
resAuthorLink.textContent = `@${thread.author_handle}`;
|
resAuthorLink.textContent = `@${thread.author_handle}`;
|
||||||
resAuthorLink.href = thread.canonical_url;
|
resAuthorLink.href = thread.canonical_url;
|
||||||
if (thread.author_avatar) {
|
|
||||||
resAvatar.src = thread.author_avatar;
|
|
||||||
resAvatar.classList.remove("hidden");
|
|
||||||
} else {
|
|
||||||
resAvatar.classList.add("hidden");
|
resAvatar.classList.add("hidden");
|
||||||
}
|
|
||||||
|
|
||||||
resSegmentCount.textContent = thread.segment_count;
|
resSegmentCount.textContent = thread.segment_count;
|
||||||
resWordCount.textContent = thread.total_words;
|
resWordCount.textContent = thread.total_words;
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 10 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
@@ -0,0 +1,15 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" width="64" height="64">
|
||||||
|
<defs>
|
||||||
|
<linearGradient id="logo-grad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||||
|
<stop offset="0%" stop-color="#1f6feb" />
|
||||||
|
<stop offset="100%" stop-color="#8957e5" />
|
||||||
|
</linearGradient>
|
||||||
|
</defs>
|
||||||
|
<!-- Rounded badge background -->
|
||||||
|
<rect width="64" height="64" rx="16" fill="url(#logo-grad)" />
|
||||||
|
<!-- Centered Chain/Link Icon -->
|
||||||
|
<g transform="translate(8, 8) scale(2)" fill="none" stroke="#ffffff" stroke-width="2.3" stroke-linecap="round" stroke-linejoin="round">
|
||||||
|
<path d="M10 13a5 5 0 0 0 7.54.54l3-3a5 5 0 0 0-7.07-7.07l-1.72 1.71" />
|
||||||
|
<path d="M14 11a5 5 0 0 0-7.54-.54l-3 3a5 5 0 0 0 7.07 7.07l1.71-1.71" />
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 739 B |
@@ -4,6 +4,11 @@
|
|||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>Thread Ripper - Stitch Threads into Single Posts</title>
|
<title>Thread Ripper - Stitch Threads into Single Posts</title>
|
||||||
|
<!-- Favicon and Shortcut Icons matching the app logo -->
|
||||||
|
<link rel="icon" type="image/svg+xml" href="/favicon.svg">
|
||||||
|
<link rel="alternate icon" type="image/png" href="/favicon.png">
|
||||||
|
<link rel="shortcut icon" href="/favicon.ico">
|
||||||
|
<link rel="apple-touch-icon" href="/apple-touch-icon.png">
|
||||||
<link rel="stylesheet" href="style.css">
|
<link rel="stylesheet" href="style.css">
|
||||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||||
|
|||||||
+15
-31
@@ -5,7 +5,6 @@ from .models import PostSegment
|
|||||||
def clean_segment_text(text: str) -> str:
|
def clean_segment_text(text: str) -> str:
|
||||||
"""Clean up whitespace and trailing artifacts from text."""
|
"""Clean up whitespace and trailing artifacts from text."""
|
||||||
lines = [l.strip() for l in text.split("\n")]
|
lines = [l.strip() for l in text.split("\n")]
|
||||||
# Remove empty lines while preserving deliberate paragraph breaks
|
|
||||||
cleaned_lines = []
|
cleaned_lines = []
|
||||||
prev_blank = False
|
prev_blank = False
|
||||||
for line in lines:
|
for line in lines:
|
||||||
@@ -17,8 +16,8 @@ def clean_segment_text(text: str) -> str:
|
|||||||
prev_blank = True
|
prev_blank = True
|
||||||
return "\n".join(cleaned_lines).strip()
|
return "\n".join(cleaned_lines).strip()
|
||||||
|
|
||||||
def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment]) -> str:
|
def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
|
||||||
"""Stitches segments into a single cohesive Markdown document."""
|
"""Stitches segments into a single cohesive Markdown document without graphics."""
|
||||||
lines = []
|
lines = []
|
||||||
title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post"
|
title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post"
|
||||||
if len(title) > 80:
|
if len(title) > 80:
|
||||||
@@ -35,16 +34,15 @@ def build_markdown(author_name: str, author_handle: str, canonical_url: str, seg
|
|||||||
|
|
||||||
for idx, seg in enumerate(segments):
|
for idx, seg in enumerate(segments):
|
||||||
cleaned_text = clean_segment_text(seg.text)
|
cleaned_text = clean_segment_text(seg.text)
|
||||||
if not cleaned_text and not seg.images:
|
if not cleaned_text:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if cleaned_text:
|
|
||||||
lines.append(cleaned_text)
|
lines.append(cleaned_text)
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
# Include media images if any
|
if include_images:
|
||||||
for img in seg.images:
|
for img in seg.images:
|
||||||
alt = img.alt_text or f"Thread Image {idx + 1}"
|
alt = img.alt_text or f"Image {idx + 1}"
|
||||||
lines.append(f"")
|
lines.append(f"")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
@@ -69,8 +67,8 @@ def build_plain_text(author_name: str, author_handle: str, canonical_url: str, s
|
|||||||
output.append("=" * 60)
|
output.append("=" * 60)
|
||||||
return "\n".join(output).strip()
|
return "\n".join(output).strip()
|
||||||
|
|
||||||
def build_standalone_html(author_name: str, author_handle: str, author_avatar: str, canonical_url: str, segments: List[PostSegment]) -> str:
|
def build_standalone_html(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
|
||||||
"""Generates an elegant, responsive standalone HTML file."""
|
"""Generates an elegant, text-focused standalone HTML document without clutter."""
|
||||||
title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}"
|
title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}"
|
||||||
if len(title) > 80:
|
if len(title) > 80:
|
||||||
title = title[:77] + "..."
|
title = title[:77] + "..."
|
||||||
@@ -78,10 +76,11 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
content_sections = []
|
content_sections = []
|
||||||
for idx, seg in enumerate(segments):
|
for idx, seg in enumerate(segments):
|
||||||
cleaned_text = clean_segment_text(seg.text)
|
cleaned_text = clean_segment_text(seg.text)
|
||||||
|
if not cleaned_text:
|
||||||
|
continue
|
||||||
paragraphs = cleaned_text.split("\n")
|
paragraphs = cleaned_text.split("\n")
|
||||||
section_html = "<div class='thread-segment'>"
|
section_html = "<div class='thread-segment'>"
|
||||||
|
|
||||||
# Render bullet points or paragraphs cleanly
|
|
||||||
in_list = False
|
in_list = False
|
||||||
for p in paragraphs:
|
for p in paragraphs:
|
||||||
p = p.strip()
|
p = p.strip()
|
||||||
@@ -100,7 +99,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
if in_list:
|
if in_list:
|
||||||
section_html += "</ul>"
|
section_html += "</ul>"
|
||||||
|
|
||||||
# Attached images
|
if include_images:
|
||||||
for img in seg.images:
|
for img in seg.images:
|
||||||
section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>"
|
section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>"
|
||||||
|
|
||||||
@@ -108,7 +107,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
content_sections.append(section_html)
|
content_sections.append(section_html)
|
||||||
|
|
||||||
body_content = "\n".join(content_sections)
|
body_content = "\n".join(content_sections)
|
||||||
avatar_tag = f"<img class='avatar' src='{author_avatar}' alt='{author_handle}' />" if author_avatar else ""
|
|
||||||
|
|
||||||
html = f"""<!DOCTYPE html>
|
html = f"""<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
@@ -139,7 +137,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||||
background: var(--bg);
|
background: var(--bg);
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
line-height: 1.65;
|
line-height: 1.7;
|
||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 32px 16px;
|
padding: 32px 16px;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -155,22 +153,14 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
box-shadow: 0 8px 30px rgba(0,0,0,0.12);
|
box-shadow: 0 8px 30px rgba(0,0,0,0.12);
|
||||||
}}
|
}}
|
||||||
.header {{
|
.header {{
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 16px;
|
|
||||||
margin-bottom: 28px;
|
margin-bottom: 28px;
|
||||||
padding-bottom: 20px;
|
padding-bottom: 20px;
|
||||||
border-bottom: 1px solid var(--border);
|
border-bottom: 1px solid var(--border);
|
||||||
}}
|
}}
|
||||||
.avatar {{
|
|
||||||
width: 54px;
|
|
||||||
height: 54px;
|
|
||||||
border-radius: 50%;
|
|
||||||
object-fit: cover;
|
|
||||||
}}
|
|
||||||
.author-info h2 {{
|
.author-info h2 {{
|
||||||
margin: 0;
|
margin: 0 0 6px 0;
|
||||||
font-size: 1.25rem;
|
font-size: 1.35rem;
|
||||||
|
font-weight: 700;
|
||||||
}}
|
}}
|
||||||
.author-info a {{
|
.author-info a {{
|
||||||
color: var(--accent);
|
color: var(--accent);
|
||||||
@@ -178,7 +168,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
}}
|
}}
|
||||||
.thread-segment {{
|
.thread-segment {{
|
||||||
margin-bottom: 24px;
|
margin-bottom: 20px;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
}}
|
}}
|
||||||
.thread-segment p {{
|
.thread-segment p {{
|
||||||
@@ -191,11 +181,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
.thread-segment li {{
|
.thread-segment li {{
|
||||||
margin-bottom: 8px;
|
margin-bottom: 8px;
|
||||||
}}
|
}}
|
||||||
.segment-image img {{
|
|
||||||
max-width: 100%;
|
|
||||||
border-radius: 12px;
|
|
||||||
margin-top: 12px;
|
|
||||||
}}
|
|
||||||
.footer {{
|
.footer {{
|
||||||
margin-top: 36px;
|
margin-top: 36px;
|
||||||
padding-top: 20px;
|
padding-top: 20px;
|
||||||
@@ -214,7 +199,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
<body>
|
<body>
|
||||||
<article class="article">
|
<article class="article">
|
||||||
<div class="header">
|
<div class="header">
|
||||||
{avatar_tag}
|
|
||||||
<div class="author-info">
|
<div class="author-info">
|
||||||
<h2>{author_name}</h2>
|
<h2>{author_name}</h2>
|
||||||
<a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a>
|
<a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a>
|
||||||
|
|||||||
Reference in new issue
Block a user