chrisf/thread-ripper
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Performance optimizations: abort binary assets, domcontentloaded, adaptive scroll, browser pre-warming (7x speedup)

Browse Source
This commit is contained in:
airent106 committed 2026-10-06 22:01:40 -06:00
1 parent c1129017d0
commit 56d9aa4d30
2 files changed
+64 -45
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

thread_ripper/scraper.py
+61 -45
View File
@@ -11,32 +11,39 @@ class ThreadScraper:
self.headless = headless self.headless = headless
self._browser: Optional[Browser] = None self._browser: Optional[Browser] = None
self._playwright = None self._playwright = None
self._lock = asyncio.Lock()
async def get_browser(self) -> Browser: async def get_browser(self) -> Browser:
if self._browser is None or not self._browser.is_connected(): async with self._lock:
if self._playwright is None: if self._browser is None or not self._browser.is_connected():
self._playwright = await async_playwright().start() if self._playwright is None:
self._browser = await self._playwright.chromium.launch( self._playwright = await async_playwright().start()
headless=self.headless, self._browser = await self._playwright.chromium.launch(
args=[ headless=self.headless,
"--no-sandbox", args=[
"--disable-setuid-sandbox", "--no-sandbox",
"--disable-dev-shm-usage", "--disable-setuid-sandbox",
"--disable-gpu", "--disable-dev-shm-usage",
] "--disable-gpu",
) "--disable-extensions",
return self._browser "--disable-component-update",
"--disable-background-networking",
"--mute-audio"
]
)
return self._browser
async def close(self): async def close(self):
if self._browser: async with self._lock:
await self._browser.close() if self._browser:
self._browser = None await self._browser.close()
if self._playwright: self._browser = None
await self._playwright.stop() if self._playwright:
self._playwright = None await self._playwright.stop()
self._playwright = None
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 5) -> RippedThread: async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 4) -> RippedThread:
"""Scrapes and rips a Threads post and its chained continuation posts.""" """Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
browser = await self.get_browser() browser = await self.get_browser()
context: BrowserContext = await browser.new_context( context: BrowserContext = await browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
@@ -45,14 +52,26 @@ class ThreadScraper:
page: Page = await context.new_page() page: Page = await context.new_page()
try: try:
# Navigate and follow redirects # 1. Performance optimization: Block unnecessary binary media downloads
await page.goto(url, wait_until="networkidle", timeout=30000) # We only need the DOM attributes (src, alt, text) which are in the HTML payload
await page.route(
"**/*",
lambda route: route.abort()
if route.request.resource_type in ["image", "media", "font"]
else route.continue_()
)
# 2. Fast navigation using domcontentloaded instead of networkidle
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
canonical_url = page.url canonical_url = page.url
# Small delay for hydration # 3. Wait for the primary post container to render
await page.wait_for_timeout(2500) try:
await page.wait_for_selector("div[data-pressable-container='true']", timeout=7000)
except Exception:
pass # Fall back to evaluating whatever rendered
# Extract Open Graph metadata # 4. Extract Open Graph metadata
og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''") og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''")
og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''") og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''")
og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''") og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''")
@@ -61,12 +80,10 @@ class ThreadScraper:
author_handle = "" author_handle = ""
author_name = "" author_name = ""
# Check canonical URL for /@username
match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url) match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url)
if match_handle: if match_handle:
author_handle = match_handle.group(1) author_handle = match_handle.group(1)
# Check og:title e.g. "Beth Parent McMillian (@bethmcmilliandesigns) on Threads"
if og_title: if og_title:
m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title) m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title)
if m_title: if m_title:
@@ -77,13 +94,19 @@ class ThreadScraper:
if not author_name: if not author_name:
author_name = author_handle or "Threads Author" author_name = author_handle or "Threads Author"
# Author profile avatar
author_avatar = og_image if og_image else None author_avatar = og_image if og_image else None
# Progressively scroll to load dynamic replies in the thread # 5. Smart adaptive scrolling for thread continuation replies
prev_containers_count = 0
for _ in range(max_scrolls): for _ in range(max_scrolls):
await page.evaluate("window.scrollBy(0, 1200)") containers = await page.query_selector_all("div[data-pressable-container='true']")
await page.wait_for_timeout(1000) curr_count = len(containers)
# If we already have multiple containers and count didn't change on last scroll, settle early
if curr_count > 1 and curr_count == prev_containers_count:
break
prev_containers_count = curr_count
await page.evaluate("window.scrollBy(0, 1400)")
await page.wait_for_timeout(350)
# Query all pressable post containers # Query all pressable post containers
containers = await page.query_selector_all("div[data-pressable-container='true']") containers = await page.query_selector_all("div[data-pressable-container='true']")
@@ -92,7 +115,6 @@ class ThreadScraper:
seen_texts = set() seen_texts = set()
for idx, c in enumerate(containers): for idx, c in enumerate(containers):
# 1. Identify post author handle
user_links = await c.query_selector_all("a[href*='/@']") user_links = await c.query_selector_all("a[href*='/@']")
post_handle = None post_handle = None
post_url = None post_url = None
@@ -113,7 +135,7 @@ class ThreadScraper:
if badge: if badge:
is_author_badge = True is_author_badge = True
# Determine if this segment belongs to the original author # Determine if this segment belongs to original author
is_author_post = False is_author_post = False
if idx == 0: if idx == 0:
is_author_post = True is_author_post = True
@@ -124,21 +146,18 @@ class ThreadScraper:
elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()): elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()):
is_author_post = True is_author_post = True
# If author_only mode is on and this is not by the author, skip
if mode == "author_only" and not is_author_post: if mode == "author_only" and not is_author_post:
continue continue
# 2. Extract text with dir='auto' # Extract text lines
dir_els = await c.query_selector_all("[dir='auto']") dir_els = await c.query_selector_all("[dir='auto']")
segment_text_parts = [] segment_text_parts = []
for del_el in dir_els: for del_el in dir_els:
txt = (await del_el.inner_text()).strip() txt = (await del_el.inner_text()).strip()
if not txt: if not txt:
continue continue
# Ignore handle, author badge, bullet dots, stats
if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle: if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle:
continue continue
# Ignore pure stat numbers (e.g., '16.2K', '5d', '2K', '711')
if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt): if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt):
continue continue
if txt not in segment_text_parts: if txt not in segment_text_parts:
@@ -148,21 +167,19 @@ class ThreadScraper:
if not combined_text and not is_author_post: if not combined_text and not is_author_post:
continue continue
# Deduplicate identical consecutive segments
text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest() text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest()
if text_hash in seen_texts and combined_text: if text_hash in seen_texts and combined_text:
continue continue
seen_texts.add(text_hash) seen_texts.add(text_hash)
# 3. Extract media images inside post (exclude profile pictures) # Extract post images
imgs = await c.query_selector_all("img") imgs = await c.query_selector_all("img")
post_images: List[MediaItem] = [] post_images: List[MediaItem] = []
for img in imgs: for img in imgs:
src = await img.get_attribute("src") src = await img.get_attribute("src")
alt = await img.get_attribute("alt") or "" alt = await img.get_attribute("alt") or ""
if src: if src:
# Exclude small avatars or generic SVG icons if "profile_pic" in src or ("scontent" in src and "206x206" in src):
if "profile_pic" in src or "scontent" in src and "206x206" in src:
continue continue
post_images.append(MediaItem(type="image", url=src, alt_text=alt)) post_images.append(MediaItem(type="image", url=src, alt_text=alt))
@@ -179,7 +196,7 @@ class ThreadScraper:
) )
) )
# If no segments could be extracted from containers, fallback to OG description # Fallback to OG description if empty
if not raw_segments and og_desc: if not raw_segments and og_desc:
raw_segments.append( raw_segments.append(
PostSegment( PostSegment(
@@ -194,12 +211,11 @@ class ThreadScraper:
) )
) )
# Construct stitched outputs # Build stitched outputs
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments) markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments)
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments) plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments) html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments)
# Generate unique ID for this ripped thread
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12] thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}" title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
if len(title) > 90: if len(title) > 90:
thread_ripper/server.py
+3
View File
@@ -1,4 +1,5 @@
import os import os
import asyncio
from contextlib import asynccontextmanager from contextlib import asynccontextmanager
from fastapi import FastAPI, HTTPException, Response from fastapi import FastAPI, HTTPException, Response
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
@@ -14,6 +15,8 @@ scraper = ThreadScraper(headless=True)
@asynccontextmanager @asynccontextmanager
async def lifespan(app: FastAPI): async def lifespan(app: FastAPI):
# Pre-warm Chromium in background so initial user requests are instant
asyncio.create_task(scraper.get_browser())
yield yield
await scraper.close() await scraper.close()