Performance optimizations: abort binary assets, domcontentloaded, adaptive scroll, browser pre-warming (7x speedup)
This commit is contained in:
airent106 committed
1 parent
c1129017d0
commit
56d9aa4d30
2 files changed
+64
-45
No files matched your search
+61
-45
@@ -11,32 +11,39 @@ class ThreadScraper:
|
||||
self.headless = headless
|
||||
self._browser: Optional[Browser] = None
|
||||
self._playwright = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get_browser(self) -> Browser:
|
||||
if self._browser is None or not self._browser.is_connected():
|
||||
if self._playwright is None:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self.headless,
|
||||
args=[
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
]
|
||||
)
|
||||
return self._browser
|
||||
async with self._lock:
|
||||
if self._browser is None or not self._browser.is_connected():
|
||||
if self._playwright is None:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self.headless,
|
||||
args=[
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--disable-extensions",
|
||||
"--disable-component-update",
|
||||
"--disable-background-networking",
|
||||
"--mute-audio"
|
||||
]
|
||||
)
|
||||
return self._browser
|
||||
|
||||
async def close(self):
|
||||
if self._browser:
|
||||
await self._browser.close()
|
||||
self._browser = None
|
||||
if self._playwright:
|
||||
await self._playwright.stop()
|
||||
self._playwright = None
|
||||
async with self._lock:
|
||||
if self._browser:
|
||||
await self._browser.close()
|
||||
self._browser = None
|
||||
if self._playwright:
|
||||
await self._playwright.stop()
|
||||
self._playwright = None
|
||||
|
||||
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 5) -> RippedThread:
|
||||
"""Scrapes and rips a Threads post and its chained continuation posts."""
|
||||
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 4) -> RippedThread:
|
||||
"""Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
|
||||
browser = await self.get_browser()
|
||||
context: BrowserContext = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
@@ -45,14 +52,26 @@ class ThreadScraper:
|
||||
page: Page = await context.new_page()
|
||||
|
||||
try:
|
||||
# Navigate and follow redirects
|
||||
await page.goto(url, wait_until="networkidle", timeout=30000)
|
||||
# 1. Performance optimization: Block unnecessary binary media downloads
|
||||
# We only need the DOM attributes (src, alt, text) which are in the HTML payload
|
||||
await page.route(
|
||||
"**/*",
|
||||
lambda route: route.abort()
|
||||
if route.request.resource_type in ["image", "media", "font"]
|
||||
else route.continue_()
|
||||
)
|
||||
|
||||
# 2. Fast navigation using domcontentloaded instead of networkidle
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=20000)
|
||||
canonical_url = page.url
|
||||
|
||||
# Small delay for hydration
|
||||
await page.wait_for_timeout(2500)
|
||||
# 3. Wait for the primary post container to render
|
||||
try:
|
||||
await page.wait_for_selector("div[data-pressable-container='true']", timeout=7000)
|
||||
except Exception:
|
||||
pass # Fall back to evaluating whatever rendered
|
||||
|
||||
# Extract Open Graph metadata
|
||||
# 4. Extract Open Graph metadata
|
||||
og_title = await page.evaluate("() => document.querySelector('meta[property=\"og:title\"]')?.content || ''")
|
||||
og_image = await page.evaluate("() => document.querySelector('meta[property=\"og:image\"]')?.content || ''")
|
||||
og_desc = await page.evaluate("() => document.querySelector('meta[property=\"og:description\"]')?.content || ''")
|
||||
@@ -61,12 +80,10 @@ class ThreadScraper:
|
||||
author_handle = ""
|
||||
author_name = ""
|
||||
|
||||
# Check canonical URL for /@username
|
||||
match_handle = re.search(r"/@([a-zA-Z0-9_\.]+)", canonical_url)
|
||||
if match_handle:
|
||||
author_handle = match_handle.group(1)
|
||||
|
||||
# Check og:title e.g. "Beth Parent McMillian (@bethmcmilliandesigns) on Threads"
|
||||
if og_title:
|
||||
m_title = re.search(r"^(.*?)\s*\(@?([a-zA-Z0-9_\.]+)\)", og_title)
|
||||
if m_title:
|
||||
@@ -77,13 +94,19 @@ class ThreadScraper:
|
||||
if not author_name:
|
||||
author_name = author_handle or "Threads Author"
|
||||
|
||||
# Author profile avatar
|
||||
author_avatar = og_image if og_image else None
|
||||
|
||||
# Progressively scroll to load dynamic replies in the thread
|
||||
# 5. Smart adaptive scrolling for thread continuation replies
|
||||
prev_containers_count = 0
|
||||
for _ in range(max_scrolls):
|
||||
await page.evaluate("window.scrollBy(0, 1200)")
|
||||
await page.wait_for_timeout(1000)
|
||||
containers = await page.query_selector_all("div[data-pressable-container='true']")
|
||||
curr_count = len(containers)
|
||||
# If we already have multiple containers and count didn't change on last scroll, settle early
|
||||
if curr_count > 1 and curr_count == prev_containers_count:
|
||||
break
|
||||
prev_containers_count = curr_count
|
||||
await page.evaluate("window.scrollBy(0, 1400)")
|
||||
await page.wait_for_timeout(350)
|
||||
|
||||
# Query all pressable post containers
|
||||
containers = await page.query_selector_all("div[data-pressable-container='true']")
|
||||
@@ -92,7 +115,6 @@ class ThreadScraper:
|
||||
seen_texts = set()
|
||||
|
||||
for idx, c in enumerate(containers):
|
||||
# 1. Identify post author handle
|
||||
user_links = await c.query_selector_all("a[href*='/@']")
|
||||
post_handle = None
|
||||
post_url = None
|
||||
@@ -113,7 +135,7 @@ class ThreadScraper:
|
||||
if badge:
|
||||
is_author_badge = True
|
||||
|
||||
# Determine if this segment belongs to the original author
|
||||
# Determine if this segment belongs to original author
|
||||
is_author_post = False
|
||||
if idx == 0:
|
||||
is_author_post = True
|
||||
@@ -124,21 +146,18 @@ class ThreadScraper:
|
||||
elif post_handle and author_handle and (post_handle.lower() == author_handle.lower()):
|
||||
is_author_post = True
|
||||
|
||||
# If author_only mode is on and this is not by the author, skip
|
||||
if mode == "author_only" and not is_author_post:
|
||||
continue
|
||||
|
||||
# 2. Extract text with dir='auto'
|
||||
# Extract text lines
|
||||
dir_els = await c.query_selector_all("[dir='auto']")
|
||||
segment_text_parts = []
|
||||
for del_el in dir_els:
|
||||
txt = (await del_el.inner_text()).strip()
|
||||
if not txt:
|
||||
continue
|
||||
# Ignore handle, author badge, bullet dots, stats
|
||||
if txt == post_handle or txt == "Author" or txt == "·" or txt == author_handle:
|
||||
continue
|
||||
# Ignore pure stat numbers (e.g., '16.2K', '5d', '2K', '711')
|
||||
if re.match(r"^(\d+(\.\d+)?[KMkmdh]?|\d+)$", txt):
|
||||
continue
|
||||
if txt not in segment_text_parts:
|
||||
@@ -148,21 +167,19 @@ class ThreadScraper:
|
||||
if not combined_text and not is_author_post:
|
||||
continue
|
||||
|
||||
# Deduplicate identical consecutive segments
|
||||
text_hash = hashlib.md5(combined_text.encode("utf-8")).hexdigest()
|
||||
if text_hash in seen_texts and combined_text:
|
||||
continue
|
||||
seen_texts.add(text_hash)
|
||||
|
||||
# 3. Extract media images inside post (exclude profile pictures)
|
||||
# Extract post images
|
||||
imgs = await c.query_selector_all("img")
|
||||
post_images: List[MediaItem] = []
|
||||
for img in imgs:
|
||||
src = await img.get_attribute("src")
|
||||
alt = await img.get_attribute("alt") or ""
|
||||
if src:
|
||||
# Exclude small avatars or generic SVG icons
|
||||
if "profile_pic" in src or "scontent" in src and "206x206" in src:
|
||||
if "profile_pic" in src or ("scontent" in src and "206x206" in src):
|
||||
continue
|
||||
post_images.append(MediaItem(type="image", url=src, alt_text=alt))
|
||||
|
||||
@@ -179,7 +196,7 @@ class ThreadScraper:
|
||||
)
|
||||
)
|
||||
|
||||
# If no segments could be extracted from containers, fallback to OG description
|
||||
# Fallback to OG description if empty
|
||||
if not raw_segments and og_desc:
|
||||
raw_segments.append(
|
||||
PostSegment(
|
||||
@@ -194,12 +211,11 @@ class ThreadScraper:
|
||||
)
|
||||
)
|
||||
|
||||
# Construct stitched outputs
|
||||
# Build stitched outputs
|
||||
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments)
|
||||
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
|
||||
html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments)
|
||||
|
||||
# Generate unique ID for this ripped thread
|
||||
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
|
||||
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
|
||||
if len(title) > 90:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, HTTPException, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
@@ -14,6 +15,8 @@ scraper = ThreadScraper(headless=True)
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Pre-warm Chromium in background so initial user requests are instant
|
||||
asyncio.create_task(scraper.get_browser())
|
||||
yield
|
||||
await scraper.close()
|
||||
|
||||
|
||||
Reference in new issue
Block a user