Strip all graphics and images from final results (Markdown, HTML, text, and UI)
This commit is contained in:
1 parent
7609d70dd1
commit
8a91a05c2a
5 files changed
+48
-62
No files matched your search
@@ -39,7 +39,8 @@ class RippedThread(BaseModel):
|
|||||||
class RipRequest(BaseModel):
|
class RipRequest(BaseModel):
|
||||||
url: str
|
url: str
|
||||||
mode: str = "author_only" # 'author_only' or 'all_replies'
|
mode: str = "author_only" # 'author_only' or 'all_replies'
|
||||||
max_scrolls: int = 6
|
include_images: bool = False
|
||||||
|
max_scrolls: int = 4
|
||||||
|
|
||||||
class HistorySummary(BaseModel):
|
class HistorySummary(BaseModel):
|
||||||
id: str
|
id: str
|
||||||
|
|||||||
+15
-14
@@ -42,7 +42,7 @@ class ThreadScraper:
|
|||||||
await self._playwright.stop()
|
await self._playwright.stop()
|
||||||
self._playwright = None
|
self._playwright = None
|
||||||
|
|
||||||
async def rip(self, url: str, mode: str = "author_only", max_scrolls: int = 4) -> RippedThread:
|
async def rip(self, url: str, mode: str = "author_only", include_images: bool = False, max_scrolls: int = 4) -> RippedThread:
|
||||||
"""Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
|
"""Scrapes and rips a Threads post and chained continuation posts with optimized speed."""
|
||||||
browser = await self.get_browser()
|
browser = await self.get_browser()
|
||||||
context: BrowserContext = await browser.new_context(
|
context: BrowserContext = await browser.new_context(
|
||||||
@@ -58,7 +58,7 @@ class ThreadScraper:
|
|||||||
"**/*",
|
"**/*",
|
||||||
lambda route: route.abort()
|
lambda route: route.abort()
|
||||||
if route.request.resource_type in ["image", "media", "font"]
|
if route.request.resource_type in ["image", "media", "font"]
|
||||||
else route.continue_()
|
else route.continue_(),
|
||||||
)
|
)
|
||||||
|
|
||||||
# 2. Fast navigation using domcontentloaded instead of networkidle
|
# 2. Fast navigation using domcontentloaded instead of networkidle
|
||||||
@@ -172,16 +172,17 @@ class ThreadScraper:
|
|||||||
continue
|
continue
|
||||||
seen_texts.add(text_hash)
|
seen_texts.add(text_hash)
|
||||||
|
|
||||||
# Extract post images
|
# Extract post images (only real content photos, never profile avatars)
|
||||||
imgs = await c.query_selector_all("img")
|
|
||||||
post_images: List[MediaItem] = []
|
post_images: List[MediaItem] = []
|
||||||
for img in imgs:
|
if include_images:
|
||||||
src = await img.get_attribute("src")
|
imgs = await c.query_selector_all("img")
|
||||||
alt = await img.get_attribute("alt") or ""
|
for img in imgs:
|
||||||
if src:
|
src = await img.get_attribute("src")
|
||||||
if "profile_pic" in src or ("scontent" in src and "206x206" in src):
|
alt = await img.get_attribute("alt") or ""
|
||||||
continue
|
if src:
|
||||||
post_images.append(MediaItem(type="image", url=src, alt_text=alt))
|
if "profile_pic" in src or "profile picture" in alt.lower() or ("scontent" in src and "206x206" in src):
|
||||||
|
continue
|
||||||
|
post_images.append(MediaItem(type="image", url=src, alt_text=alt))
|
||||||
|
|
||||||
raw_segments.append(
|
raw_segments.append(
|
||||||
PostSegment(
|
PostSegment(
|
||||||
@@ -211,10 +212,10 @@ class ThreadScraper:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Build stitched outputs
|
# Build stitched outputs without graphics by default
|
||||||
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments)
|
markdown_out = build_markdown(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
|
||||||
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
|
plain_text_out = build_plain_text(author_name, author_handle, canonical_url, raw_segments)
|
||||||
html_out = build_standalone_html(author_name, author_handle, author_avatar, canonical_url, raw_segments)
|
html_out = build_standalone_html(author_name, author_handle, canonical_url, raw_segments, include_images=include_images)
|
||||||
|
|
||||||
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
|
thread_id = hashlib.md5(canonical_url.split("?")[0].encode("utf-8")).hexdigest()[:12]
|
||||||
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
|
title = raw_segments[0].text.split("\n")[0] if raw_segments and raw_segments[0].text else f"Thread by @{author_handle}"
|
||||||
|
|||||||
@@ -42,7 +42,12 @@ async def rip_endpoint(req: RipRequest):
|
|||||||
raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.")
|
raise HTTPException(status_code=400, detail="Invalid Threads URL. Please provide a valid threads.net or threads.com link.")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
ripped = await scraper.rip(url=req.url, mode=req.mode, max_scrolls=req.max_scrolls)
|
ripped = await scraper.rip(
|
||||||
|
url=req.url,
|
||||||
|
mode=req.mode,
|
||||||
|
include_images=req.include_images,
|
||||||
|
max_scrolls=req.max_scrolls
|
||||||
|
)
|
||||||
storage.save(ripped)
|
storage.save(ripped)
|
||||||
return ripped
|
return ripped
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
const response = await fetch("/api/rip", {
|
const response = await fetch("/api/rip", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: { "Content-Type": "application/json" },
|
||||||
body: JSON.stringify({ url, mode, max_scrolls: 4 })
|
body: JSON.stringify({ url, mode, include_images: false, max_scrolls: 4 })
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
@@ -161,16 +161,11 @@ document.addEventListener("DOMContentLoaded", () => {
|
|||||||
function renderThread(thread) {
|
function renderThread(thread) {
|
||||||
resultSection.classList.remove("hidden");
|
resultSection.classList.remove("hidden");
|
||||||
|
|
||||||
// Header info
|
// Header info (clean text header, no graphic avatar)
|
||||||
resAuthorName.textContent = thread.author_name;
|
resAuthorName.textContent = thread.author_name;
|
||||||
resAuthorLink.textContent = `@${thread.author_handle}`;
|
resAuthorLink.textContent = `@${thread.author_handle}`;
|
||||||
resAuthorLink.href = thread.canonical_url;
|
resAuthorLink.href = thread.canonical_url;
|
||||||
if (thread.author_avatar) {
|
resAvatar.classList.add("hidden");
|
||||||
resAvatar.src = thread.author_avatar;
|
|
||||||
resAvatar.classList.remove("hidden");
|
|
||||||
} else {
|
|
||||||
resAvatar.classList.add("hidden");
|
|
||||||
}
|
|
||||||
|
|
||||||
resSegmentCount.textContent = thread.segment_count;
|
resSegmentCount.textContent = thread.segment_count;
|
||||||
resWordCount.textContent = thread.total_words;
|
resWordCount.textContent = thread.total_words;
|
||||||
|
|||||||
+22
-38
@@ -5,7 +5,6 @@ from .models import PostSegment
|
|||||||
def clean_segment_text(text: str) -> str:
|
def clean_segment_text(text: str) -> str:
|
||||||
"""Clean up whitespace and trailing artifacts from text."""
|
"""Clean up whitespace and trailing artifacts from text."""
|
||||||
lines = [l.strip() for l in text.split("\n")]
|
lines = [l.strip() for l in text.split("\n")]
|
||||||
# Remove empty lines while preserving deliberate paragraph breaks
|
|
||||||
cleaned_lines = []
|
cleaned_lines = []
|
||||||
prev_blank = False
|
prev_blank = False
|
||||||
for line in lines:
|
for line in lines:
|
||||||
@@ -17,8 +16,8 @@ def clean_segment_text(text: str) -> str:
|
|||||||
prev_blank = True
|
prev_blank = True
|
||||||
return "\n".join(cleaned_lines).strip()
|
return "\n".join(cleaned_lines).strip()
|
||||||
|
|
||||||
def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment]) -> str:
|
def build_markdown(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
|
||||||
"""Stitches segments into a single cohesive Markdown document."""
|
"""Stitches segments into a single cohesive Markdown document without graphics."""
|
||||||
lines = []
|
lines = []
|
||||||
title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post"
|
title = segments[0].text.split("\n")[0] if segments and segments[0].text else "Threads Post"
|
||||||
if len(title) > 80:
|
if len(title) > 80:
|
||||||
@@ -35,18 +34,17 @@ def build_markdown(author_name: str, author_handle: str, canonical_url: str, seg
|
|||||||
|
|
||||||
for idx, seg in enumerate(segments):
|
for idx, seg in enumerate(segments):
|
||||||
cleaned_text = clean_segment_text(seg.text)
|
cleaned_text = clean_segment_text(seg.text)
|
||||||
if not cleaned_text and not seg.images:
|
if not cleaned_text:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if cleaned_text:
|
lines.append(cleaned_text)
|
||||||
lines.append(cleaned_text)
|
lines.append("")
|
||||||
lines.append("")
|
|
||||||
|
|
||||||
# Include media images if any
|
if include_images:
|
||||||
for img in seg.images:
|
for img in seg.images:
|
||||||
alt = img.alt_text or f"Thread Image {idx + 1}"
|
alt = img.alt_text or f"Image {idx + 1}"
|
||||||
lines.append(f"")
|
lines.append(f"")
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
lines.append("---")
|
lines.append("---")
|
||||||
lines.append(f"*Ripped with Thread Ripper from [Threads]({canonical_url})*")
|
lines.append(f"*Ripped with Thread Ripper from [Threads]({canonical_url})*")
|
||||||
@@ -69,8 +67,8 @@ def build_plain_text(author_name: str, author_handle: str, canonical_url: str, s
|
|||||||
output.append("=" * 60)
|
output.append("=" * 60)
|
||||||
return "\n".join(output).strip()
|
return "\n".join(output).strip()
|
||||||
|
|
||||||
def build_standalone_html(author_name: str, author_handle: str, author_avatar: str, canonical_url: str, segments: List[PostSegment]) -> str:
|
def build_standalone_html(author_name: str, author_handle: str, canonical_url: str, segments: List[PostSegment], include_images: bool = False) -> str:
|
||||||
"""Generates an elegant, responsive standalone HTML file."""
|
"""Generates an elegant, text-focused standalone HTML document without clutter."""
|
||||||
title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}"
|
title = segments[0].text.split("\n")[0] if segments and segments[0].text else f"Thread by @{author_handle}"
|
||||||
if len(title) > 80:
|
if len(title) > 80:
|
||||||
title = title[:77] + "..."
|
title = title[:77] + "..."
|
||||||
@@ -78,10 +76,11 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
content_sections = []
|
content_sections = []
|
||||||
for idx, seg in enumerate(segments):
|
for idx, seg in enumerate(segments):
|
||||||
cleaned_text = clean_segment_text(seg.text)
|
cleaned_text = clean_segment_text(seg.text)
|
||||||
|
if not cleaned_text:
|
||||||
|
continue
|
||||||
paragraphs = cleaned_text.split("\n")
|
paragraphs = cleaned_text.split("\n")
|
||||||
section_html = "<div class='thread-segment'>"
|
section_html = "<div class='thread-segment'>"
|
||||||
|
|
||||||
# Render bullet points or paragraphs cleanly
|
|
||||||
in_list = False
|
in_list = False
|
||||||
for p in paragraphs:
|
for p in paragraphs:
|
||||||
p = p.strip()
|
p = p.strip()
|
||||||
@@ -100,15 +99,14 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
if in_list:
|
if in_list:
|
||||||
section_html += "</ul>"
|
section_html += "</ul>"
|
||||||
|
|
||||||
# Attached images
|
if include_images:
|
||||||
for img in seg.images:
|
for img in seg.images:
|
||||||
section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>"
|
section_html += f"<div class='segment-image'><img src='{img.url}' alt='{img.alt_text or ''}' loading='lazy' /></div>"
|
||||||
|
|
||||||
section_html += "</div>"
|
section_html += "</div>"
|
||||||
content_sections.append(section_html)
|
content_sections.append(section_html)
|
||||||
|
|
||||||
body_content = "\n".join(content_sections)
|
body_content = "\n".join(content_sections)
|
||||||
avatar_tag = f"<img class='avatar' src='{author_avatar}' alt='{author_handle}' />" if author_avatar else ""
|
|
||||||
|
|
||||||
html = f"""<!DOCTYPE html>
|
html = f"""<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en">
|
||||||
@@ -139,7 +137,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||||
background: var(--bg);
|
background: var(--bg);
|
||||||
color: var(--text);
|
color: var(--text);
|
||||||
line-height: 1.65;
|
line-height: 1.7;
|
||||||
margin: 0;
|
margin: 0;
|
||||||
padding: 32px 16px;
|
padding: 32px 16px;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -155,22 +153,14 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
box-shadow: 0 8px 30px rgba(0,0,0,0.12);
|
box-shadow: 0 8px 30px rgba(0,0,0,0.12);
|
||||||
}}
|
}}
|
||||||
.header {{
|
.header {{
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 16px;
|
|
||||||
margin-bottom: 28px;
|
margin-bottom: 28px;
|
||||||
padding-bottom: 20px;
|
padding-bottom: 20px;
|
||||||
border-bottom: 1px solid var(--border);
|
border-bottom: 1px solid var(--border);
|
||||||
}}
|
}}
|
||||||
.avatar {{
|
|
||||||
width: 54px;
|
|
||||||
height: 54px;
|
|
||||||
border-radius: 50%;
|
|
||||||
object-fit: cover;
|
|
||||||
}}
|
|
||||||
.author-info h2 {{
|
.author-info h2 {{
|
||||||
margin: 0;
|
margin: 0 0 6px 0;
|
||||||
font-size: 1.25rem;
|
font-size: 1.35rem;
|
||||||
|
font-weight: 700;
|
||||||
}}
|
}}
|
||||||
.author-info a {{
|
.author-info a {{
|
||||||
color: var(--accent);
|
color: var(--accent);
|
||||||
@@ -178,7 +168,7 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
}}
|
}}
|
||||||
.thread-segment {{
|
.thread-segment {{
|
||||||
margin-bottom: 24px;
|
margin-bottom: 20px;
|
||||||
font-size: 1.1rem;
|
font-size: 1.1rem;
|
||||||
}}
|
}}
|
||||||
.thread-segment p {{
|
.thread-segment p {{
|
||||||
@@ -191,11 +181,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
.thread-segment li {{
|
.thread-segment li {{
|
||||||
margin-bottom: 8px;
|
margin-bottom: 8px;
|
||||||
}}
|
}}
|
||||||
.segment-image img {{
|
|
||||||
max-width: 100%;
|
|
||||||
border-radius: 12px;
|
|
||||||
margin-top: 12px;
|
|
||||||
}}
|
|
||||||
.footer {{
|
.footer {{
|
||||||
margin-top: 36px;
|
margin-top: 36px;
|
||||||
padding-top: 20px;
|
padding-top: 20px;
|
||||||
@@ -214,7 +199,6 @@ def build_standalone_html(author_name: str, author_handle: str, author_avatar: s
|
|||||||
<body>
|
<body>
|
||||||
<article class="article">
|
<article class="article">
|
||||||
<div class="header">
|
<div class="header">
|
||||||
{avatar_tag}
|
|
||||||
<div class="author-info">
|
<div class="author-info">
|
||||||
<h2>{author_name}</h2>
|
<h2>{author_name}</h2>
|
||||||
<a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a>
|
<a href="{canonical_url}" target="_blank">@{author_handle} on Threads</a>
|
||||||
|
|||||||
Reference in new issue
Block a user