70 lines
2.7 KiB
Python
70 lines
2.7 KiB
Python
import argparse
|
|
import asyncio
|
|
import sys
|
|
from pathlib import Path
|
|
from .scraper import ThreadScraper
|
|
from .storage import ThreadStorage
|
|
|
|
def parse_args():
|
|
parser = argparse.ArgumentParser(description="Thread Ripper - Unroll and stitch Threads posts into a single post.")
|
|
parser.add_argument("url", help="Threads URL (e.g., https://www.threads.com/share/BASy90mpUT/ or https://www.threads.net/@user/post/...)")
|
|
parser.add_argument("-o", "--output", help="Output file path (e.g. output.md, output.txt, output.html)")
|
|
parser.add_argument("-f", "--format", choices=["markdown", "txt", "html", "json"], default="markdown", help="Output format (default: markdown)")
|
|
parser.add_argument("--mode", choices=["author_only", "all_replies"], default="author_only", help="Extraction mode (default: author_only)")
|
|
parser.add_argument("--scrolls", type=int, default=5, help="Number of lazy-load scrolls (default: 5)")
|
|
return parser.parse_args()
|
|
|
|
async def main_async():
|
|
args = parse_args()
|
|
print(f"[*] Thread Ripper v1.0.0")
|
|
print(f"[*] Target URL: {args.url}")
|
|
print(f"[*] Mode: {args.mode}")
|
|
|
|
scraper = ThreadScraper(headless=True)
|
|
try:
|
|
print("[*] Fetching and parsing thread...")
|
|
result = await scraper.rip(url=args.url, mode=args.mode, max_scrolls=args.scrolls)
|
|
|
|
# Save to persistent storage as well
|
|
storage = ThreadStorage("data")
|
|
storage.save(result)
|
|
|
|
print(f"\n[+] Successfully ripped thread!")
|
|
print(f" Author: {result.author_name} (@{result.author_handle})")
|
|
print(f" Segments stitched: {result.segment_count}")
|
|
print(f" Total words: {result.total_words}")
|
|
print(f" Thread ID: {result.id}")
|
|
|
|
content = ""
|
|
if args.format == "markdown":
|
|
content = result.stitched_markdown
|
|
elif args.format == "txt":
|
|
content = result.stitched_text
|
|
elif args.format == "html":
|
|
content = result.stitched_html
|
|
elif args.format == "json":
|
|
content = result.model_dump_json(indent=2)
|
|
|
|
if args.output:
|
|
out_path = Path(args.output)
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
print(f"[+] Output saved to: {out_path.resolve()}")
|
|
else:
|
|
print("\n" + "=" * 40 + " RIPPED POST PREVIEW " + "=" * 40)
|
|
print(content[:1500] + ("\n... [truncated]" if len(content) > 1500 else ""))
|
|
print("=" * 102)
|
|
|
|
except Exception as e:
|
|
print(f"[-] Error: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
finally:
|
|
await scraper.close()
|
|
|
|
def main():
|
|
asyncio.run(main_async())
|
|
|
|
if __name__ == "__main__":
|
|
main()
|