# /// script # requires-python = ">=3.10" # dependencies = ["firecrawl-py==4.57.0"] # /// """Crawl a site with your own Firecrawl and save every page as Markdown. uv run crawl.py URL [LIMIT] Pages go to ~/crawls//.md, with index.jsonl listing each page's URL, title and status. FIRECRAWL_API_URL is the API (default http://localhost:3002); FIRECRAWL_API_KEY, if set, is sent as a bearer token, which is where a VM token goes when the API is on another VM. """ import json import os import re import sys import time from pathlib import Path from urllib.parse import urlparse from firecrawl import Firecrawl url = sys.argv[1] limit = int(sys.argv[2]) if len(sys.argv) > 2 else 50 fc = Firecrawl(api_url=os.environ.get("FIRECRAWL_API_URL", "http://localhost:3002")) start = time.time() job = fc.crawl(url, limit=limit, only_main_content=True, formats=["markdown"], poll_interval=2) out = Path.home() / "crawls" / urlparse(url).netloc out.mkdir(parents=True, exist_ok=True) with open(out / "index.jsonl", "w") as index: for doc in job.data: page = doc.metadata.url or doc.metadata.source_url path = urlparse(page).path.strip("/") or "index" name = re.sub(r"[^A-Za-z0-9._-]+", "_", path.removesuffix(".html")) + ".md" (out / name).write_text(doc.markdown or "") index.write(json.dumps({"url": page, "title": " ".join((doc.metadata.title or "").split()), "status": doc.metadata.status_code, "file": name}) + "\n") print(f"{job.status}: {len(job.data)} pages saved to {out} in {time.time() - start:.0f} s")