Sites

from webgraph import stream_site, SiteConfig

for event in stream_site(
    root,
    config=SiteConfig(),   # SiteConfig | None
    schema=None,           # dict | None -- a JSON Schema to extract facts against
    should_stop=None,      # Callable[[], bool] | None -- polled as each page lands
    seeds=None,            # Iterable[str] | None -- addresses queued at depth 1 beside the sitemap's
    builder=None,          # GraphBuilder | None -- filled in as pages arrive
):
    ...

from webgraph import SiteConfig, stream_site

pages = {}
for event in stream_site("https://docs.python.org/3/tutorial/", config=SiteConfig(max_pages=3, max_depth=1, within_path=True)):
    match event["type"]:
        case "analysis":
            print("built with", [t["name"] for t in event["technologies"]], "| strategy", event["strategy"])
        case "discovery":
            print("robots:", event["robots"]["found"], "| sitemap urls:", event["sitemaps"]["total_urls"], "| seeded:", event["seeds"])
        case "page":
            if event["ok"]:
                pages[event["url"]] = event["markdown"]          # the whole page; content_markdown is the opt-in reduction
            else:
                print("refused", event["url"], event["error"])
        case "done":
            print(event["pages_ok"], "of", event["pages_total"], "pages;", "stopped by", event["stopped_by"], "in", event["duration_seconds"], "s")

from pathlib import Path
from webgraph import SiteConfig, stream_site

out = Path("crawl"); out.mkdir(exist_ok=True)
config = SiteConfig(max_pages=200, within_path=True, exclude_paths=r"/print/|\?page=")
for event in stream_site("https://docs.python.org/3/tutorial/", config=config):
    if event["type"] == "page" and event["ok"]:
        name = event["url"].split("://", 1)[1].strip("/").replace("/", "__") or "index"
        (out / f"{name}.md").write_text(event["markdown"])

import threading
stop = threading.Event()

def crawl():
    for event in stream_site(root, should_stop=stop.is_set):
        ...

threading.Thread(target=crawl).start()
...
stop.set()      # the pages in flight finish; `done` arrives with stopped=True