`scripts/crawl_all.py`

49 lines
#!/usr/bin/env python3
"""Run every crawler (seed first), MAX_CONCURRENCY sources in parallel, then dedupe + exports + report.

  python scripts/crawl_all.py            # restartable: cached responses + checkpoints are reused
  python scripts/crawl_all.py --fresh    # rebuild data/conrad.db from scratch (HTTP cache is still reused)
  python scripts/crawl_all.py --no-cache # re-fetch everything live
"""
import importlib, json, logging, subprocess, sys, pathlib
from concurrent.futures import ThreadPoolExecutor

ROOT = pathlib.Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(threadName)s %(message)s", stream=sys.stderr)
from conrad import config, db, dedupe, exports  # noqa: E402
from conrad.crawlers import ORDER  # noqa: E402
from conrad.crawlers.base import Http  # noqa: E402


def run_one(name: str, use_cache: bool) -> dict:
    mod = importlib.import_module(f"conrad.crawlers.{name}")
    return mod.CRAWLER(conn=db.connect(), http=Http(use_cache=use_cache)).run()


def main() -> None:
    use_cache = "--no-cache" not in sys.argv
    if "--fresh" in sys.argv and config.DB_PATH.exists():
        for suffix in ("", "-wal", "-shm"):
            p = pathlib.Path(str(config.DB_PATH) + suffix)
            if p.exists():
                p.unlink()
    conn = db.connect()
    db.init_db(conn)
    results = [run_one("seed_import", use_cache)]
    # huntington first (live verification of the largest source), then the rest in parallel
    results.append(run_one("huntington", use_cache))
    rest = [n for n in ORDER if n not in ("seed_import", "huntington", "latimes", "denver_post")]
    with ThreadPoolExecutor(max_workers=config.MAX_CONCURRENCY) as pool:
        results += list(pool.map(lambda n: run_one(n, use_cache), rest))
    results += [run_one(n, use_cache) for n in ("latimes", "denver_post")]  # coverage computed after other sources
    for r in results:
        print(json.dumps(r))
    print(json.dumps({"dedupe": dedupe.run(conn)}))
    print(json.dumps({"exports": exports.export_all(conn)}))
    subprocess.run([sys.executable, str(ROOT / "scripts/report.py")], check=True)


if __name__ == "__main__":
    main()