← back to Paul Conrad Archive

scripts/crawl_source.py

21 lines

#!/usr/bin/env python3
"""Run one crawler: python scripts/crawl_source.py loc [--no-cache]"""
import importlib, json, logging, sys, pathlib
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s", stream=sys.stderr)
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1] / "src"))
from conrad import db  # noqa: E402
from conrad.crawlers.base import Http  # noqa: E402


def run(name: str, use_cache: bool = True) -> dict:
    mod = importlib.import_module(f"conrad.crawlers.{name}")
    conn = db.connect()
    db.init_db(conn)
    return mod.CRAWLER(conn=conn, http=Http(use_cache=use_cache)).run()


if __name__ == "__main__":
    args = [a for a in sys.argv[1:] if not a.startswith("--")]
    for n in args:
        print(json.dumps(run(n, use_cache="--no-cache" not in sys.argv)))