41 lines
1.4 KiB
Python
41 lines
1.4 KiB
Python
"""CLI: python -m web_fetcher fetch SPECS.json [--cache DIR] [--out FILE]
|
|
|
|
SPECS.json is a JSON list of spec objects (or {"specs": [...]}). Output is
|
|
JSON Lines, one Observation per line.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
|
|
from web_fetcher.registry import run_specs
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(prog="web_fetcher")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
f = sub.add_parser("fetch", help="run fetch specs")
|
|
f.add_argument("specs")
|
|
f.add_argument("--cache", default=None, help="content-addressed cache directory")
|
|
f.add_argument("--out", default="-", help="output JSONL path (default stdout)")
|
|
f.add_argument("--only", action="append", default=[], help="run only these spec ids")
|
|
args = ap.parse_args(argv)
|
|
with open(args.specs, encoding="utf-8") as fh:
|
|
data = json.load(fh)
|
|
specs = data["specs"] if isinstance(data, dict) else data
|
|
if args.only:
|
|
specs = [s for s in specs if s.get("id") in set(args.only)]
|
|
obs = run_specs(specs, cache_dir=args.cache)
|
|
lines = "\n".join(o.to_json() for o in obs) + "\n"
|
|
if args.out == "-":
|
|
sys.stdout.write(lines)
|
|
else:
|
|
with open(args.out, "w", encoding="utf-8") as fh:
|
|
fh.write(lines)
|
|
return 0 if all(o.status != "error" for o in obs) else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|