"""Score a benchmark submission offline — identical to the leaderboard scoring.

    python evaluate.py my_submission.csv

The submission is a CSV with a ``pair_id,lr`` header: one finite, strictly
positive likelihood ratio for every pair in ``pairs.csv.gz`` (exact coverage).
Requires numpy; ``scoring.py`` (shipped in this kit) must sit next to this file.
"""

import csv
import gzip
import json
import sys
from pathlib import Path

import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parent))
import scoring  # noqa: E402  (the kit's verbatim copy of the server scorer)

HERE = Path(__file__).resolve().parent


def read_pairs():
    with gzip.open(HERE / "pairs.csv.gz", "rt", newline="") as fh:
        return list(csv.DictReader(fh))


def read_submission(path):
    opener = gzip.open if str(path).endswith(".gz") else open
    with opener(path, "rt", newline="") as fh:
        reader = csv.DictReader(fh)
        if "pair_id" not in reader.fieldnames or "lr" not in reader.fieldnames:
            raise SystemExit("submission needs a 'pair_id,lr' header")
        return {row["pair_id"]: float(row["lr"]) for row in reader}


def main():
    if len(sys.argv) != 2:
        raise SystemExit(__doc__)
    pairs = read_pairs()
    lrs_by_id = read_submission(sys.argv[1])

    wanted = {p["pair_id"] for p in pairs}
    missing = wanted - set(lrs_by_id)
    unknown = set(lrs_by_id) - wanted
    if missing or unknown:
        raise SystemExit(
            f"coverage mismatch: {len(missing)} pair(s) missing, "
            f"{len(unknown)} unknown pair id(s) — submit exactly one LR per frozen pair"
        )

    lrs = np.array([lrs_by_id[p["pair_id"]] for p in pairs])
    labels = np.array([int(p["label"]) for p in pairs], dtype=float)
    fold_members = {}
    for i, p in enumerate(pairs):
        for f in p["folds"].split(";"):
            if f:
                fold_members.setdefault(int(f), []).append(i)
    folds = [(k, np.array(v)) for k, v in sorted(fold_members.items())]

    metrics = scoring.score_submission(lrs, labels, folds)
    provenance = json.loads((HERE / "provenance.json").read_text())
    print(json.dumps({"split": provenance["name"], "split_hash": provenance["split_hash"],
                      **{k: v for k, v in metrics.items() if k != "folds"}}, indent=1))


if __name__ == "__main__":
    main()
