"""Stage 18 — build the machine-readable MEMBER REGISTRY (v1). Stage 2 : a MEMBER is one (family, configuration) pair. The Stage-1 pool is 9 families; the cascade-cutoff columns are configurations, so registry v1 = 23 members: 7 sklearn families {logistic, hgb, rf, mlp, ridge_sev, hgb_sev, mlp_sev} x 3 cascade cutoffs {1.0, 0.90, 0.99} + cnn_c1 + ann_torch_c1 (torch families, single-level only). Each member emits ONE binary EHD vote per grid cell per block (prob >= its leakage-safe per-region x block threshold) — the uniform "distributional -> single vote" rule stated once here. This stage does NOT retrain anything. It records provenance (family, full hyperparameters, config hash, the stored-prediction file each member's votes come from, status) and attaches the CV scores that justify the status, reused from the already-computed pooled out-of-sample member scores (results/tables/v6_member_scores_by_block.csv). Registry versions: v1 = this pool; v2 = after pruning (stage 19); v3 = after expansion (stage 20). Outputs: outputs/member_registry/registry_v1.json full records (machine-readable) outputs/member_registry/registry_v1.csv flat member x block score table """ from __future__ import annotations import os, sys, json, hashlib, logging import numpy as np import pandas as pd sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) from ehd.config import load_config log = logging.getLogger("stage18") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") # ---- family default hyperparameters, as trained in Stage 1 (src/ehd/calibration.py, gridmodels.py) FAMILY_HP = { "logistic": {"kind": "classifier", "estimator": "sklearn.LogisticRegression", "hp": {"C": 1.0, "max_iter": 1000, "standardized": True}}, "hgb": {"kind": "classifier", "estimator": "sklearn.HistGradientBoostingClassifier", "hp": {"max_depth": None, "learning_rate": 0.06, "max_iter": 300, "l2": 0.0}}, "rf": {"kind": "classifier", "estimator": "sklearn.RandomForestClassifier", "hp": {"n_estimators": 300, "max_depth": None, "min_samples_leaf": 100}}, "mlp": {"kind": "classifier", "estimator": "sklearn.MLPClassifier", "hp": {"hidden_layer_sizes": [64, 32], "alpha": 1e-3, "max_iter": 80, "standardized": True}}, "ridge_sev": {"kind": "sev_regressor", "estimator": "sklearn.Ridge->isotonic", "hp": {"alpha": 1.0, "standardized": True, "predictand": "sev=Tx-min(Tx90p,35)"}}, "hgb_sev": {"kind": "sev_regressor", "estimator": "sklearn.HistGradientBoostingRegressor->isotonic", "hp": {"learning_rate": 0.06, "max_iter": 300, "min_samples_leaf": 200, "l2": 0.0, "predictand": "sev=Tx-min(Tx90p,35)"}}, "mlp_sev": {"kind": "sev_regressor", "estimator": "sklearn.MLPRegressor->isotonic", "hp": {"hidden_layer_sizes": [64, 32], "alpha": 1e-3, "max_iter": 80, "standardized": True, "predictand": "sev=Tx-min(Tx90p,35)"}}, "cnn": {"kind": "torch_cnn", "estimator": "TorchCNN (fully-convolutional, CPU)", "hp": {"channels": 32, "lr": 1e-3, "max_epoch": 30, "batch": 64, "arch": "Conv3x3 x3 + 1x1 head, block maps as channels"}}, "ann_torch": {"kind": "torch_ann", "estimator": "TorchANN (deep MLP, CPU)", "hp": {"hidden": [128, 64, 32], "dropout": 0.2, "lr": 1e-3, "max_epoch": 60, "batch": 8192, "patience": 6, "pos_weight": "class-imbalance"}}, } SKLEARN_FAMILIES = ["logistic", "hgb", "rf", "mlp", "ridge_sev", "hgb_sev", "mlp_sev"] CUTOFFS = ["1", "0.9", "0.99"] # cascade LV1 screen cutoffs (1 = no screen) TORCH_MEMBERS = ["cnn", "ann_torch"] # single-level only (cutoff 1) VOTE_RULE = ("Each member's calibrated P(yEHDE) is thresholded at its leakage-safe per-region x " "block decision threshold (Peirce-optimal, fit on the fold's training years) to give " "one binary yEHDE vote per grid cell per block. Distributional/severity members are " "reduced to P(yEHDE) by an isotonic map fit on TRAIN before this rule.") def cfg_hash(family, cutoff): payload = json.dumps({"family": family, "cutoff": cutoff, "hp": FAMILY_HP[family]}, sort_keys=True).encode() return hashlib.sha1(payload).hexdigest()[:12] def provenance(family, cutoff): if family in TORCH_MEMBERS: return {"loyo_pred_glob": "outputs/loyo_pred_v2_cnn/pred_v2cnn_.npz", "prob_column": f"p_{family}_c{cutoff}", "trained_by": "scripts/12b_run_cnn_ann.py (SLURM job 1401269)"} return {"loyo_pred_glob": "outputs/loyo_pred_v2/pred_v2_.npz", "prob_column": f"p_{family}_c{cutoff}", "trained_by": "scripts/12_run_loyo_v2.py (submit_loyo_v2.slurm)"} def main(): cfg = load_config() root = cfg["paths"]["root"] outdir = os.path.join(cfg["paths"]["outputs"], "member_registry") os.makedirs(outdir, exist_ok=True) # per-member per-block scores already computed out-of-sample (stage 17) sc = pd.read_csv(os.path.join(cfg["paths"]["results"], "tables", "v6_member_scores_by_block.csv")) blocks = list(cfg["lead_blocks"].keys()) w26 = [b for b in blocks if b != "W1"] # operational value lives at W2-W6 members = [] for fam in SKLEARN_FAMILIES: for cut in CUTOFFS: members.append((fam, cut)) for fam in TORCH_MEMBERS: members.append((fam, "1")) records = [] flat_rows = [] for fam, cut in members: mid = f"{fam}_c{cut}" msc = sc[sc["member"] == mid] by_block = {} for _, r in msc.iterrows(): by_block[r["block"]] = {k: (None if pd.isna(r[k]) else float(r[k])) for k in ["POD", "POFD", "FAR", "CSI", "Peirce", "SEDI", "freq_bias", "base_rate"]} # summary used by the pruning criterion: mean Peirce (PSS) and mean FAR over W2-W6 pss_w26 = np.nanmean([by_block[b]["Peirce"] for b in w26 if b in by_block]) if by_block else np.nan far_w26 = np.nanmean([by_block[b]["FAR"] for b in w26 if b in by_block]) if by_block else np.nan rec = dict( member_id=mid, family=fam, configuration={"cascade_cutoff": float(cut)}, estimator=FAMILY_HP[fam]["estimator"], kind=FAMILY_HP[fam]["kind"], hyperparameters=FAMILY_HP[fam]["hp"], config_hash=cfg_hash(fam, cut), provenance=provenance(fam, cut), status="active", status_reason="Stage-1 pool member; retained pending stage-19 pruning.", registry_version="v1", cv_summary_W2_W6={"mean_Peirce": None if np.isnan(pss_w26) else round(pss_w26, 4), "mean_FAR": None if np.isnan(far_w26) else round(far_w26, 4)}, cv_scores_by_block=by_block) records.append(rec) for b in blocks: row = dict(member_id=mid, family=fam, cascade_cutoff=float(cut), block=b, config_hash=rec["config_hash"], status="active") row.update(by_block.get(b, {})) flat_rows.append(row) reg = dict( registry_version="v1", description="Stage-1 member pool (perturbed-parameter multi-model ensemble).", member_definition="A member is one (family, configuration) pair; configs are cascade cutoffs.", vote_rule=VOTE_RULE, n_members=len(records), families=sorted({f for f, _ in members}), leakage="All member votes are out-of-sample LOYO (2003-2022); thresholds fit on train years only.", created=pd.Timestamp.utcnow().isoformat(), members=records) with open(os.path.join(outdir, "registry_v1.json"), "w") as f: json.dump(reg, f, indent=2) pd.DataFrame(flat_rows).to_csv(os.path.join(outdir, "registry_v1.csv"), index=False) log.info("registry v1: %d members, %d families -> %s", len(records), len(reg["families"]), outdir) # short console summary of the W2-W6 pruning-relevant metric summ = (pd.DataFrame([(r["member_id"], r["cv_summary_W2_W6"]["mean_Peirce"], r["cv_summary_W2_W6"]["mean_FAR"]) for r in records], columns=["member_id", "mean_Peirce_W2_W6", "mean_FAR_W2_W6"]) .sort_values("mean_Peirce_W2_W6", ascending=False)) print(summ.to_string(index=False)) if __name__ == "__main__": main()