"""Abstract data interfaces + placeholder preprocessing pipeline. These declare the SHAPE of the Phase 1 data layer without committing to any ECMWF GRIB specifics. The conceptual steps the user named -- inspect_grib -> extract_valid_times -> standardize_coordinates -> standardize_units -> align_forecast_to_observation are present as an interface. Only the parts that are genuinely metadata-independent are implemented now: * inspect_grib() -- runs ecCodes/CDO CLI and returns RAW output (no key mapping); this is how we will DISCOVER the schema, so it assumes nothing. * align_forecast_to_observation() -- pure date arithmetic via utils.timealign. Everything that needs to know real GRIB keys / dimension names / level conventions raises NotImplementedError until the schema is frozen (docs/canonical_schema.md, Phase 1 step 4/5). """ from __future__ import annotations import os import subprocess from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import Any from utils.timealign import align_forecast_to_observation as _align # -------------------------------------------------------------------------- # Canonical schema placeholder -- FROZEN only after real GRIB inspection. # -------------------------------------------------------------------------- @dataclass class CanonicalField: """Target internal representation of one predictor field. Field names are provisional; the concrete GRIB->canonical mapping is filled after inspection.""" name: str # canonical variable name (e.g. 't850') values: Any = None # (lead, lat, lon) array, units standardized lat: Any = None # ascending degrees_north (convention TBD -> freeze) lon: Any = None # degrees_east (convention TBD -> freeze) init_time: Any = None # datetime64 (reforecast init = dataDate/mars.hdate) model_version_date: Any = None # datetime64 (mars.date; cycle-consistency tracking) valid_time: Any = None # (lead,) datetime64 = init + lead lead: Any = None # (lead,) numeric lead_unit: str = "TBC" # frozen after inspection level_type: str = "TBC" # 'sfc' | 'pl' level: Any = None # hPa for pl, else None member_type: str = "cf" # control forecast units: str = "TBC" provenance: dict = field(default_factory=dict) # -------------------------------------------------------------------------- # Abstract sources # -------------------------------------------------------------------------- class ObservationSource(ABC): """Local-day gridded Tx (VnGAD). Implemented against on-disk NetCDF; does not depend on GRIB.""" @abstractmethod def dates(self): """Return the observation local-day axis (datetime64[D]).""" @abstractmethod def tx(self, start=None, end=None): """Return Tx (time, lat, lon) in degC for the date range.""" class PredictorSource(ABC): """ECMWF S2S control-forecast predictors. CONCRETE loader is written after the canonical schema is frozen.""" @abstractmethod def list_initializations(self): ... @abstractmethod def load_canonical(self, init) -> "list[CanonicalField]": """Return canonical fields for one initialization. NotImplemented until the GRIB->canonical mapping is frozen.""" class Preprocessor(ABC): """GRIB -> canonical schema. All concrete methods deferred to Phase 1.""" @abstractmethod def extract_valid_times(self, raw): ... @abstractmethod def standardize_coordinates(self, raw): ... @abstractmethod def standardize_units(self, raw): ... # -------------------------------------------------------------------------- # Implemented, metadata-independent helpers # -------------------------------------------------------------------------- def inspect_grib(path, keys=None, tool="grib_ls"): """Return RAW ecCodes/CDO output for a GRIB file -- the discovery step. Intentionally does NOT parse into a schema: we run the tool and hand back stdout/stderr so a human/agent can READ the real structure before freezing any mapping. ``tool`` in {grib_ls, grib_dump, cdo_sinfo}. ``keys`` (list) only used with grib_get. """ if not os.path.exists(path): raise FileNotFoundError(path) cmds = { "grib_ls": ["grib_ls", path], "grib_dump": ["grib_dump", "-O", path], "grib_get": ["grib_get", "-p", ",".join(keys or []), path], "cdo_sinfo": ["cdo", "sinfon", path], } if tool not in cmds: raise ValueError(f"unknown tool {tool!r}; use {sorted(cmds)}") try: p = subprocess.run(cmds[tool], capture_output=True, text=True, timeout=120) return {"tool": tool, "cmd": " ".join(cmds[tool]), "returncode": p.returncode, "stdout": p.stdout, "stderr": p.stderr} except FileNotFoundError: return {"tool": tool, "error": f"{tool} not on PATH"} except Exception as e: # pragma: no cover return {"tool": tool, "error": f"{type(e).__name__}: {e}"} def align_forecast_to_observation(init, leads, obs_days, lead_unit="h", tz_hours=7): """Delegate to the generic, tested date-alignment helper (no GRIB assumptions).""" return _align(init, leads, obs_days, lead_unit=lead_unit, tz_hours=tz_hours)