Climate-Mood-Analysis/scripts/rebuild_nsrdb_representative_point_ghi_summary.py

#!/usr/bin/env python3
"""
Rebuild county representative-point GHI summaries from cached NSRDB raw CSV responses.

This does not call the NSRDB API. It reads data/nrel/representative_point_csv/*.csv, computes:

avgSolarGhiKwhM2Day = sum(hourly GHI) / 1000 / 365

and writes a county-keyed CSV compatible with build_county_climate_data.py.
"""

from __future__ import annotations

import argparse
import csv
from pathlib import Path


DEFAULT_POINTS_CSV = Path("data/nrel/county_representative_points.csv")
DEFAULT_REPRESENTATIVE_POINT_CSV_DIR = Path("data/nrel/representative_point_csv")
DEFAULT_OUTPUT_CSV = Path("data/nrel/county_representative_point_ghi_summary.csv")


def read_county_points(points_csv: Path) -> list[dict[str, str]]:
    """Read county representative points keyed by county FIPS."""
    with points_csv.open(newline="", encoding="utf-8") as handle:
        return list(csv.DictReader(handle))


def extract_ghi_values(csv_path: Path) -> list[float]:
    """Extract numeric hourly GHI values from a cached NSRDB CSV response."""
    with csv_path.open(newline="", encoding="utf-8-sig") as handle:
        reader = csv.reader(handle)
        ghi_index: int | None = None
        values: list[float] = []

        for row in reader:
            if ghi_index is None:
                if {"Year", "Month", "Day", "Hour", "Minute", "GHI"}.issubset(set(row)):
                    ghi_index = row.index("GHI")
                continue

            if len(row) <= ghi_index or not row[ghi_index].strip():
                continue
            values.append(float(row[ghi_index]))

    if ghi_index is None:
        raise ValueError(f"Could not find NSRDB data header with a GHI column in {csv_path}.")
    if not values:
        raise ValueError(f"No GHI values were found in {csv_path}.")
    return values


def select_raw_csv(representative_point_csv_dir: Path, county_fips: str) -> Path | None:
    """Return the cached raw CSV for a county, preferring polar data when both exist."""
    matches = sorted(representative_point_csv_dir.glob(f"{county_fips}_*_ghi.csv"))
    if not matches:
        return None

    polar_matches = [path for path in matches if "_polar-tmy_" in path.name]
    return polar_matches[0] if polar_matches else matches[0]


def source_label(raw_csv: Path) -> str:
    """Return a readable source label from the cached filename."""
    if "_polar-tmy_" in raw_csv.name:
        return "NSRDB Polar TMY PSM v4 representative point"
    if "_goes-tmy_" in raw_csv.name:
        return "NSRDB GOES TMY PSM v4 representative point"
    return "NSRDB TMY representative point"


def summarize_county(point: dict[str, str], raw_csv: Path) -> dict[str, str]:
    """Summarize one county raw CSV into the app solar metric."""
    ghi_values = extract_ghi_values(raw_csv)
    avg_daily_ghi = sum(ghi_values) / 1000 / 365

    return {
        "county_fips": point["county_fips"],
        "county_name": point["county_name"],
        "state_fips": point["state_fips"],
        "state_abbr": point["state_abbr"],
        "lat": point["lat"],
        "lon": point["lon"],
        "avgSolarGhiKwhM2Day": f"{avg_daily_ghi:.3f}",
        "ghi_rows": str(len(ghi_values)),
        "ghi_min": f"{min(ghi_values):.1f}",
        "ghi_max": f"{max(ghi_values):.1f}",
        "source": source_label(raw_csv),
        "raw_csv": str(raw_csv),
    }


def write_summary_csv(rows: list[dict[str, str]], output_csv: Path) -> None:
    """Write rebuilt county solar summaries."""
    output_csv.parent.mkdir(parents=True, exist_ok=True)
    with output_csv.open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(
            handle,
            fieldnames=[
                "county_fips",
                "county_name",
                "state_fips",
                "state_abbr",
                "lat",
                "lon",
                "avgSolarGhiKwhM2Day",
                "ghi_rows",
                "ghi_min",
                "ghi_max",
                "source",
                "raw_csv",
            ],
        )
        writer.writeheader()
        writer.writerows(rows)


def parse_args() -> argparse.Namespace:
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--points-csv", type=Path, default=DEFAULT_POINTS_CSV)
    parser.add_argument(
        "--representative-point-csv-dir",
        dest="representative_point_csv_dir",
        type=Path,
        default=DEFAULT_REPRESENTATIVE_POINT_CSV_DIR,
    )
    parser.add_argument("--output-csv", type=Path, default=DEFAULT_OUTPUT_CSV)
    return parser.parse_args()


def main() -> None:
    """Run the cached raw CSV summarizer."""
    args = parse_args()
    rows: list[dict[str, str]] = []
    missing: list[str] = []
    failed: list[str] = []

    for point in read_county_points(args.points_csv):
        county_fips = point["county_fips"]
        raw_csv = select_raw_csv(args.representative_point_csv_dir, county_fips)
        if raw_csv is None:
            missing.append(county_fips)
            continue

        try:
            rows.append(summarize_county(point, raw_csv))
        except (OSError, ValueError) as exc:
            failed.append(f"{county_fips}: {exc}")

    write_summary_csv(rows, args.output_csv)
    print(f"Wrote {len(rows)} county representative-point solar summaries to {args.output_csv}")
    if missing:
        print(f"Missing raw CSVs for {len(missing)} counties: {', '.join(missing[:20])}")
    if failed:
        print(f"Failed to summarize {len(failed)} counties:")
        for message in failed[:20]:
            print(f"  {message}")


if __name__ == "__main__":
    main()