Files
Climate-Mood-Analysis/scripts/summarize_nsrdb_county_polygon_cloud_archives.py
T
2026-06-11 14:50:33 -04:00

528 lines
21 KiB
Python

#!/usr/bin/env python3
"""
Summarize downloaded NSRDB county polygon cloud archives.
Each downloaded polygon archive contains one CSV per NSRDB grid site that
intersects the county polygon or tile. This script computes area-weighted
county-level cloudiness metrics across those site CSVs, combining multiple tile
archives back into one county summary when present.
Primary metric:
cloudinessIndexPct = 1 - mean(clamped(GHI / Clearsky GHI, 0, 1))
Despite the legacy "Pct" field name, the primary index is stored on a 0-1 scale.
Cloud Type bucket fields are stored as percentages.
"""
from __future__ import annotations
import argparse
import csv
import math
import re
import statistics
import zipfile
from pathlib import Path
from summarize_nsrdb_county_polygon_archives import (
area_weighted_average,
archive_groups_by_county,
archive_signature,
county_fips_from_archive,
existing_archive_signature,
extract_site_lon_lat,
format_float,
load_county_geometries,
read_grouped_lookup,
read_lookup,
)
DEFAULT_ARCHIVE_DIR = Path("data/nrel/polygon_cloud_archives")
DEFAULT_COUNTIES_GEOJSON = Path("data/geojson-counties-fips.json")
DEFAULT_REQUESTS_CSV = Path("data/nrel/county_polygon_cloud_request_manifest.csv")
DEFAULT_REPRESENTATIVE_POINT_CSV = Path("data/nrel/county_representative_point_cloud_summary.csv")
DEFAULT_OUTPUT_CSV = Path("data/nrel/county_polygon_cloud_summary.csv")
DEFAULT_CELL_SIZE_M = 4000.0
DEFAULT_ARCHIVE_GLOB = "*.zip"
DEFAULT_MIN_CLEARSKY_GHI = 50.0
SUMMARY_FIELDS = [
"county_fips",
"county_name",
"state_abbr",
"archive_zip",
"polygon_sites",
"request_site_count",
"rows_per_site_min",
"rows_per_site_max",
"daylight_rows_per_site_min",
"daylight_rows_per_site_max",
"cloudinessIndexPct",
"areaWeightedCloudinessIndexPct",
"areaWeightedAvgObservedToClearskyRatio",
"areaWeightedSites",
"weightedCellAreaKm2",
"countyAreaKm2",
"site_cloudiness_min",
"site_cloudiness_max",
"site_cloudiness_stddev",
"clearOrProbablyClearPct",
"cloudyOrObscuredPct",
"fogPct",
"waterCloudPct",
"iceCloudPct",
"cirrusPct",
"unknownCloudTypePct",
"representativePointCloudinessIndexPct",
"areaWeightedMinusRepresentativePoint",
"areaWeightedPctDiffFromRepresentativePoint",
]
def normalized_header(value: str) -> str:
"""Normalize a CSV header to make NSRDB spelling variations easier to match."""
return re.sub(r"[^a-z0-9]+", "", value.lower())
def find_column(header: list[str], candidates: set[str]) -> int | None:
"""Return the first header index whose normalized name matches a candidate."""
normalized_candidates = {normalized_header(candidate) for candidate in candidates}
for index, value in enumerate(header):
if normalized_header(value) in normalized_candidates:
return index
return None
def find_data_header(rows: list[list[str]]) -> int | None:
"""Return the NSRDB hourly data header row index."""
required = {"Year", "Month", "Day", "Hour", "Minute"}
for index, row in enumerate(rows):
if required.issubset(set(row)):
return index
return None
def parse_optional_float(row: list[str], index: int | None) -> float | None:
"""Parse an optional float field from a CSV row."""
if index is None or len(row) <= index:
return None
value = row[index].strip()
if not value:
return None
try:
parsed = float(value)
except ValueError:
return None
return parsed if math.isfinite(parsed) else None
def parse_cloud_type(row: list[str], index: int | None) -> str:
"""Parse an optional Cloud Type code as a stable string."""
if index is None or len(row) <= index:
return ""
value = row[index].strip()
if not value:
return ""
try:
return str(int(float(value)))
except ValueError:
return value
def pct(count: int, total: int) -> float:
"""Return a percentage or NaN when there is no denominator."""
return count / total * 100 if total else math.nan
def site_cloud_metrics(csv_text: str, min_clearsky_ghi: float) -> dict[str, float | int]:
"""Return site-level cloudiness metrics from one NSRDB CSV response."""
rows = list(csv.reader(csv_text.splitlines()))
header_index = find_data_header(rows)
if header_index is None:
raise ValueError("Could not find the NSRDB data header row.")
header = rows[header_index]
ghi_index = find_column(header, {"GHI", "ghi"})
clearsky_ghi_index = find_column(
header,
{
"Clearsky GHI",
"Clear Sky GHI",
"Clear-sky GHI",
"clearsky_ghi",
"clear_sky_ghi",
},
)
cloud_type_index = find_column(header, {"Cloud Type", "cloud_type", "cloudtype"})
if ghi_index is None:
raise ValueError("Could not find a GHI column in the NSRDB response.")
if clearsky_ghi_index is None:
raise ValueError("Could not find a Clearsky GHI column in the NSRDB response.")
if cloud_type_index is None:
raise ValueError("Could not find a Cloud Type column in the NSRDB response.")
all_rows = 0
daylight_rows = 0
ratio_sum = 0.0
daylight_cloud_type_counts: dict[str, int] = {}
for row in rows[header_index + 1 :]:
if not row:
continue
all_rows += 1
ghi = parse_optional_float(row, ghi_index)
clearsky_ghi = parse_optional_float(row, clearsky_ghi_index)
cloud_type = parse_cloud_type(row, cloud_type_index)
if ghi is None or clearsky_ghi is None or clearsky_ghi < min_clearsky_ghi:
continue
ratio_sum += min(1.0, max(0.0, ghi / clearsky_ghi))
daylight_rows += 1
if cloud_type:
daylight_cloud_type_counts[cloud_type] = daylight_cloud_type_counts.get(cloud_type, 0) + 1
if daylight_rows == 0:
raise ValueError("No daylight rows with valid GHI and Clearsky GHI were found.")
avg_ratio = ratio_sum / daylight_rows
cloudiness = 1 - avg_ratio
clear_or_probably_clear = daylight_cloud_type_counts.get("0", 0) + daylight_cloud_type_counts.get("1", 0)
cloudy_or_obscured = sum(
daylight_cloud_type_counts.get(code, 0)
for code in ["2", "3", "4", "5", "6", "7", "8", "9", "11", "12"]
)
water_cloud = (
daylight_cloud_type_counts.get("3", 0)
+ daylight_cloud_type_counts.get("4", 0)
+ daylight_cloud_type_counts.get("5", 0)
)
ice_cloud = (
daylight_cloud_type_counts.get("6", 0)
+ daylight_cloud_type_counts.get("8", 0)
+ daylight_cloud_type_counts.get("9", 0)
)
unknown_cloud_type = daylight_cloud_type_counts.get("10", 0) + daylight_cloud_type_counts.get("-15", 0)
return {
"cloudiness": cloudiness,
"avg_ratio": avg_ratio,
"all_rows": all_rows,
"daylight_rows": daylight_rows,
"clear_or_probably_clear_pct": pct(clear_or_probably_clear, daylight_rows),
"cloudy_or_obscured_pct": pct(cloudy_or_obscured, daylight_rows),
"fog_pct": pct(daylight_cloud_type_counts.get("2", 0), daylight_rows),
"water_cloud_pct": pct(water_cloud, daylight_rows),
"ice_cloud_pct": pct(ice_cloud, daylight_rows),
"cirrus_pct": pct(daylight_cloud_type_counts.get("7", 0), daylight_rows),
"unknown_cloud_type_pct": pct(unknown_cloud_type, daylight_rows),
}
def weighted_metric(
site_metrics: list[dict[str, float | int]],
metric_key: str,
site_lon_lats: list[tuple[float, float]],
county_geometry: list[list[list[tuple[float, float]]]] | None,
cell_size_m: float,
) -> dict[str, float | int]:
"""Area-weight one metric across sites using estimated grid-cell overlap."""
values = [float(metrics[metric_key]) for metrics in site_metrics]
return area_weighted_average(values, site_lon_lats, county_geometry, cell_size_m)
def summarize_archives(
paths: list[Path],
county_geometry: list[list[list[tuple[float, float]]]] | None,
cell_size_m: float,
min_clearsky_ghi: float,
) -> dict[str, float | int]:
"""Summarize all NSRDB site CSVs in one or more ZIP archives."""
site_metrics: list[dict[str, float | int]] = []
site_lon_lats: list[tuple[float, float]] = []
empty_csv_members: list[str] = []
for path in paths:
with zipfile.ZipFile(path) as archive:
bad_member = archive.testzip()
if bad_member:
raise zipfile.BadZipFile(
f"CRC check failed for {bad_member} in {path.name}"
)
csv_names = sorted(name for name in archive.namelist() if name.lower().endswith(".csv"))
for csv_name in csv_names:
csv_text = archive.read(csv_name).decode("utf-8-sig")
if not csv_text.strip():
empty_csv_members.append(f"{path.name}:{csv_name}")
continue
site_metrics.append(site_cloud_metrics(csv_text, min_clearsky_ghi))
site_lon_lats.append(extract_site_lon_lat(csv_text))
if not site_metrics:
raise ValueError(f"No CSV files found in {', '.join(str(path) for path in paths)}")
cloudiness_values = [float(metrics["cloudiness"]) for metrics in site_metrics]
row_counts = [int(metrics["all_rows"]) for metrics in site_metrics]
daylight_row_counts = [int(metrics["daylight_rows"]) for metrics in site_metrics]
weighted_cloudiness = weighted_metric(site_metrics, "cloudiness", site_lon_lats, county_geometry, cell_size_m)
weighted_avg_ratio = weighted_metric(site_metrics, "avg_ratio", site_lon_lats, county_geometry, cell_size_m)
return {
"polygon_sites": len(site_metrics),
"rows_per_site_min": min(row_counts),
"rows_per_site_max": max(row_counts),
"daylight_rows_per_site_min": min(daylight_row_counts),
"daylight_rows_per_site_max": max(daylight_row_counts),
"area_weighted_cloudiness": weighted_cloudiness["area_weighted_avg"],
"area_weighted_avg_ratio": weighted_avg_ratio["area_weighted_avg"],
"area_weighted_sites": weighted_cloudiness["area_weighted_sites"],
"weighted_cell_area_km2": weighted_cloudiness["weighted_cell_area_km2"],
"county_area_km2": weighted_cloudiness["county_area_km2"],
"site_cloudiness_min": min(cloudiness_values),
"site_cloudiness_max": max(cloudiness_values),
"site_cloudiness_stddev": statistics.pstdev(cloudiness_values) if len(cloudiness_values) > 1 else 0.0,
"clear_or_probably_clear_pct": weighted_metric(
site_metrics, "clear_or_probably_clear_pct", site_lon_lats, county_geometry, cell_size_m
)["area_weighted_avg"],
"cloudy_or_obscured_pct": weighted_metric(
site_metrics, "cloudy_or_obscured_pct", site_lon_lats, county_geometry, cell_size_m
)["area_weighted_avg"],
"fog_pct": weighted_metric(site_metrics, "fog_pct", site_lon_lats, county_geometry, cell_size_m)[
"area_weighted_avg"
],
"water_cloud_pct": weighted_metric(
site_metrics, "water_cloud_pct", site_lon_lats, county_geometry, cell_size_m
)["area_weighted_avg"],
"ice_cloud_pct": weighted_metric(site_metrics, "ice_cloud_pct", site_lon_lats, county_geometry, cell_size_m)[
"area_weighted_avg"
],
"cirrus_pct": weighted_metric(site_metrics, "cirrus_pct", site_lon_lats, county_geometry, cell_size_m)[
"area_weighted_avg"
],
"unknown_cloud_type_pct": weighted_metric(
site_metrics, "unknown_cloud_type_pct", site_lon_lats, county_geometry, cell_size_m
)["area_weighted_avg"],
"empty_csv_members": len(empty_csv_members),
}
def archive_part_id(path: Path) -> str:
"""Return county or tileNNN for one downloaded archive filename."""
parts = path.stem.split("_")
return parts[1] if len(parts) > 1 and parts[1].startswith("tile") else "county"
def missing_archive_parts(
archive_paths: list[Path],
request_rows: list[dict[str, str]],
) -> list[str]:
"""Return requested county/tile archive parts that are not downloaded."""
expected = {
row.get("tile_id", "").strip() or "county"
for row in request_rows
if row.get("download_url", "").strip()
}
actual = {archive_part_id(path) for path in archive_paths}
return sorted(expected - actual)
def build_summary_row(
archive_paths: list[Path],
polygon_summary: dict[str, float | int],
request_rows: list[dict[str, str]],
representative_point_row: dict[str, str],
) -> dict[str, str]:
"""Build one output row from polygon and optional representative-point summaries."""
county_fips = county_fips_from_archive(archive_paths[0])
request_row = request_rows[0] if request_rows else {}
request_site_counts = [
int(row["site_count"])
for row in request_rows
if str(row.get("site_count", "")).strip().isdigit()
]
archive_zip = ";".join(str(path) for path in archive_paths)
area_weighted_cloudiness = float(polygon_summary["area_weighted_cloudiness"])
representative_point_cloudiness = (
float(representative_point_row["cloudinessIndexPct"])
if representative_point_row.get("cloudinessIndexPct")
else None
)
weighted_minus_representative_point = (
area_weighted_cloudiness - representative_point_cloudiness
if representative_point_cloudiness is not None
else None
)
weighted_pct_diff_representative_point = (
weighted_minus_representative_point / representative_point_cloudiness * 100
if representative_point_cloudiness not in (None, 0)
else None
)
return {
"county_fips": county_fips,
"county_name": request_row.get("county_name") or representative_point_row.get("county_name", ""),
"state_abbr": request_row.get("state_abbr") or representative_point_row.get("state_abbr", ""),
"archive_zip": archive_zip,
"polygon_sites": str(polygon_summary["polygon_sites"]),
"request_site_count": str(sum(request_site_counts)) if request_site_counts else request_row.get("site_count", ""),
"rows_per_site_min": str(polygon_summary["rows_per_site_min"]),
"rows_per_site_max": str(polygon_summary["rows_per_site_max"]),
"daylight_rows_per_site_min": str(polygon_summary["daylight_rows_per_site_min"]),
"daylight_rows_per_site_max": str(polygon_summary["daylight_rows_per_site_max"]),
"cloudinessIndexPct": format_float(area_weighted_cloudiness, 4),
"areaWeightedCloudinessIndexPct": format_float(area_weighted_cloudiness, 4),
"areaWeightedAvgObservedToClearskyRatio": format_float(float(polygon_summary["area_weighted_avg_ratio"]), 4),
"areaWeightedSites": str(polygon_summary["area_weighted_sites"]),
"weightedCellAreaKm2": format_float(float(polygon_summary["weighted_cell_area_km2"]), 1),
"countyAreaKm2": format_float(float(polygon_summary["county_area_km2"]), 1),
"site_cloudiness_min": format_float(float(polygon_summary["site_cloudiness_min"]), 4),
"site_cloudiness_max": format_float(float(polygon_summary["site_cloudiness_max"]), 4),
"site_cloudiness_stddev": format_float(float(polygon_summary["site_cloudiness_stddev"]), 4),
"clearOrProbablyClearPct": format_float(float(polygon_summary["clear_or_probably_clear_pct"]), 2),
"cloudyOrObscuredPct": format_float(float(polygon_summary["cloudy_or_obscured_pct"]), 2),
"fogPct": format_float(float(polygon_summary["fog_pct"]), 2),
"waterCloudPct": format_float(float(polygon_summary["water_cloud_pct"]), 2),
"iceCloudPct": format_float(float(polygon_summary["ice_cloud_pct"]), 2),
"cirrusPct": format_float(float(polygon_summary["cirrus_pct"]), 2),
"unknownCloudTypePct": format_float(float(polygon_summary["unknown_cloud_type_pct"]), 2),
"representativePointCloudinessIndexPct": format_float(representative_point_cloudiness, 4),
"areaWeightedMinusRepresentativePoint": format_float(weighted_minus_representative_point, 4),
"areaWeightedPctDiffFromRepresentativePoint": format_float(weighted_pct_diff_representative_point),
}
def write_summary(rows: list[dict[str, str]], output_csv: Path) -> None:
"""Write polygon cloud summary rows to disk."""
output_csv.parent.mkdir(parents=True, exist_ok=True)
with output_csv.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=SUMMARY_FIELDS)
writer.writeheader()
writer.writerows(rows)
def run(args: argparse.Namespace) -> None:
"""Summarize downloaded polygon cloud archives."""
county_geometries = load_county_geometries(args.counties_geojson)
request_rows = read_grouped_lookup(args.requests_csv, "county_fips")
representative_point_rows = read_lookup(args.representative_point_csv, "county_fips")
existing_rows = (
read_lookup(args.output_csv, "county_fips")
if args.reuse_existing_output
else {}
)
archives = sorted(args.archive_dir.glob(args.archive_glob))
if not archives:
print(f"No archives matched {args.archive_dir / args.archive_glob}")
return
rows: list[dict[str, str]] = []
archive_groups = archive_groups_by_county(archives)
total = len(archive_groups)
reused_count = 0
for index, county_fips in enumerate(sorted(archive_groups), start=1):
archive_paths = sorted(archive_groups[county_fips])
county_request_rows = request_rows.get(county_fips, [])
missing_parts = missing_archive_parts(archive_paths, county_request_rows)
if missing_parts:
print(
f"[{index}/{total}] Skipped {county_fips}: incomplete archive set; "
f"missing {', '.join(missing_parts)}."
)
continue
existing_row = existing_rows.get(county_fips)
if existing_row and existing_archive_signature(existing_row) == archive_signature(archive_paths):
rows.append({field: existing_row.get(field, "") for field in SUMMARY_FIELDS})
reused_count += 1
continue
try:
polygon_summary = summarize_archives(
archive_paths,
county_geometries.get(county_fips),
args.cell_size_m,
args.min_clearsky_ghi,
)
row = build_summary_row(
archive_paths,
polygon_summary,
county_request_rows,
representative_point_rows.get(county_fips, {}),
)
rows.append(row)
except (OSError, ValueError, zipfile.BadZipFile) as error:
print(f"[{index}/{total}] Failed {county_fips}: {error}")
continue
tile_note = f", archives={len(archive_paths)}" if len(archive_paths) > 1 else ""
empty_note = (
f", skipped_empty_csvs={polygon_summary['empty_csv_members']}"
if polygon_summary["empty_csv_members"]
else ""
)
print(
f"[{index}/{total}] {county_fips}: "
f"sites={row['polygon_sites']}, weighted={row['areaWeightedCloudinessIndexPct']}, "
f"representative_point={row['representativePointCloudinessIndexPct'] or 'n/a'}"
f"{tile_note}{empty_note}"
)
write_summary(rows, args.output_csv)
if reused_count:
print(f"Reused {reused_count} existing rows whose archive sets still match.")
print(f"Wrote {len(rows)} rows to {args.output_csv}")
def parse_args() -> argparse.Namespace:
"""Parse command-line arguments."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--archive-dir", type=Path, default=DEFAULT_ARCHIVE_DIR)
parser.add_argument("--archive-glob", default=DEFAULT_ARCHIVE_GLOB)
parser.add_argument("--counties-geojson", type=Path, default=DEFAULT_COUNTIES_GEOJSON)
parser.add_argument("--requests-csv", type=Path, default=DEFAULT_REQUESTS_CSV)
parser.add_argument(
"--representative-point-csv",
type=Path,
default=DEFAULT_REPRESENTATIVE_POINT_CSV,
help="County representative-point cloud summary CSV.",
)
parser.add_argument("--output-csv", type=Path, default=DEFAULT_OUTPUT_CSV)
parser.add_argument(
"--cell-size-m",
type=float,
default=DEFAULT_CELL_SIZE_M,
help="Estimated NSRDB grid-cell side length in meters. GOES TMY is 4 km.",
)
parser.add_argument(
"--min-clearsky-ghi",
type=float,
default=DEFAULT_MIN_CLEARSKY_GHI,
help="Minimum Clearsky GHI W/m2 for daylight cloudiness ratio rows.",
)
parser.add_argument(
"--reuse-existing-output",
action="store_true",
help=(
"Reuse rows already present in --output-csv when the recorded archive filenames "
"match the current archive directory, and summarize only missing or changed counties."
),
)
args = parser.parse_args()
if args.cell_size_m <= 0:
parser.error("--cell-size-m must be greater than 0.")
if args.min_clearsky_ghi < 0:
parser.error("--min-clearsky-ghi must be 0 or greater.")
return args
def main() -> None:
"""Run the polygon cloud summarizer."""
run(parse_args())
if __name__ == "__main__":
main()