Remove locally extreme days metric
This commit is contained in:
Justin Fisher
@@ -1,11 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Apply NOAA daily extreme-day and polygon solar metrics to the app CSV.
|
||||
Apply NOAA absolute-temperature threshold and polygon solar metrics to the app CSV.
|
||||
|
||||
The browser app already reads the `extremeDays` column, so this script updates
|
||||
that column to the average locally extreme days/year while preserving the older
|
||||
proxy value in `oldExtremeDays`. It also writes an absolute daily heat/cold
|
||||
bucket metric from the same NOAA nClimGrid-Daily county data.
|
||||
This script writes an absolute daily heat/cold threshold metric from NOAA
|
||||
nClimGrid-Daily county data. Locally percentile-based extreme-day metrics are
|
||||
not included in the app CSV.
|
||||
|
||||
When an NSRDB polygon archive summary is available, this also replaces the
|
||||
older representative-point solar GHI value with the county polygon value. Rows
|
||||
@@ -25,32 +24,17 @@ DEFAULT_COMPARISON = REPO_ROOT / "data" / "noaa" / "county_locally_extreme_days_
|
||||
DEFAULT_POLYGON_SOLAR_GHI = REPO_ROOT / "data" / "nrel" / "county_polygon_ghi_summary.csv"
|
||||
DEFAULT_REPRESENTATIVE_POINT_SOLAR_GHI = REPO_ROOT / "data" / "nrel" / "county_representative_point_ghi_summary.csv"
|
||||
|
||||
BASE_FIELDS = [
|
||||
"countyFips",
|
||||
"countyName",
|
||||
"state",
|
||||
"koppenZone",
|
||||
"avgTempF",
|
||||
"annualPrecipIn",
|
||||
"seasonalityIndex",
|
||||
"wettestPrecipMonth",
|
||||
"driestPrecipMonth",
|
||||
LEGACY_FIELDS = {
|
||||
"absoluteExtremeHeatDays",
|
||||
"absoluteExtremeColdDays",
|
||||
"extremeDays",
|
||||
"locallyExtremeDays",
|
||||
"locallyExtremeHotDays",
|
||||
"locallyExtremeColdDays",
|
||||
"absoluteExtremeDays",
|
||||
"oldExtremeDays",
|
||||
"locallyExtremeAnalysisYears",
|
||||
"locallyExtremeSourceFips",
|
||||
"locallyExtremeFipsAdjustment",
|
||||
"avgSolarGhiKwhM2Day",
|
||||
"source",
|
||||
]
|
||||
|
||||
LEGACY_FIELDS = {
|
||||
"absoluteExtremeHeatDays",
|
||||
"absoluteExtremeColdDays",
|
||||
}
|
||||
|
||||
SOLAR_SOURCE_TAGS = {
|
||||
@@ -132,26 +116,28 @@ def load_solar_ghi_lookup(solar_ghi_csv: Path) -> dict[str, str]:
|
||||
return lookup
|
||||
|
||||
|
||||
def append_source_tag(source: str, has_local_value: bool) -> str:
|
||||
"""Add the locally extreme provenance tags without duplicating old tags."""
|
||||
old_source = source.strip() or "unknown-source"
|
||||
old_source = old_source.replace(" + absoluteExtremeHeatDays=tmax-gte-95f", "")
|
||||
old_source = old_source.replace(" + absoluteExtremeColdDays=tmin-lte-0f", "")
|
||||
if not has_local_value:
|
||||
return f"{old_source} + missing-noaa-locally-extreme-days"
|
||||
|
||||
has_local_tag = "extremeDays=locally-extreme-noaa-nclimgrid-daily-1991-2025" in old_source
|
||||
has_absolute_tag = "absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f" in old_source
|
||||
if has_local_tag and has_absolute_tag:
|
||||
return old_source
|
||||
if has_local_tag:
|
||||
return f"{old_source} + absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f"
|
||||
|
||||
return (
|
||||
f"{old_source} + extremeDays=locally-extreme-noaa-nclimgrid-daily-1991-2025 "
|
||||
"+ absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f "
|
||||
"+ oldExtremeDays=previous-monthly-proxy"
|
||||
def replace_extreme_source_tags(source: str, has_absolute_value: bool) -> str:
|
||||
"""Remove retired extreme-day tags and retain absolute-threshold provenance."""
|
||||
source = source.replace("noaa-nclimgrid-1991-2020 (monthly-proxy)", "noaa-nclimgrid-1991-2020")
|
||||
retired_prefixes = (
|
||||
"extremeDays=",
|
||||
"oldExtremeDays=",
|
||||
"missing-noaa-locally-extreme-days",
|
||||
"missing-noaa-absolute-extreme-days",
|
||||
)
|
||||
parts = [
|
||||
part.strip()
|
||||
for part in (source.strip() or "unknown-source").split("+")
|
||||
if part.strip()
|
||||
and "locally-extreme" not in part
|
||||
and not part.strip().startswith(retired_prefixes)
|
||||
and part.strip() != "absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f"
|
||||
]
|
||||
if has_absolute_value:
|
||||
parts.append("absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f")
|
||||
else:
|
||||
parts.append("missing-noaa-absolute-extreme-days")
|
||||
return " + ".join(parts)
|
||||
|
||||
|
||||
def replace_solar_source_tag(source: str, solar_source_tag: str) -> str:
|
||||
@@ -175,52 +161,35 @@ def apply_locally_extreme_metric(
|
||||
representative_point_solar_ghi_csv: Path,
|
||||
out: Path,
|
||||
) -> None:
|
||||
"""Replace app extreme-day fields and solar GHI with improved metrics."""
|
||||
"""Replace the app absolute threshold and solar GHI metrics."""
|
||||
original_fields, climate_rows = read_csv_rows(climate_data)
|
||||
local_lookup = load_locally_extreme_lookup(comparison_csv)
|
||||
polygon_solar_lookup = load_solar_ghi_lookup(polygon_solar_ghi_csv)
|
||||
representative_solar_lookup = load_solar_ghi_lookup(representative_point_solar_ghi_csv)
|
||||
|
||||
extra_fields = [
|
||||
field
|
||||
for field in original_fields
|
||||
if field not in BASE_FIELDS and field not in LEGACY_FIELDS
|
||||
]
|
||||
fieldnames = BASE_FIELDS + extra_fields
|
||||
fieldnames = [field for field in original_fields if field not in LEGACY_FIELDS]
|
||||
if "absoluteExtremeDays" not in fieldnames:
|
||||
insert_after = "driestPrecipMonth"
|
||||
insert_at = fieldnames.index(insert_after) + 1 if insert_after in fieldnames else len(fieldnames)
|
||||
fieldnames.insert(insert_at, "absoluteExtremeDays")
|
||||
|
||||
updated_count = 0
|
||||
missing_count = 0
|
||||
absolute_updated_count = 0
|
||||
absolute_missing_count = 0
|
||||
polygon_solar_count = 0
|
||||
representative_solar_count = 0
|
||||
missing_solar_count = 0
|
||||
for row in climate_rows:
|
||||
county_fips = normalize_fips(row.get("countyFips", ""))
|
||||
local_row = local_lookup.get(county_fips)
|
||||
old_extreme_days = (row.get("oldExtremeDays") or row.get("extremeDays") or "").strip()
|
||||
|
||||
row["oldExtremeDays"] = old_extreme_days
|
||||
if local_row:
|
||||
row["extremeDays"] = local_row.get("avgLocallyExtremeDays", "")
|
||||
row["locallyExtremeDays"] = local_row.get("avgLocallyExtremeDays", "")
|
||||
row["locallyExtremeHotDays"] = local_row.get("avgHotExtremeDays", "")
|
||||
row["locallyExtremeColdDays"] = local_row.get("avgColdExtremeDays", "")
|
||||
row["absoluteExtremeDays"] = local_row.get("avgAbsoluteExtremeDays", "")
|
||||
row["locallyExtremeAnalysisYears"] = local_row.get("analysisYears", "")
|
||||
row["locallyExtremeSourceFips"] = local_row.get("countyFips", "")
|
||||
row["locallyExtremeFipsAdjustment"] = local_row.get("fipsAdjustment", "")
|
||||
row["source"] = append_source_tag(row.get("source", ""), has_local_value=True)
|
||||
updated_count += 1
|
||||
row["source"] = replace_extreme_source_tags(row.get("source", ""), has_absolute_value=True)
|
||||
absolute_updated_count += 1
|
||||
else:
|
||||
row["extremeDays"] = ""
|
||||
row["locallyExtremeDays"] = ""
|
||||
row["locallyExtremeHotDays"] = ""
|
||||
row["locallyExtremeColdDays"] = ""
|
||||
row["absoluteExtremeDays"] = ""
|
||||
row["locallyExtremeAnalysisYears"] = ""
|
||||
row["locallyExtremeSourceFips"] = ""
|
||||
row["locallyExtremeFipsAdjustment"] = ""
|
||||
row["source"] = append_source_tag(row.get("source", ""), has_local_value=False)
|
||||
missing_count += 1
|
||||
row["source"] = replace_extreme_source_tags(row.get("source", ""), has_absolute_value=False)
|
||||
absolute_missing_count += 1
|
||||
|
||||
polygon_solar_value = polygon_solar_lookup.get(county_fips)
|
||||
representative_solar_value = representative_solar_lookup.get(county_fips)
|
||||
@@ -250,8 +219,8 @@ def apply_locally_extreme_metric(
|
||||
writer.writerows(climate_rows)
|
||||
|
||||
print(f"Wrote {len(climate_rows)} rows to {out}")
|
||||
print(f"Updated active extremeDays from locally extreme data for {updated_count} rows.")
|
||||
print(f"Left active extremeDays blank for {missing_count} rows without locally extreme data.")
|
||||
print(f"Updated absoluteExtremeDays for {absolute_updated_count} rows.")
|
||||
print(f"Left absoluteExtremeDays blank for {absolute_missing_count} rows without NOAA data.")
|
||||
print(f"Updated avgSolarGhiKwhM2Day from polygon archives for {polygon_solar_count} rows.")
|
||||
print(f"Kept representative-point solar fallback for {representative_solar_count} rows.")
|
||||
print(f"Left avgSolarGhiKwhM2Day blank for {missing_solar_count} rows without solar data.")
|
||||
@@ -260,7 +229,7 @@ def apply_locally_extreme_metric(
|
||||
def parse_args() -> argparse.Namespace:
|
||||
"""Parse command-line paths for the climate-data update."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Apply locally extreme days/year and polygon solar GHI to climate-data.csv."
|
||||
description="Apply absolute threshold days/year and polygon solar GHI to climate-data.csv."
|
||||
)
|
||||
parser.add_argument("--climate-data", type=Path, default=DEFAULT_CLIMATE_DATA)
|
||||
parser.add_argument("--comparison-csv", type=Path, default=DEFAULT_COMPARISON)
|
||||
|
||||
Reference in New Issue
Block a user