Knou/Climate-Mood-Analysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove locally extreme days metric

Browse Source
This commit is contained in:
Justin Fisher
2026-06-11 15:23:53 -04:00
parent b8d59e3788
commit ce9672a734
3 changed files with 3269 additions and 3325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app.js
+4 -29
View File
@@ -360,25 +360,7 @@ const METRICS = {
}
};
const DETAIL_ONLY_METRICS = {
oldExtremeDays: {
label: "Previous Extreme Days / Year",
type: "numeric",
unit: "days",
source: {
description:
"Older app proxy preserved for comparison. It came from NOAA 1991-2020 normals using monthly proxy thresholds before the daily nClimGrid replacement was added.",
links: [
{
label: "NOAA NCEI U.S. Climate Normals",
url: "https://www.ncei.noaa.gov/products/land-based-station/us-climate-normals"
}
]
}
}
};
const DETAIL_METRICS = { ...METRICS, ...DETAIL_ONLY_METRICS };
const DETAIL_METRICS = METRICS;
const DATA_METRIC_KEYS = Object.keys(METRICS);
const COUNTY_DETAIL_METRIC_KEYS = [...DATA_METRIC_KEYS];
const METRIC_GROUPS = [
@@ -390,7 +372,7 @@ const METRIC_GROUPS = [
{
key: "temperatureExtremes",
label: "Temperature & Extremes",
metricKeys: ["avgTempF", "avgDiurnalTempRangeF", "extremeDays", "absoluteExtremeDays", "humidHeatDays"]
metricKeys: ["avgTempF", "avgDiurnalTempRangeF", "absoluteExtremeDays", "humidHeatDays"]
},
{
key: "precipitationMoisture",
@@ -534,12 +516,9 @@ function getSourceEntries() {
return [...BASE_SOURCE_ENTRIES, ...metricEntries];
}
// Keep retired internal audit tags out of the selected-county panel.
// Normalize source text for the selected-county panel.
function getVisibleSourceTag(sourceText) {
return String(sourceText)
.split(" + ")
.filter((tag) => !tag.includes("locally-extreme") && !tag.startsWith("extremeDays=") && !tag.startsWith("oldExtremeDays="))
.join(" + ");
return String(sourceText);
}
// Render the scrollable sources dialog without injecting source text as HTML.
@@ -914,10 +893,6 @@ function sanitizeOverrideRecord(rawRecord) {
}
});
Object.keys(DETAIL_ONLY_METRICS).forEach((metricKey) => {
sanitizedRecord[metricKey] = parseNullableNumber(rawRecord[metricKey]);
});
return sanitizedRecord;
}
data/climate-data.csv
+3222 -3222
View File
File diff suppressed because it is too large Load Diff
scripts/apply_locally_extreme_metric_to_climate_data.py
+43 -74
View File
@@ -1,11 +1,10 @@
#!/usr/bin/env python3
"""
Apply NOAA daily extreme-day and polygon solar metrics to the app CSV.
Apply NOAA absolute-temperature threshold and polygon solar metrics to the app CSV.
The browser app already reads the `extremeDays` column, so this script updates
that column to the average locally extreme days/year while preserving the older
proxy value in `oldExtremeDays`. It also writes an absolute daily heat/cold
bucket metric from the same NOAA nClimGrid-Daily county data.
This script writes an absolute daily heat/cold threshold metric from NOAA
nClimGrid-Daily county data. Locally percentile-based extreme-day metrics are
not included in the app CSV.
When an NSRDB polygon archive summary is available, this also replaces the
older representative-point solar GHI value with the county polygon value. Rows
@@ -25,32 +24,17 @@ DEFAULT_COMPARISON = REPO_ROOT / "data" / "noaa" / "county_locally_extreme_days_
DEFAULT_POLYGON_SOLAR_GHI = REPO_ROOT / "data" / "nrel" / "county_polygon_ghi_summary.csv"
DEFAULT_REPRESENTATIVE_POINT_SOLAR_GHI = REPO_ROOT / "data" / "nrel" / "county_representative_point_ghi_summary.csv"
BASE_FIELDS = [
"countyFips",
"countyName",
"state",
"koppenZone",
"avgTempF",
"annualPrecipIn",
"seasonalityIndex",
"wettestPrecipMonth",
"driestPrecipMonth",
LEGACY_FIELDS = {
"absoluteExtremeHeatDays",
"absoluteExtremeColdDays",
"extremeDays",
"locallyExtremeDays",
"locallyExtremeHotDays",
"locallyExtremeColdDays",
"absoluteExtremeDays",
"oldExtremeDays",
"locallyExtremeAnalysisYears",
"locallyExtremeSourceFips",
"locallyExtremeFipsAdjustment",
"avgSolarGhiKwhM2Day",
"source",
]
LEGACY_FIELDS = {
"absoluteExtremeHeatDays",
"absoluteExtremeColdDays",
}
SOLAR_SOURCE_TAGS = {
@@ -132,26 +116,28 @@ def load_solar_ghi_lookup(solar_ghi_csv: Path) -> dict[str, str]:
return lookup
def append_source_tag(source: str, has_local_value: bool) -> str:
"""Add the locally extreme provenance tags without duplicating old tags."""
old_source = source.strip() or "unknown-source"
old_source = old_source.replace(" + absoluteExtremeHeatDays=tmax-gte-95f", "")
old_source = old_source.replace(" + absoluteExtremeColdDays=tmin-lte-0f", "")
if not has_local_value:
return f"{old_source} + missing-noaa-locally-extreme-days"
has_local_tag = "extremeDays=locally-extreme-noaa-nclimgrid-daily-1991-2025" in old_source
has_absolute_tag = "absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f" in old_source
if has_local_tag and has_absolute_tag:
return old_source
if has_local_tag:
return f"{old_source} + absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f"
return (
f"{old_source} + extremeDays=locally-extreme-noaa-nclimgrid-daily-1991-2025 "
"+ absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f "
"+ oldExtremeDays=previous-monthly-proxy"
def replace_extreme_source_tags(source: str, has_absolute_value: bool) -> str:
"""Remove retired extreme-day tags and retain absolute-threshold provenance."""
source = source.replace("noaa-nclimgrid-1991-2020 (monthly-proxy)", "noaa-nclimgrid-1991-2020")
retired_prefixes = (
"extremeDays=",
"oldExtremeDays=",
"missing-noaa-locally-extreme-days",
"missing-noaa-absolute-extreme-days",
)
parts = [
part.strip()
for part in (source.strip() or "unknown-source").split("+")
if part.strip()
and "locally-extreme" not in part
and not part.strip().startswith(retired_prefixes)
and part.strip() != "absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f"
]
if has_absolute_value:
parts.append("absoluteExtremeDays=tmax-gte-95f-or-tmin-lte-0f")
else:
parts.append("missing-noaa-absolute-extreme-days")
return " + ".join(parts)
def replace_solar_source_tag(source: str, solar_source_tag: str) -> str:
@@ -175,52 +161,35 @@ def apply_locally_extreme_metric(
representative_point_solar_ghi_csv: Path,
out: Path,
) -> None:
"""Replace app extreme-day fields and solar GHI with improved metrics."""
"""Replace the app absolute threshold and solar GHI metrics."""
original_fields, climate_rows = read_csv_rows(climate_data)
local_lookup = load_locally_extreme_lookup(comparison_csv)
polygon_solar_lookup = load_solar_ghi_lookup(polygon_solar_ghi_csv)
representative_solar_lookup = load_solar_ghi_lookup(representative_point_solar_ghi_csv)
extra_fields = [
field
for field in original_fields
if field not in BASE_FIELDS and field not in LEGACY_FIELDS
]
fieldnames = BASE_FIELDS + extra_fields
fieldnames = [field for field in original_fields if field not in LEGACY_FIELDS]
if "absoluteExtremeDays" not in fieldnames:
insert_after = "driestPrecipMonth"
insert_at = fieldnames.index(insert_after) + 1 if insert_after in fieldnames else len(fieldnames)
fieldnames.insert(insert_at, "absoluteExtremeDays")
updated_count = 0
missing_count = 0
absolute_updated_count = 0
absolute_missing_count = 0
polygon_solar_count = 0
representative_solar_count = 0
missing_solar_count = 0
for row in climate_rows:
county_fips = normalize_fips(row.get("countyFips", ""))
local_row = local_lookup.get(county_fips)
old_extreme_days = (row.get("oldExtremeDays") or row.get("extremeDays") or "").strip()
row["oldExtremeDays"] = old_extreme_days
if local_row:
row["extremeDays"] = local_row.get("avgLocallyExtremeDays", "")
row["locallyExtremeDays"] = local_row.get("avgLocallyExtremeDays", "")
row["locallyExtremeHotDays"] = local_row.get("avgHotExtremeDays", "")
row["locallyExtremeColdDays"] = local_row.get("avgColdExtremeDays", "")
row["absoluteExtremeDays"] = local_row.get("avgAbsoluteExtremeDays", "")
row["locallyExtremeAnalysisYears"] = local_row.get("analysisYears", "")
row["locallyExtremeSourceFips"] = local_row.get("countyFips", "")
row["locallyExtremeFipsAdjustment"] = local_row.get("fipsAdjustment", "")
row["source"] = append_source_tag(row.get("source", ""), has_local_value=True)
updated_count += 1
row["source"] = replace_extreme_source_tags(row.get("source", ""), has_absolute_value=True)
absolute_updated_count += 1
else:
row["extremeDays"] = ""
row["locallyExtremeDays"] = ""
row["locallyExtremeHotDays"] = ""
row["locallyExtremeColdDays"] = ""
row["absoluteExtremeDays"] = ""
row["locallyExtremeAnalysisYears"] = ""
row["locallyExtremeSourceFips"] = ""
row["locallyExtremeFipsAdjustment"] = ""
row["source"] = append_source_tag(row.get("source", ""), has_local_value=False)
missing_count += 1
row["source"] = replace_extreme_source_tags(row.get("source", ""), has_absolute_value=False)
absolute_missing_count += 1
polygon_solar_value = polygon_solar_lookup.get(county_fips)
representative_solar_value = representative_solar_lookup.get(county_fips)
@@ -250,8 +219,8 @@ def apply_locally_extreme_metric(
writer.writerows(climate_rows)
print(f"Wrote {len(climate_rows)} rows to {out}")
print(f"Updated active extremeDays from locally extreme data for {updated_count} rows.")
print(f"Left active extremeDays blank for {missing_count} rows without locally extreme data.")
print(f"Updated absoluteExtremeDays for {absolute_updated_count} rows.")
print(f"Left absoluteExtremeDays blank for {absolute_missing_count} rows without NOAA data.")
print(f"Updated avgSolarGhiKwhM2Day from polygon archives for {polygon_solar_count} rows.")
print(f"Kept representative-point solar fallback for {representative_solar_count} rows.")
print(f"Left avgSolarGhiKwhM2Day blank for {missing_solar_count} rows without solar data.")
@@ -260,7 +229,7 @@ def apply_locally_extreme_metric(
def parse_args() -> argparse.Namespace:
"""Parse command-line paths for the climate-data update."""
parser = argparse.ArgumentParser(
description="Apply locally extreme days/year and polygon solar GHI to climate-data.csv."
description="Apply absolute threshold days/year and polygon solar GHI to climate-data.csv."
)
parser.add_argument("--climate-data", type=Path, default=DEFAULT_CLIMATE_DATA)
parser.add_argument("--comparison-csv", type=Path, default=DEFAULT_COMPARISON)