Knou/Climate-Mood-Analysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete Köppen-Geiger filter review with Mixed climate class

Browse Source 
Classify each county by area-weighted Köppen class shares: a county is
predominantly its top class when that class covers at least 50% of its
land and leads the runner-up by at least 5 percentage points; otherwise
it is Mixed (133 of 3,143 counties in the 50 states and DC).

- Add build_county_koppen_metric.py (writes data/metrics/koppen.csv) and
  apply_koppen_metric_to_climate_data.py (writes koppenZone plus
  koppenPrimaryClass/koppenSecondaryClass for Mixed counties).
- Move shared helpers into scripts/common/ (county loading, Köppen
  legend, area-weighted raster shares); fix the 180th-meridian raster
  window for Aleutians West.
- Add check_climate_data.py to validate the app CSV.
- Draw Mixed counties in app.js as diagonal stripes of their top two
  classes, fixed to the ground and following the map at every zoom, with
  a crossfade only when the stripe size changes. Filtering a class also
  matches Mixed counties where it is primary or secondary.
- Document the rule, display, and pipeline plan in docs/ and update the
  README and data-source notes.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Knou
2026-09-14 02:54:25 -04:00
co-authored by Claude Opus 5
parent 92fbbfb2e9
commit 4d2b3e3d44
20 changed files with 6400 additions and 3450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/check_climate_data.py
+427
View File
@@ -0,0 +1,427 @@
"""Validate data/climate-data.csv before the browser app loads it.
Checks file format, columns, county keys, the county geometry join, per-metric
types and ranges, allowed blank values, and cross-field consistency. Exits with
status 1 when any check fails.
Run:
.venv\\Scripts\\python.exe scripts\\check_climate_data.py
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, FrozenSet, List, Optional, Sequence, Tuple
PROJECT_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_CLIMATE_CSV = PROJECT_ROOT / "data" / "climate-data.csv"
DEFAULT_COUNTIES_GEOJSON = PROJECT_ROOT / "data" / "geojson-counties-fips.json"
DEFAULT_METRIC_SOURCES = PROJECT_ROOT / "data" / "metric_sources.json"
FIPS_PATTERN = re.compile(r"\d{5}")
STATE_PATTERN = re.compile(r"[A-Z]{2}")
# The distinct-value check only runs on full-size files, not small fixtures.
COLLAPSE_CHECK_MIN_ROWS = 100
KOPPEN_CODES = (
"Af", "Am", "Aw", "BWh", "BWk", "BSh", "BSk",
"Csa", "Csb", "Csc", "Cwa", "Cwb", "Cwc", "Cfa", "Cfb", "Cfc",
"Dsa", "Dsb", "Dsc", "Dsd", "Dwa", "Dwb", "Dwc", "Dwd",
"Dfa", "Dfb", "Dfc", "Dfd", "ET", "EF",
)
# Counties with no predominant class; see docs/filter-calculations.md, section 1.
MIXED_KOPPEN_CLASS = "Mixed"
MONTH_NAMES = (
"January", "February", "March", "April", "May", "June",
"July", "August", "September", "October", "November", "December",
)
# NOAA nClimGrid and gridMET cover the contiguous U.S. only.
OUTSIDE_CONUS = frozenset({"AK", "HI", "PR"})
# NSRDB summaries were not requested for Puerto Rico.
NSRDB_NOT_REQUESTED = frozenset({"PR"})
# NOAA county daily files do not include Lexington city, VA.
NOAA_DAILY_MISSING_FIPS = frozenset({"51678"})
@dataclass(frozen=True)
class MetricRule:
"""Validation rule for one metric column."""
kind: str
minimum: Optional[float] = None
maximum: Optional[float] = None
integer: bool = False
categories: Tuple[str, ...] = ()
blank_states: FrozenSet[str] = frozenset()
blank_fips: FrozenSet[str] = frozenset()
min_distinct: int = 2
def _numeric(
minimum: float,
maximum: float,
*,
integer: bool = False,
blank_states: FrozenSet[str] = frozenset(),
blank_fips: FrozenSet[str] = frozenset(),
) -> MetricRule:
"""Build a numeric rule with a plausible physical range."""
return MetricRule(
"numeric",
minimum=minimum,
maximum=maximum,
integer=integer,
blank_states=blank_states,
blank_fips=blank_fips,
min_distinct=10,
)
def _categorical(categories: Sequence[str], *, blank_states: FrozenSet[str] = frozenset()) -> MetricRule:
"""Build a categorical rule limited to an allowed value list."""
return MetricRule("categorical", categories=tuple(categories), blank_states=blank_states)
# Ranges are physical plausibility limits, deliberately wider than the current
# data. They are not the app's color-scale bounds.
METRIC_RULES: Dict[str, MetricRule] = {
"koppenZone": _categorical(KOPPEN_CODES + (MIXED_KOPPEN_CLASS,)),
"avgTempF": _numeric(20, 85, blank_states=OUTSIDE_CONUS),
"avgDiurnalTempRangeF": _numeric(5, 40, blank_states=OUTSIDE_CONUS, blank_fips=NOAA_DAILY_MISSING_FIPS),
"annualPrecipIn": _numeric(1, 150, blank_states=OUTSIDE_CONUS),
"seasonalityIndex": _numeric(0, 100, integer=True, blank_states=OUTSIDE_CONUS),
"wettestPrecipMonth": _categorical(MONTH_NAMES, blank_states=OUTSIDE_CONUS),
"driestPrecipMonth": _categorical(MONTH_NAMES, blank_states=OUTSIDE_CONUS),
"absoluteExtremeDays": _numeric(0, 366, blank_states=OUTSIDE_CONUS, blank_fips=NOAA_DAILY_MISSING_FIPS),
"meanDailyGlobalHorizontalRadiationKwhM2Day": _numeric(1.5, 7.5, blank_states=NSRDB_NOT_REQUESTED),
"clearSkyGhiReductionIndex": _numeric(0, 1, blank_states=NSRDB_NOT_REQUESTED),
"avgSummerSpecificHumidityGKg": _numeric(1, 25, blank_states=OUTSIDE_CONUS),
"humidHeatDays": _numeric(0, 366, blank_states=OUTSIDE_CONUS),
}
IDENTITY_COLUMNS = ("countyFips", "countyName", "state")
# Stripe classes the app draws for Mixed Koppen counties; blank otherwise.
KOPPEN_STRIPE_COLUMNS = ("koppenPrimaryClass", "koppenSecondaryClass")
AUDIT_COLUMNS = ("humidHeatSourceFips", "humidHeatFipsAdjustment", "source")
EXPECTED_COLUMNS = IDENTITY_COLUMNS + tuple(METRIC_RULES) + KOPPEN_STRIPE_COLUMNS + AUDIT_COLUMNS
CHECK_FORMAT = "file format"
CHECK_COLUMNS = "columns"
CHECK_COUNTIES = "county keys"
CHECK_GEOMETRY = "map geometry join"
CHECK_BLANKS = "blank values"
CHECK_VALUES = "value types and ranges"
CHECK_CROSS = "cross-field consistency"
CHECK_SOURCES = "metric sources file"
@dataclass
class Report:
"""Problems found per check, in the order checks ran."""
checks: Dict[str, List[str]] = field(default_factory=dict)
notes: List[str] = field(default_factory=list)
row_count: int = 0
column_count: int = 0
def start(self, check: str) -> None:
self.checks.setdefault(check, [])
def add(self, check: str, message: str) -> None:
self.checks.setdefault(check, []).append(message)
@property
def problem_count(self) -> int:
return sum(len(messages) for messages in self.checks.values())
@property
def ok(self) -> bool:
return self.problem_count == 0
def read_climate_csv(path: Path) -> Tuple[List[str], List[List[str]]]:
"""Read the raw header and rows without DictReader's silent padding."""
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.reader(handle)
headers = next(reader, [])
rows = [row for row in reader if row]
return headers, rows
def check_structure(
report: Report, headers: List[str], raw_rows: List[List[str]]
) -> Tuple[List[str], List[Dict[str, str]]]:
"""Check encoding, header names, and row widths; return rows as dicts."""
report.start(CHECK_FORMAT)
report.start(CHECK_COLUMNS)
if headers and headers[0].startswith("\ufeff"):
report.add(CHECK_FORMAT, "file starts with a UTF-8 BOM; the app would not find the first column")
headers = [headers[0].lstrip("\ufeff")] + headers[1:]
seen = set()
for name in headers:
if name in seen:
report.add(CHECK_COLUMNS, f"duplicate column {name!r}")
seen.add(name)
for name in EXPECTED_COLUMNS:
if name not in seen:
report.add(CHECK_COLUMNS, f"missing column {name!r}")
for name in headers:
if name not in EXPECTED_COLUMNS:
report.add(CHECK_COLUMNS, f"unexpected column {name!r} (add it to check_climate_data.py if intended)")
rows: List[Dict[str, str]] = []
for index, raw in enumerate(raw_rows):
if len(raw) != len(headers):
report.add(CHECK_FORMAT, f"line {index + 2}: {len(raw)} fields, expected {len(headers)}")
continue
rows.append(dict(zip(headers, raw)))
report.row_count = len(raw_rows)
report.column_count = len(headers)
return headers, rows
def check_counties(report: Report, rows: List[Dict[str, str]]) -> None:
"""Check FIPS format and uniqueness, names, and state/FIPS agreement."""
report.start(CHECK_COUNTIES)
seen = set()
prefix_states: Dict[str, set] = {}
state_prefixes: Dict[str, set] = {}
for row in rows:
fips = row.get("countyFips") or ""
if not FIPS_PATTERN.fullmatch(fips):
report.add(CHECK_COUNTIES, f"invalid countyFips {fips!r}")
continue
if fips in seen:
report.add(CHECK_COUNTIES, f"duplicate countyFips {fips}")
seen.add(fips)
if not (row.get("countyName") or "").strip():
report.add(CHECK_COUNTIES, f"{fips}: countyName is blank")
state = row.get("state") or ""
if not STATE_PATTERN.fullmatch(state):
report.add(CHECK_COUNTIES, f"{fips}: invalid state {state!r}")
continue
prefix_states.setdefault(fips[:2], set()).add(state)
state_prefixes.setdefault(state, set()).add(fips[:2])
for prefix, states in sorted(prefix_states.items()):
if len(states) > 1:
report.add(CHECK_COUNTIES, f"state FIPS {prefix} is labeled as {sorted(states)}")
for state, prefixes in sorted(state_prefixes.items()):
if len(prefixes) > 1:
report.add(CHECK_COUNTIES, f"state {state} spans FIPS prefixes {sorted(prefixes)}")
def _feature_fips(feature: dict) -> str:
"""Return the 5-digit county FIPS stored on a GeoJSON feature."""
props = feature.get("properties") or {}
raw = feature.get("id") or props.get("id") or str(props.get("GEO_ID") or "")[-5:]
text = str(raw or "").strip()
return text.zfill(5) if text.isdigit() else text
def check_geometry(report: Report, rows: List[Dict[str, str]], geojson_path: Path) -> None:
"""Check that CSV counties and map polygons match one-to-one."""
report.start(CHECK_GEOMETRY)
if not geojson_path.exists():
report.add(CHECK_GEOMETRY, f"county geometry file not found: {geojson_path}")
return
features = json.loads(geojson_path.read_text(encoding="utf-8")).get("features", [])
geo_fips = set()
for feature in features:
fips = _feature_fips(feature)
if not FIPS_PATTERN.fullmatch(fips):
report.add(CHECK_GEOMETRY, f"map feature without a valid county FIPS: {fips!r}")
continue
if fips in geo_fips:
report.add(CHECK_GEOMETRY, f"map has duplicate polygons for {fips}")
geo_fips.add(fips)
state_fips = str((feature.get("properties") or {}).get("STATE") or "").strip()
if state_fips and state_fips.zfill(2) != fips[:2]:
report.add(CHECK_GEOMETRY, f"map feature {fips} has STATE {state_fips}")
csv_fips = {row.get("countyFips") or "" for row in rows}
for fips in sorted(csv_fips - geo_fips):
report.add(CHECK_GEOMETRY, f"{fips} is in the CSV but has no map polygon")
for fips in sorted(geo_fips - csv_fips):
report.add(CHECK_GEOMETRY, f"{fips} has a map polygon but no CSV row")
def check_metrics(report: Report, headers: List[str], rows: List[Dict[str, str]]) -> None:
"""Check each metric's blanks, types, ranges, and categories."""
report.start(CHECK_BLANKS)
report.start(CHECK_VALUES)
for key, rule in METRIC_RULES.items():
if key not in headers:
continue
present: List[object] = []
for row in rows:
fips = row.get("countyFips") or "?"
state = row.get("state") or ""
value = row.get(key) or ""
if value == "":
if state not in rule.blank_states and fips not in rule.blank_fips:
report.add(CHECK_BLANKS, f"{fips} ({state}): {key} is blank")
continue
if value != value.strip():
report.add(CHECK_VALUES, f"{fips}: {key}={value!r} has surrounding whitespace")
continue
if rule.kind == "categorical":
if value in rule.categories:
present.append(value)
else:
report.add(CHECK_VALUES, f"{fips}: {key}={value!r} is not an allowed category")
continue
try:
number = float(value)
except ValueError:
report.add(CHECK_VALUES, f"{fips}: {key}={value!r} is not a number")
continue
if not math.isfinite(number):
report.add(CHECK_VALUES, f"{fips}: {key}={value!r} is not finite")
continue
if rule.integer and not number.is_integer():
report.add(CHECK_VALUES, f"{fips}: {key}={value} should be a whole number")
if not rule.minimum <= number <= rule.maximum:
report.add(CHECK_VALUES, f"{fips}: {key}={value} outside {rule.minimum:g}..{rule.maximum:g}")
present.append(number)
if len(rows) >= COLLAPSE_CHECK_MIN_ROWS and len(set(present)) < rule.min_distinct:
report.add(
CHECK_VALUES,
f"{key} has only {len(set(present))} distinct values; the column may have been overwritten",
)
def check_cross_fields(report: Report, rows: List[Dict[str, str]]) -> None:
"""Check relationships between columns within each row."""
report.start(CHECK_CROSS)
for row in rows:
fips = row.get("countyFips") or "?"
wet = row.get("wettestPrecipMonth") or ""
dry = row.get("driestPrecipMonth") or ""
if bool(wet) != bool(dry):
report.add(CHECK_CROSS, f"{fips}: only one of wettestPrecipMonth/driestPrecipMonth is set")
elif wet and wet == dry:
report.add(CHECK_CROSS, f"{fips}: wettest and driest month are both {wet}")
heat = row.get("humidHeatDays") or ""
heat_source = row.get("humidHeatSourceFips") or ""
if bool(heat) != bool(heat_source):
report.add(CHECK_CROSS, f"{fips}: humidHeatDays and humidHeatSourceFips must both be set or both blank")
if heat_source and not FIPS_PATTERN.fullmatch(heat_source):
report.add(CHECK_CROSS, f"{fips}: invalid humidHeatSourceFips {heat_source!r}")
elif heat_source and heat_source != fips and not (row.get("humidHeatFipsAdjustment") or "").strip():
report.add(CHECK_CROSS, f"{fips}: uses proxy county {heat_source} without a humidHeatFipsAdjustment note")
zone = row.get("koppenZone") or ""
primary = row.get("koppenPrimaryClass") or ""
secondary = row.get("koppenSecondaryClass") or ""
if zone == MIXED_KOPPEN_CLASS:
if not (primary and secondary):
report.add(CHECK_CROSS, f"{fips}: Mixed Koppen county needs koppenPrimaryClass and koppenSecondaryClass")
elif primary not in KOPPEN_CODES or secondary not in KOPPEN_CODES:
report.add(CHECK_CROSS, f"{fips}: invalid Koppen stripe classes {primary!r}/{secondary!r}")
elif primary == secondary:
report.add(CHECK_CROSS, f"{fips}: Koppen stripe classes are both {primary}")
elif primary or secondary:
report.add(CHECK_CROSS, f"{fips}: Koppen stripe classes are set but koppenZone is {zone or 'blank'}, not Mixed")
if not (row.get("source") or "").strip():
report.add(CHECK_CROSS, f"{fips}: source is blank")
def check_metric_sources(report: Report, path: Path) -> None:
"""Check that the metric metadata file parses and names real metrics."""
report.start(CHECK_SOURCES)
if not path.exists():
report.add(CHECK_SOURCES, f"{path.name} not found")
return
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
report.add(CHECK_SOURCES, f"{path.name} is not valid JSON: {exc}")
return
if not isinstance(data, dict) or not isinstance(data.get("metrics"), dict):
report.add(CHECK_SOURCES, f'{path.name} must be a JSON object with a "metrics" object')
return
documented = set(data["metrics"])
for key in sorted(documented - set(METRIC_RULES)):
report.add(CHECK_SOURCES, f"{path.name} describes unknown metric {key!r}")
covered = len(documented & set(METRIC_RULES))
report.notes.append(f"{path.name} documents {covered} of {len(METRIC_RULES)} metrics.")
def run_checks(csv_path: Path, geojson_path: Path, metric_sources_path: Path) -> Report:
"""Run every check and return the combined report."""
report = Report()
headers, raw_rows = read_climate_csv(csv_path)
headers, rows = check_structure(report, headers, raw_rows)
check_counties(report, rows)
check_geometry(report, rows, geojson_path)
check_metrics(report, headers, rows)
check_cross_fields(report, rows)
check_metric_sources(report, metric_sources_path)
return report
def print_report(report: Report, csv_path: Path, max_examples: int) -> None:
"""Print a pass/fail line per check with example problems."""
print(f"Checked {csv_path}: {report.row_count} rows, {report.column_count} columns")
for check, messages in report.checks.items():
status = "PASS" if not messages else f"FAIL ({len(messages)})"
print(f" {status:<11}{check}")
for message in messages[:max_examples]:
print(f" - {message}")
if len(messages) > max_examples:
print(f" ... and {len(messages) - max_examples} more")
for note in report.notes:
print(f" note: {note}")
print("Result: PASS" if report.ok else f"Result: FAIL ({report.problem_count} problems)")
def parse_args() -> argparse.Namespace:
"""Define and parse command-line options for this checker."""
parser = argparse.ArgumentParser(description="Validate the browser app's county climate CSV.")
parser.add_argument("--csv", type=Path, default=DEFAULT_CLIMATE_CSV, help="Climate data CSV path.")
parser.add_argument("--geojson", type=Path, default=DEFAULT_COUNTIES_GEOJSON, help="County GeoJSON path.")
parser.add_argument("--metric-sources", type=Path, default=DEFAULT_METRIC_SOURCES, help="Metric metadata JSON path.")
parser.add_argument("--max-examples", type=int, default=10, help="Problems to print per failing check.")
return parser.parse_args()
def main() -> int:
args = parse_args()
if not args.csv.exists():
print(f"Climate data CSV not found: {args.csv}", file=sys.stderr)
return 2
report = run_checks(args.csv, args.geojson, args.metric_sources)
print_report(report, args.csv, args.max_examples)
return 0 if report.ok else 1
if __name__ == "__main__":
sys.exit(main())