Justin Fisher
833 lines
29 KiB
Python
833 lines
29 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Fetch NSRDB representative-point inputs for a county cloud-cover metric.
|
|
|
|
This uses the direct single-point NSRDB CSV endpoint rather than polygon archive
|
|
requests. It is faster for first-pass cloud metrics because it downloads one CSV
|
|
per county point and can run multiple county requests concurrently.
|
|
|
|
Default requested attributes:
|
|
|
|
ghi,clearsky_ghi,cloud_type
|
|
|
|
The summary metric is based on daylight rows with valid GHI and Clearsky GHI:
|
|
|
|
cloudinessIndexPct = 1 - mean(clamped(GHI / Clearsky GHI))
|
|
|
|
where the ratio is clamped to [0, 1] so occasional above-clear-sky modeled GHI
|
|
does not create negative cloudiness.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import concurrent.futures
|
|
import csv
|
|
import getpass
|
|
import math
|
|
import re
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
|
|
DEFAULT_POINTS_CSV = Path("data/nrel/county_representative_points.csv")
|
|
DEFAULT_OUTPUT_CSV = Path("data/nrel/county_representative_point_cloud_summary.csv")
|
|
DEFAULT_ERROR_CSV = Path("data/nrel/county_representative_point_cloud_error_log.csv")
|
|
DEFAULT_CACHE_DIR = Path("data/nrel/representative_point_cloud_csv")
|
|
DEFAULT_ENDPOINT = "https://developer.nlr.gov/api/nsrdb/v2/solar/nsrdb-GOES-tmy-v4-0-0-download.csv"
|
|
DEFAULT_POLAR_ENDPOINT = "https://developer.nlr.gov/api/nsrdb/v2/solar/nsrdb-polar-tmy-v4-0-0-download.csv"
|
|
DEFAULT_TMY_NAME = "tmy-2024"
|
|
DEFAULT_POLAR_TMY_NAME = "tmy"
|
|
DEFAULT_ATTRIBUTES = "ghi,clearsky_ghi,cloud_type"
|
|
DEFAULT_INTERVAL = 60
|
|
DEFAULT_POLAR_MIN_LATITUDE = 60.0
|
|
DEFAULT_MIN_CLEARSKY_GHI = 50.0
|
|
EMAIL_PATTERN = re.compile(r"[\w.!#$%&'*+/=?^`{|}~-]+@[\w-]+(?:\.[\w-]+)+")
|
|
SENSITIVE_QUERY_PATTERN = re.compile(r"((?:api_key|email)=)([^&\s,\"']+)", re.IGNORECASE)
|
|
|
|
CLOUD_TYPE_LABELS = {
|
|
"-15": "na",
|
|
"0": "clear",
|
|
"1": "probably_clear",
|
|
"2": "fog",
|
|
"3": "water",
|
|
"4": "super_cooled_water",
|
|
"5": "mixed",
|
|
"6": "opaque_ice",
|
|
"7": "cirrus",
|
|
"8": "overlapping",
|
|
"9": "overshooting",
|
|
"10": "unknown",
|
|
"11": "dust",
|
|
"12": "smoke",
|
|
}
|
|
|
|
CLOUD_SUMMARY_FIELDS = [
|
|
"county_fips",
|
|
"county_name",
|
|
"state_fips",
|
|
"state_abbr",
|
|
"lat",
|
|
"lon",
|
|
"cloudinessIndexPct",
|
|
"avgObservedToClearskyRatio",
|
|
"daylightRows",
|
|
"allRows",
|
|
"clearOrProbablyClearPct",
|
|
"cloudyOrObscuredPct",
|
|
"fogPct",
|
|
"waterCloudPct",
|
|
"iceCloudPct",
|
|
"cirrusPct",
|
|
"unknownCloudTypePct",
|
|
"ghi_min",
|
|
"ghi_max",
|
|
"clearsky_ghi_min",
|
|
"clearsky_ghi_max",
|
|
"source",
|
|
"raw_csv",
|
|
]
|
|
|
|
|
|
def prompt_for_secret(prompt: str, current_value: str | None) -> str:
|
|
"""Prompt for a sensitive value only when it was not supplied."""
|
|
if current_value:
|
|
return current_value
|
|
return getpass.getpass(prompt).strip()
|
|
|
|
|
|
def prompt_for_text(prompt: str, current_value: str | None) -> str:
|
|
"""Prompt for normal text only when it was not supplied."""
|
|
if current_value:
|
|
return current_value
|
|
return input(prompt).strip()
|
|
|
|
|
|
def read_county_points(points_csv: Path) -> list[dict[str, str]]:
|
|
"""Read the county point CSV."""
|
|
with points_csv.open(newline="", encoding="utf-8") as handle:
|
|
return list(csv.DictReader(handle))
|
|
|
|
|
|
def select_county_points(
|
|
rows: list[dict[str, str]],
|
|
start: int,
|
|
limit: int | None,
|
|
completed_fips: set[str],
|
|
overwrite: bool,
|
|
) -> list[dict[str, str]]:
|
|
"""Return the requested batch, skipping completed counties unless overwriting."""
|
|
if start < 0:
|
|
raise ValueError("--start must be 0 or greater.")
|
|
if limit is not None and limit < 1:
|
|
raise ValueError("--limit must be 1 or greater.")
|
|
|
|
candidates = rows[start:]
|
|
if not overwrite:
|
|
candidates = [
|
|
row
|
|
for row in candidates
|
|
if row.get("county_fips") not in completed_fips
|
|
]
|
|
|
|
if limit is None:
|
|
return candidates
|
|
return candidates[:limit]
|
|
|
|
|
|
def read_existing_summary_rows(output_csv: Path) -> dict[str, dict[str, str]]:
|
|
"""Read already summarized county cloud rows keyed by FIPS."""
|
|
if not output_csv.exists():
|
|
return {}
|
|
|
|
with output_csv.open(newline="", encoding="utf-8-sig") as handle:
|
|
return {
|
|
row["county_fips"]: row
|
|
for row in csv.DictReader(handle)
|
|
if row.get("county_fips")
|
|
}
|
|
|
|
|
|
def read_previous_error_fips(error_csv: Path) -> set[str]:
|
|
"""Read county FIPS values from a previous error log."""
|
|
if not error_csv.exists():
|
|
return set()
|
|
|
|
with error_csv.open(newline="", encoding="utf-8") as handle:
|
|
return {
|
|
row["county_fips"]
|
|
for row in csv.DictReader(handle)
|
|
if row.get("county_fips")
|
|
}
|
|
|
|
|
|
def is_polar_candidate(point: dict[str, str], min_latitude: float) -> bool:
|
|
"""Return whether a point is in the latitude range for the Polar endpoint."""
|
|
try:
|
|
return float(point["lat"]) >= min_latitude
|
|
except (KeyError, ValueError):
|
|
return False
|
|
|
|
|
|
def build_nsrdb_url(
|
|
endpoint: str,
|
|
api_key: str,
|
|
email: str,
|
|
point: dict[str, str],
|
|
name: str,
|
|
attributes: str,
|
|
interval: int,
|
|
) -> str:
|
|
"""Build a single-point NSRDB CSV request URL."""
|
|
wkt = f"POINT({point['lon']} {point['lat']})"
|
|
query = {
|
|
"api_key": api_key,
|
|
"wkt": wkt,
|
|
"attributes": attributes,
|
|
"names": name,
|
|
"utc": "false",
|
|
"leap_day": "false",
|
|
"interval": str(interval),
|
|
"email": email,
|
|
}
|
|
return f"{endpoint}?{urllib.parse.urlencode(query)}"
|
|
|
|
|
|
def redact_url(url: str) -> str:
|
|
"""Return a request URL with sensitive query values removed."""
|
|
parsed_url = urllib.parse.urlsplit(url)
|
|
query = urllib.parse.parse_qsl(parsed_url.query, keep_blank_values=True)
|
|
redacted_query = [
|
|
(key, "<redacted>" if key in {"api_key", "email"} else value)
|
|
for key, value in query
|
|
]
|
|
return urllib.parse.urlunsplit(
|
|
parsed_url._replace(query=urllib.parse.urlencode(redacted_query))
|
|
)
|
|
|
|
|
|
def redact_sensitive_text(text: str) -> str:
|
|
"""Remove likely credentials and email addresses from log text."""
|
|
without_query_values = SENSITIVE_QUERY_PATTERN.sub(r"\1<redacted>", text)
|
|
return EMAIL_PATTERN.sub("<redacted>", without_query_values)
|
|
|
|
|
|
class NsrdDataRequestError(RuntimeError):
|
|
"""Store a failed NSRDB request with sanitized context for logging."""
|
|
|
|
def __init__(
|
|
self,
|
|
message: str,
|
|
status_code: int | None = None,
|
|
url: str | None = None,
|
|
response: str = "",
|
|
) -> None:
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.url = url
|
|
self.response = response
|
|
|
|
|
|
class RequestRateLimiter:
|
|
"""Coordinate network request starts across worker threads."""
|
|
|
|
def __init__(self, delay: float) -> None:
|
|
self.delay = max(0.0, delay)
|
|
self._lock = threading.Lock()
|
|
self._next_request_at = 0.0
|
|
|
|
def wait(self) -> None:
|
|
"""Wait until the next request can start."""
|
|
if self.delay <= 0:
|
|
return
|
|
|
|
with self._lock:
|
|
now = time.monotonic()
|
|
wait_seconds = max(0.0, self._next_request_at - now)
|
|
self._next_request_at = max(now, self._next_request_at) + self.delay
|
|
|
|
if wait_seconds:
|
|
time.sleep(wait_seconds)
|
|
|
|
|
|
def build_http_request_error(error: urllib.error.HTTPError, url: str) -> NsrdDataRequestError:
|
|
"""Create a structured NSRDB request error from an HTTPError."""
|
|
body = error.read().decode("utf-8-sig", errors="replace").strip()
|
|
body_excerpt = redact_sensitive_text(body[:1000]) if body else "No response body returned."
|
|
redacted_url = redact_url(url)
|
|
retry_after = error.headers.get("Retry-After") if error.headers else None
|
|
retry_message = f"\n Retry-After: {retry_after}" if retry_after else ""
|
|
message = f"HTTP {error.code} {error.reason}\n URL: {redacted_url}{retry_message}\n Response: {body_excerpt}"
|
|
return NsrdDataRequestError(
|
|
message=message,
|
|
status_code=error.code,
|
|
url=redacted_url,
|
|
response=body_excerpt,
|
|
)
|
|
|
|
|
|
def download_text(url: str, timeout: int, rate_limiter: RequestRateLimiter | None = None) -> str:
|
|
"""Download text from a URL and return its decoded body."""
|
|
request = urllib.request.Request(url, headers={"User-Agent": "county-climate-explorer/0.1"})
|
|
try:
|
|
if rate_limiter is not None:
|
|
rate_limiter.wait()
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
return response.read().decode("utf-8-sig")
|
|
except urllib.error.HTTPError as error:
|
|
raise build_http_request_error(error, url) from error
|
|
|
|
|
|
def cache_suffix(attributes: str) -> str:
|
|
"""Return a stable, readable suffix for the cached CSV filename."""
|
|
normalized = re.sub(r"[^a-z0-9]+", "-", attributes.lower()).strip("-")
|
|
return normalized or "cloud"
|
|
|
|
|
|
def read_or_download_county_csv(
|
|
point: dict[str, str],
|
|
endpoint: str,
|
|
api_key: str,
|
|
email: str,
|
|
name: str,
|
|
source_key: str,
|
|
attributes: str,
|
|
interval: int,
|
|
cache_dir: Path,
|
|
timeout: int,
|
|
overwrite: bool,
|
|
rate_limiter: RequestRateLimiter | None = None,
|
|
) -> tuple[str, Path]:
|
|
"""Return cached NSRDB CSV text, downloading it first when needed."""
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
cache_path = cache_dir / f"{point['county_fips']}_{source_key}_{name}_{cache_suffix(attributes)}.csv"
|
|
|
|
if cache_path.exists() and not overwrite:
|
|
return cache_path.read_text(encoding="utf-8-sig"), cache_path
|
|
|
|
url = build_nsrdb_url(endpoint, api_key, email, point, name, attributes, interval)
|
|
csv_text = download_text(url, timeout, rate_limiter)
|
|
cache_path.write_text(csv_text, encoding="utf-8")
|
|
return csv_text, cache_path
|
|
|
|
|
|
def normalized_header(value: str) -> str:
|
|
"""Normalize a CSV header to make NSRDB spelling variations easier to match."""
|
|
return re.sub(r"[^a-z0-9]+", "", value.lower())
|
|
|
|
|
|
def find_column(header: list[str], candidates: set[str]) -> int | None:
|
|
"""Return the first header index whose normalized name matches a candidate."""
|
|
normalized_candidates = {normalized_header(candidate) for candidate in candidates}
|
|
for index, value in enumerate(header):
|
|
if normalized_header(value) in normalized_candidates:
|
|
return index
|
|
return None
|
|
|
|
|
|
def find_data_header(rows: list[list[str]]) -> int | None:
|
|
"""Return the NSRDB hourly data header row index."""
|
|
required = {"Year", "Month", "Day", "Hour", "Minute"}
|
|
for index, row in enumerate(rows):
|
|
if required.issubset(set(row)):
|
|
return index
|
|
return None
|
|
|
|
|
|
def parse_optional_float(row: list[str], index: int | None) -> float | None:
|
|
"""Parse an optional float field from a CSV row."""
|
|
if index is None or len(row) <= index:
|
|
return None
|
|
value = row[index].strip()
|
|
if not value:
|
|
return None
|
|
try:
|
|
parsed = float(value)
|
|
except ValueError:
|
|
return None
|
|
if not math.isfinite(parsed):
|
|
return None
|
|
return parsed
|
|
|
|
|
|
def parse_cloud_type(row: list[str], index: int | None) -> str:
|
|
"""Parse an optional cloud type code as a string."""
|
|
if index is None or len(row) <= index:
|
|
return ""
|
|
value = row[index].strip()
|
|
if not value:
|
|
return ""
|
|
try:
|
|
return str(int(float(value)))
|
|
except ValueError:
|
|
return value
|
|
|
|
|
|
def pct(count: int, total: int) -> float:
|
|
"""Return a percentage or NaN when there is no denominator."""
|
|
return count / total * 100 if total else math.nan
|
|
|
|
|
|
def fmt(value: float | None, places: int = 3) -> str:
|
|
"""Format an optional float for CSV output."""
|
|
if value is None or not math.isfinite(value):
|
|
return ""
|
|
return f"{value:.{places}f}"
|
|
|
|
|
|
def summarize_cloud_metrics(
|
|
point: dict[str, str],
|
|
csv_text: str,
|
|
source_file: Path,
|
|
source_label: str,
|
|
min_clearsky_ghi: float,
|
|
) -> dict[str, str]:
|
|
"""Convert hourly GHI/Clearsky GHI/Cloud Type rows into county metrics."""
|
|
rows = list(csv.reader(csv_text.splitlines()))
|
|
header_index = find_data_header(rows)
|
|
if header_index is None:
|
|
raise ValueError("Could not find the NSRDB data header row.")
|
|
|
|
header = rows[header_index]
|
|
ghi_index = find_column(header, {"GHI", "ghi"})
|
|
clearsky_ghi_index = find_column(
|
|
header,
|
|
{
|
|
"Clearsky GHI",
|
|
"Clear Sky GHI",
|
|
"Clear-sky GHI",
|
|
"clearsky_ghi",
|
|
"clear_sky_ghi",
|
|
},
|
|
)
|
|
cloud_type_index = find_column(header, {"Cloud Type", "cloud_type", "cloudtype"})
|
|
if ghi_index is None:
|
|
raise ValueError("Could not find a GHI column in the NSRDB response.")
|
|
if clearsky_ghi_index is None:
|
|
raise ValueError("Could not find a Clearsky GHI column in the NSRDB response.")
|
|
if cloud_type_index is None:
|
|
raise ValueError("Could not find a Cloud Type column in the NSRDB response.")
|
|
|
|
all_rows = 0
|
|
daylight_rows = 0
|
|
ratio_sum = 0.0
|
|
ghi_values: list[float] = []
|
|
clearsky_values: list[float] = []
|
|
daylight_cloud_type_counts: dict[str, int] = {}
|
|
|
|
for row in rows[header_index + 1 :]:
|
|
if not row:
|
|
continue
|
|
all_rows += 1
|
|
ghi = parse_optional_float(row, ghi_index)
|
|
clearsky_ghi = parse_optional_float(row, clearsky_ghi_index)
|
|
cloud_type = parse_cloud_type(row, cloud_type_index)
|
|
if ghi is None or clearsky_ghi is None:
|
|
continue
|
|
ghi_values.append(ghi)
|
|
clearsky_values.append(clearsky_ghi)
|
|
if clearsky_ghi < min_clearsky_ghi:
|
|
continue
|
|
|
|
ratio_sum += min(1.0, max(0.0, ghi / clearsky_ghi))
|
|
daylight_rows += 1
|
|
if cloud_type:
|
|
daylight_cloud_type_counts[cloud_type] = daylight_cloud_type_counts.get(cloud_type, 0) + 1
|
|
|
|
if daylight_rows == 0:
|
|
raise ValueError("No daylight rows with valid GHI and Clearsky GHI were found.")
|
|
|
|
avg_ratio = ratio_sum / daylight_rows
|
|
cloudiness_pct = 1 - avg_ratio
|
|
clear_or_probably_clear = daylight_cloud_type_counts.get("0", 0) + daylight_cloud_type_counts.get("1", 0)
|
|
cloudy_or_obscured = sum(
|
|
daylight_cloud_type_counts.get(code, 0)
|
|
for code in ["2", "3", "4", "5", "6", "7", "8", "9", "11", "12"]
|
|
)
|
|
water_cloud = (
|
|
daylight_cloud_type_counts.get("3", 0)
|
|
+ daylight_cloud_type_counts.get("4", 0)
|
|
+ daylight_cloud_type_counts.get("5", 0)
|
|
)
|
|
ice_cloud = (
|
|
daylight_cloud_type_counts.get("6", 0)
|
|
+ daylight_cloud_type_counts.get("8", 0)
|
|
+ daylight_cloud_type_counts.get("9", 0)
|
|
)
|
|
unknown_cloud_type = daylight_cloud_type_counts.get("10", 0) + daylight_cloud_type_counts.get("-15", 0)
|
|
|
|
return {
|
|
"county_fips": point["county_fips"],
|
|
"county_name": point["county_name"],
|
|
"state_fips": point["state_fips"],
|
|
"state_abbr": point["state_abbr"],
|
|
"lat": point["lat"],
|
|
"lon": point["lon"],
|
|
"cloudinessIndexPct": fmt(cloudiness_pct, 4),
|
|
"avgObservedToClearskyRatio": fmt(avg_ratio, 4),
|
|
"daylightRows": str(daylight_rows),
|
|
"allRows": str(all_rows),
|
|
"clearOrProbablyClearPct": fmt(pct(clear_or_probably_clear, daylight_rows), 2),
|
|
"cloudyOrObscuredPct": fmt(pct(cloudy_or_obscured, daylight_rows), 2),
|
|
"fogPct": fmt(pct(daylight_cloud_type_counts.get("2", 0), daylight_rows), 2),
|
|
"waterCloudPct": fmt(pct(water_cloud, daylight_rows), 2),
|
|
"iceCloudPct": fmt(pct(ice_cloud, daylight_rows), 2),
|
|
"cirrusPct": fmt(pct(daylight_cloud_type_counts.get("7", 0), daylight_rows), 2),
|
|
"unknownCloudTypePct": fmt(pct(unknown_cloud_type, daylight_rows), 2),
|
|
"ghi_min": fmt(min(ghi_values), 1) if ghi_values else "",
|
|
"ghi_max": fmt(max(ghi_values), 1) if ghi_values else "",
|
|
"clearsky_ghi_min": fmt(min(clearsky_values), 1) if clearsky_values else "",
|
|
"clearsky_ghi_max": fmt(max(clearsky_values), 1) if clearsky_values else "",
|
|
"source": f"{source_label} representative point",
|
|
"raw_csv": str(source_file),
|
|
}
|
|
|
|
|
|
def write_summary_csv(rows: list[dict[str, str]], output_csv: Path) -> None:
|
|
"""Write fetched county cloud summaries to CSV."""
|
|
output_csv.parent.mkdir(parents=True, exist_ok=True)
|
|
with output_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=CLOUD_SUMMARY_FIELDS)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def merged_summary_rows(
|
|
all_points: list[dict[str, str]],
|
|
existing_rows_by_fips: dict[str, dict[str, str]],
|
|
new_rows: list[dict[str, str]],
|
|
) -> list[dict[str, str]]:
|
|
"""Return existing and newly fetched rows in county point order."""
|
|
rows_by_fips = dict(existing_rows_by_fips)
|
|
for row in new_rows:
|
|
rows_by_fips[row["county_fips"]] = row
|
|
|
|
ordered_rows: list[dict[str, str]] = []
|
|
seen_fips: set[str] = set()
|
|
for point in all_points:
|
|
county_fips = point.get("county_fips", "")
|
|
row = rows_by_fips.get(county_fips)
|
|
if row is not None:
|
|
ordered_rows.append(row)
|
|
seen_fips.add(county_fips)
|
|
|
|
extra_rows = [
|
|
row
|
|
for county_fips, row in sorted(rows_by_fips.items())
|
|
if county_fips not in seen_fips
|
|
]
|
|
return ordered_rows + extra_rows
|
|
|
|
|
|
def write_error_csv(rows: list[dict[str, str]], error_csv: Path) -> None:
|
|
"""Write failed county cloud fetches to CSV."""
|
|
error_csv.parent.mkdir(parents=True, exist_ok=True)
|
|
with error_csv.open("w", newline="", encoding="utf-8") as handle:
|
|
writer = csv.DictWriter(
|
|
handle,
|
|
fieldnames=[
|
|
"county_fips",
|
|
"county_name",
|
|
"state_fips",
|
|
"state_abbr",
|
|
"lat",
|
|
"lon",
|
|
"error_type",
|
|
"status_code",
|
|
"request_url",
|
|
"response",
|
|
"message",
|
|
],
|
|
)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def build_error_row(point: dict[str, str], error: Exception) -> dict[str, str]:
|
|
"""Convert a failed county fetch into a structured CSV row."""
|
|
status_code = ""
|
|
request_url = ""
|
|
response = ""
|
|
|
|
if isinstance(error, NsrdDataRequestError):
|
|
status_code = str(error.status_code or "")
|
|
request_url = error.url or ""
|
|
response = error.response
|
|
|
|
return {
|
|
"county_fips": point["county_fips"],
|
|
"county_name": point["county_name"],
|
|
"state_fips": point["state_fips"],
|
|
"state_abbr": point["state_abbr"],
|
|
"lat": point["lat"],
|
|
"lon": point["lon"],
|
|
"error_type": type(error).__name__,
|
|
"status_code": status_code,
|
|
"request_url": request_url,
|
|
"response": redact_sensitive_text(response),
|
|
"message": redact_sensitive_text(str(error)),
|
|
}
|
|
|
|
|
|
def build_request_profiles(
|
|
args: argparse.Namespace,
|
|
point: dict[str, str],
|
|
previous_error_fips: set[str],
|
|
) -> list[dict[str, str]]:
|
|
"""Build the ordered list of NSRDB endpoints to try for a county point."""
|
|
goes_profile = {
|
|
"source_key": "goes-tmy",
|
|
"source_label": "NSRDB GOES TMY PSM v4",
|
|
"endpoint": args.endpoint,
|
|
"name": args.name,
|
|
}
|
|
polar_profile = {
|
|
"source_key": "polar-tmy",
|
|
"source_label": "NSRDB Polar TMY PSM v4",
|
|
"endpoint": args.polar_endpoint,
|
|
"name": args.polar_name,
|
|
}
|
|
|
|
if not args.polar_fallback or not is_polar_candidate(point, args.polar_min_latitude):
|
|
return [goes_profile]
|
|
if point["county_fips"] in previous_error_fips:
|
|
return [polar_profile, goes_profile]
|
|
return [goes_profile, polar_profile]
|
|
|
|
|
|
def should_try_next_profile(error: Exception, profile_index: int, profiles: list[dict[str, str]]) -> bool:
|
|
"""Return whether another configured endpoint should be tried after this error."""
|
|
if profile_index >= len(profiles) - 1:
|
|
return False
|
|
if not isinstance(error, NsrdDataRequestError):
|
|
return False
|
|
return error.status_code == 400 and "No data available at the provided location" in error.response
|
|
|
|
|
|
def fetch_county_summary(
|
|
args: argparse.Namespace,
|
|
point: dict[str, str],
|
|
previous_error_fips: set[str],
|
|
api_key: str,
|
|
email: str,
|
|
rate_limiter: RequestRateLimiter,
|
|
) -> tuple[dict[str, str] | None, dict[str, str] | None, list[str]]:
|
|
"""Fetch and summarize one county point."""
|
|
profiles = build_request_profiles(args, point, previous_error_fips)
|
|
final_error: Exception | None = None
|
|
messages: list[str] = []
|
|
|
|
for profile_index, profile in enumerate(profiles):
|
|
try:
|
|
messages.append(f"Trying {profile['source_label']} ({profile['name']})")
|
|
csv_text, source_file = read_or_download_county_csv(
|
|
point=point,
|
|
endpoint=profile["endpoint"],
|
|
api_key=api_key,
|
|
email=email,
|
|
name=profile["name"],
|
|
source_key=profile["source_key"],
|
|
attributes=args.attributes,
|
|
interval=args.interval,
|
|
cache_dir=args.cache_dir,
|
|
timeout=args.timeout,
|
|
overwrite=args.overwrite,
|
|
rate_limiter=rate_limiter,
|
|
)
|
|
summary = summarize_cloud_metrics(
|
|
point=point,
|
|
csv_text=csv_text,
|
|
source_file=source_file,
|
|
source_label=profile["source_label"],
|
|
min_clearsky_ghi=args.min_clearsky_ghi,
|
|
)
|
|
return summary, None, messages
|
|
except (OSError, urllib.error.URLError, RuntimeError, ValueError) as error:
|
|
final_error = error
|
|
if should_try_next_profile(error, profile_index, profiles):
|
|
messages.append(f"{profile['source_label']} had no data; trying fallback.")
|
|
continue
|
|
break
|
|
|
|
if final_error is None:
|
|
final_error = RuntimeError("No NSRDB profiles were available for this county.")
|
|
return None, build_error_row(point, final_error), messages
|
|
|
|
|
|
def log_county_result(
|
|
index: int,
|
|
total: int,
|
|
point: dict[str, str],
|
|
summary: dict[str, str] | None,
|
|
error: dict[str, str] | None,
|
|
messages: list[str],
|
|
) -> None:
|
|
"""Print progress for one completed county."""
|
|
label = f"{point['county_fips']} {point['county_name']}, {point['state_abbr']}"
|
|
status = "Fetched" if summary is not None else "Skipped"
|
|
print(f"[{index}/{total}] {status} {label}")
|
|
for message in messages:
|
|
print(f" {message}")
|
|
if summary is not None:
|
|
print(
|
|
" "
|
|
f"cloudiness={summary['cloudinessIndexPct']}, "
|
|
f"clear_ratio={summary['avgObservedToClearskyRatio']}, "
|
|
f"daylight_rows={summary['daylightRows']}"
|
|
)
|
|
if error is not None:
|
|
print(f" Error: {error['message']}")
|
|
|
|
|
|
def fetch_batch(
|
|
args: argparse.Namespace,
|
|
all_points: list[dict[str, str]],
|
|
existing_rows_by_fips: dict[str, dict[str, str]],
|
|
) -> list[dict[str, str]]:
|
|
"""Fetch and summarize the requested county point batch."""
|
|
points = select_county_points(
|
|
rows=all_points,
|
|
start=args.start,
|
|
limit=args.limit,
|
|
completed_fips=set(existing_rows_by_fips),
|
|
overwrite=args.overwrite,
|
|
)
|
|
skipped_count = len(all_points[args.start :]) - len(
|
|
select_county_points(
|
|
rows=all_points,
|
|
start=args.start,
|
|
limit=None,
|
|
completed_fips=set(existing_rows_by_fips),
|
|
overwrite=args.overwrite,
|
|
)
|
|
)
|
|
if skipped_count and not args.overwrite:
|
|
print(f"Skipping {skipped_count} counties already present in {args.output_csv}.")
|
|
if not points:
|
|
write_error_csv([], args.error_csv)
|
|
print("No missing counties selected for fetch.")
|
|
return []
|
|
|
|
email = prompt_for_text("NLR/NREL API email: ", args.email)
|
|
api_key = prompt_for_secret("NLR/NREL API key: ", args.api_key)
|
|
|
|
previous_error_fips = read_previous_error_fips(args.error_csv)
|
|
rate_limiter = RequestRateLimiter(args.delay)
|
|
summaries_by_index: dict[int, dict[str, str]] = {}
|
|
errors_by_index: dict[int, dict[str, str]] = {}
|
|
total = len(points)
|
|
|
|
if args.workers == 1:
|
|
for index, point in enumerate(points, start=1):
|
|
summary, error, messages = fetch_county_summary(
|
|
args=args,
|
|
point=point,
|
|
previous_error_fips=previous_error_fips,
|
|
api_key=api_key,
|
|
email=email,
|
|
rate_limiter=rate_limiter,
|
|
)
|
|
if summary is not None:
|
|
summaries_by_index[index] = summary
|
|
if error is not None:
|
|
errors_by_index[index] = error
|
|
log_county_result(index, total, point, summary, error, messages)
|
|
else:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor:
|
|
future_to_context = {
|
|
executor.submit(
|
|
fetch_county_summary,
|
|
args,
|
|
point,
|
|
previous_error_fips,
|
|
api_key,
|
|
email,
|
|
rate_limiter,
|
|
): (index, point)
|
|
for index, point in enumerate(points, start=1)
|
|
}
|
|
|
|
for future in concurrent.futures.as_completed(future_to_context):
|
|
index, point = future_to_context[future]
|
|
summary, error, messages = future.result()
|
|
if summary is not None:
|
|
summaries_by_index[index] = summary
|
|
if error is not None:
|
|
errors_by_index[index] = error
|
|
log_county_result(index, total, point, summary, error, messages)
|
|
|
|
errors = [errors_by_index[index] for index in sorted(errors_by_index)]
|
|
write_error_csv(errors, args.error_csv)
|
|
return [summaries_by_index[index] for index in sorted(summaries_by_index)]
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
"""Parse command-line arguments."""
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--points-csv", type=Path, default=DEFAULT_POINTS_CSV)
|
|
parser.add_argument("--output-csv", type=Path, default=DEFAULT_OUTPUT_CSV)
|
|
parser.add_argument("--error-csv", type=Path, default=DEFAULT_ERROR_CSV)
|
|
parser.add_argument("--cache-dir", type=Path, default=DEFAULT_CACHE_DIR)
|
|
parser.add_argument("--endpoint", default=DEFAULT_ENDPOINT)
|
|
parser.add_argument("--polar-endpoint", default=DEFAULT_POLAR_ENDPOINT)
|
|
parser.add_argument("--name", default=DEFAULT_TMY_NAME, help="NSRDB GOES TMY name, such as tmy or tmy-2024.")
|
|
parser.add_argument("--polar-name", default=DEFAULT_POLAR_TMY_NAME, help="NSRDB Polar TMY name, usually tmy.")
|
|
parser.add_argument("--attributes", default=DEFAULT_ATTRIBUTES, help="Comma-delimited NSRDB attributes to request.")
|
|
parser.add_argument("--interval", type=int, default=DEFAULT_INTERVAL, help="NSRDB interval in minutes.")
|
|
parser.add_argument("--polar-min-latitude", type=float, default=DEFAULT_POLAR_MIN_LATITUDE)
|
|
parser.add_argument(
|
|
"--min-clearsky-ghi",
|
|
type=float,
|
|
default=DEFAULT_MIN_CLEARSKY_GHI,
|
|
help="Minimum Clearsky GHI W/m2 for daylight cloudiness ratio rows.",
|
|
)
|
|
parser.add_argument(
|
|
"--no-polar-fallback",
|
|
action="store_false",
|
|
dest="polar_fallback",
|
|
help="Disable retrying failed high-latitude GOES requests against the NSRDB Polar endpoint.",
|
|
)
|
|
parser.add_argument("--email", help="Email address registered with the NLR/NREL API.")
|
|
parser.add_argument("--api-key", help="API key. If omitted, the script prompts securely.")
|
|
parser.add_argument("--start", type=int, default=0, help="Zero-based row offset in the points CSV.")
|
|
parser.add_argument("--limit", type=int, default=10, help="Number of counties to fetch. Use --all for all counties.")
|
|
parser.add_argument("--all", action="store_true", help="Fetch all counties from --start onward.")
|
|
parser.add_argument("--delay", type=float, default=1.0, help="Minimum seconds between new API requests.")
|
|
parser.add_argument("--workers", type=int, default=4, help="Number of county jobs to run at once.")
|
|
parser.add_argument("--timeout", type=int, default=120, help="Request timeout in seconds.")
|
|
parser.add_argument("--overwrite", action="store_true", help="Re-download cached county CSV files.")
|
|
args = parser.parse_args()
|
|
if args.all:
|
|
args.limit = None
|
|
if args.delay < 0:
|
|
parser.error("--delay must be 0 or greater.")
|
|
if args.workers < 1:
|
|
parser.error("--workers must be 1 or greater.")
|
|
if args.interval <= 0:
|
|
parser.error("--interval must be greater than 0.")
|
|
if args.min_clearsky_ghi < 0:
|
|
parser.error("--min-clearsky-ghi must be 0 or greater.")
|
|
return args
|
|
|
|
|
|
def main() -> None:
|
|
"""Run the NSRDB representative-point cloud metric fetch."""
|
|
args = parse_args()
|
|
all_points = read_county_points(args.points_csv)
|
|
existing_rows_by_fips = read_existing_summary_rows(args.output_csv)
|
|
summaries = fetch_batch(args, all_points, existing_rows_by_fips)
|
|
output_rows = merged_summary_rows(all_points, existing_rows_by_fips, summaries)
|
|
write_summary_csv(output_rows, args.output_csv)
|
|
print(
|
|
f"Wrote {len(output_rows)} county cloud summaries to {args.output_csv} "
|
|
f"({len(summaries)} newly fetched)."
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|