Knou/Climate-Mood-Analysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial Climate Mood Analysis project

Browse Source
This commit is contained in:
Justin Fisher
2026-06-11 14:50:33 -04:00
commit 18e099ce84
34 changed files with 18807 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_nsrdb_polygon_request_state_machine.py
+355
View File
@@ -0,0 +1,355 @@
from __future__ import annotations
import sys
import unittest
import csv
from datetime import datetime, timezone
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import call, patch
SCRIPTS_DIR = Path(__file__).resolve().parents[1] / "scripts"
sys.path.insert(0, str(SCRIPTS_DIR))
from request_nsrdb_county_polygon_archives import ( # noqa: E402
ArchiveQueueMonitor,
CountyRequestEvent,
CountyRequestState,
CountyRequestStateMachine,
EXCEPTIONAL_POLYGON_WAIT,
InvalidCountyRequestTransition,
LARGE_POLYGON_WAIT,
LocalQueueCapacityError,
MEDIUM_POLYGON_WAIT,
SMALL_POLYGON_WAIT,
)
class CountyRequestStateMachineTests(unittest.TestCase):
def test_successful_request_lifecycle(self) -> None:
machine = CountyRequestStateMachine("01001 Autauga County, AL")
machine.transition(CountyRequestEvent.START)
machine.transition(CountyRequestEvent.REQUEST_SUCCEEDED)
self.assertEqual(machine.state, CountyRequestState.REQUESTED)
self.assertEqual(
machine.history,
[
CountyRequestState.PENDING,
CountyRequestState.SUBMITTING,
CountyRequestState.REQUESTED,
],
)
def test_retry_returns_to_submitting(self) -> None:
machine = CountyRequestStateMachine("01001 Autauga County, AL")
machine.transition(CountyRequestEvent.START)
machine.transition(CountyRequestEvent.RETRY_REQUIRED)
machine.transition(CountyRequestEvent.RETRY_STARTED)
self.assertEqual(machine.state, CountyRequestState.SUBMITTING)
def test_existing_request_is_skipped(self) -> None:
machine = CountyRequestStateMachine("01001 Autauga County, AL")
machine.transition(CountyRequestEvent.EXISTING_REQUEST_FOUND)
self.assertEqual(machine.state, CountyRequestState.SKIPPED)
def test_invalid_transition_is_rejected(self) -> None:
machine = CountyRequestStateMachine("01001 Autauga County, AL")
with self.assertRaisesRegex(
InvalidCountyRequestTransition,
"request_succeeded while pending",
):
machine.transition(CountyRequestEvent.REQUEST_SUCCEEDED)
class ArchiveQueueMonitorTests(unittest.TestCase):
def make_monitor(self) -> ArchiveQueueMonitor:
temp_dir = TemporaryDirectory()
self.addCleanup(temp_dir.cleanup)
return ArchiveQueueMonitor(Path(temp_dir.name), timeout=1, workers=1)
def test_dynamic_wait_uses_largest_tracked_polygon(self) -> None:
monitor = self.make_monitor()
for site_count, expected_wait in (
(110, SMALL_POLYGON_WAIT),
(111, MEDIUM_POLYGON_WAIT),
(250, MEDIUM_POLYGON_WAIT),
(251, LARGE_POLYGON_WAIT),
(550, LARGE_POLYGON_WAIT),
(551, EXCEPTIONAL_POLYGON_WAIT),
):
monitor.tracked_site_counts = {"https://example.com/job.zip": site_count}
self.assertEqual(monitor.dynamic_wait_seconds(), expected_wait)
def test_downloaded_archive_is_not_reloaded_as_pending(self) -> None:
temp_dir = TemporaryDirectory()
self.addCleanup(temp_dir.cleanup)
manifest_dir = Path(temp_dir.name)
response_dir = manifest_dir / "polygon_cloud_request_responses"
archive_dir = manifest_dir / "polygon_cloud_archives"
response_dir.mkdir()
archive_dir.mkdir()
response_path = response_dir / "01001_goes-tmy_tmy-2024_cloud_response.json"
archive_path = archive_dir / "01001_goes-tmy_tmy-2024_cloud.zip"
response_path.write_text("{}", encoding="utf-8")
archive_path.write_bytes(b"downloaded")
manifest_path = manifest_dir / "county_polygon_cloud_request_manifest.csv"
with manifest_path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(
handle,
fieldnames=[
"county_fips",
"site_count",
"download_url",
"response_json",
"submitted_at_utc",
],
)
writer.writeheader()
writer.writerow(
{
"county_fips": "01001",
"site_count": "100",
"download_url": "https://example.amazonaws.com/job.zip",
"response_json": str(response_path),
"submitted_at_utc": datetime.now(timezone.utc).isoformat(),
}
)
monitor = ArchiveQueueMonitor(manifest_dir, timeout=1, workers=1)
with patch(
"request_nsrdb_county_polygon_archives.archive_download_status",
return_value="pending",
) as status_mock:
counts = monitor.summarize()
self.assertEqual(counts["pending"], 0)
self.assertEqual(counts["total"], 0)
status_mock.assert_not_called()
def test_historical_s3_403_does_not_consume_a_queue_slot(self) -> None:
monitor = self.make_monitor()
url = "https://example.amazonaws.com/historical.zip"
monitor.jobs[url] = "01001"
with patch(
"request_nsrdb_county_polygon_archives.archive_download_status",
return_value="pending",
):
counts = monitor.summarize()
self.assertEqual(counts["pending"], 0)
self.assertEqual(counts["ambiguous_403"], 1)
def test_current_process_s3_403_remains_pending(self) -> None:
monitor = self.make_monitor()
url = "https://example.amazonaws.com/current.zip"
monitor.record_submission(
{
"county_fips": "01001",
"tile_id": "",
"site_count": "100",
"download_url": url,
}
)
with patch(
"request_nsrdb_county_polygon_archives.archive_download_status",
return_value="pending",
):
counts = monitor.summarize()
self.assertEqual(counts["pending"], 1)
self.assertEqual(counts["ambiguous_403"], 0)
def test_site_count_expires_after_recent_download_window(self) -> None:
monitor = self.make_monitor()
url = "https://example.com/job.zip"
monitor.jobs[url] = "01001"
monitor.pending_urls.add(url)
monitor.tracked_site_counts[url] = 551
with (
patch(
"request_nsrdb_county_polygon_archives.archive_download_status",
return_value="ready",
),
patch(
"request_nsrdb_county_polygon_archives.time.monotonic",
side_effect=[100.0, 100.0, 401.0],
),
):
first_counts = monitor.summarize()
second_counts = monitor.summarize()
self.assertEqual(first_counts["recently_downloaded"], 1)
self.assertEqual(second_counts["recently_downloaded"], 0)
self.assertNotIn(url, monitor.tracked_site_counts)
def test_fifteen_visible_slots_bypass_pacing_until_queue_is_full(self) -> None:
monitor = self.make_monitor()
monitor.paced_submissions = 1
counts_with_fifteen_slots = {
"pending": 5,
"recently_downloaded": 2,
"unobserved": 0,
"unknown": 0,
"ready": 0,
"total": 7,
}
counts_with_fourteen_slots = {
**counts_with_fifteen_slots,
"pending": 6,
}
full_queue_counts = {
**counts_with_fifteen_slots,
"pending": 20,
}
with (
patch.object(
monitor,
"summarize",
side_effect=[
counts_with_fifteen_slots,
counts_with_fourteen_slots,
full_queue_counts,
],
),
patch(
"request_nsrdb_county_polygon_archives.wait_for_queue",
) as wait_for_queue_mock,
):
monitor.wait_for_capacity("first", retries=0)
monitor.wait_for_capacity("second", retries=0)
with self.assertRaisesRegex(LocalQueueCapacityError, "20 possible"):
monitor.wait_for_capacity("third", retries=0)
wait_for_queue_mock.assert_not_called()
self.assertFalse(monitor.filling_open_queue)
def test_fourteen_visible_slots_still_use_recent_download_pacing(self) -> None:
monitor = self.make_monitor()
monitor.paced_submissions = 1
counts = {
"pending": 6,
"recently_downloaded": 2,
"unobserved": 0,
"unknown": 0,
"ready": 0,
"total": 8,
}
with (
patch.object(monitor, "summarize", return_value=counts),
patch(
"request_nsrdb_county_polygon_archives.wait_for_queue",
) as wait_for_queue_mock,
):
monitor.wait_for_capacity("next", retries=0)
wait_for_queue_mock.assert_called_once_with(
SMALL_POLYGON_WAIT,
"Pacing wait complete; refreshing archive status.",
)
def test_six_minute_wait_checks_queue_after_three_minutes(self) -> None:
monitor = self.make_monitor()
counts = {
"pending": 6,
"recently_downloaded": 0,
"unobserved": 0,
"unknown": 0,
"ready": 0,
"total": 6,
}
with (
patch.object(monitor, "summarize", return_value=counts) as summarize_mock,
patch(
"request_nsrdb_county_polygon_archives.time.sleep",
) as sleep_mock,
):
entered_fill_mode = monitor.wait_for_dynamic_queue(
LARGE_POLYGON_WAIT,
"Queue wait complete.",
)
self.assertFalse(entered_fill_mode)
self.assertEqual(sleep_mock.call_args_list, [call(180.0)] * 2)
summarize_mock.assert_called_once_with()
def test_six_minute_wait_enters_fill_mode_at_checkpoint(self) -> None:
monitor = self.make_monitor()
counts = {
"pending": 5,
"recently_downloaded": 0,
"unobserved": 0,
"unknown": 0,
"ready": 0,
"total": 5,
}
with (
patch.object(monitor, "summarize", return_value=counts),
patch(
"request_nsrdb_county_polygon_archives.time.sleep",
) as sleep_mock,
):
entered_fill_mode = monitor.wait_for_dynamic_queue(
LARGE_POLYGON_WAIT,
"Queue wait complete.",
)
self.assertTrue(entered_fill_mode)
self.assertTrue(monitor.filling_open_queue)
sleep_mock.assert_called_once_with(180.0)
def test_fifteen_minute_wait_checks_at_five_and_ten_minutes(self) -> None:
monitor = self.make_monitor()
counts_with_fourteen_slots = {
"pending": 6,
"recently_downloaded": 0,
"unobserved": 0,
"unknown": 0,
"ready": 0,
"total": 6,
}
counts_with_fifteen_slots = {
**counts_with_fourteen_slots,
"pending": 5,
}
with (
patch.object(
monitor,
"summarize",
side_effect=[counts_with_fourteen_slots, counts_with_fifteen_slots],
) as summarize_mock,
patch(
"request_nsrdb_county_polygon_archives.time.sleep",
) as sleep_mock,
):
entered_fill_mode = monitor.wait_for_dynamic_queue(
EXCEPTIONAL_POLYGON_WAIT,
"Queue wait complete.",
)
self.assertTrue(entered_fill_mode)
self.assertTrue(monitor.filling_open_queue)
self.assertEqual(sleep_mock.call_args_list, [call(300.0)] * 2)
self.assertEqual(summarize_mock.call_count, 2)
if __name__ == "__main__":
unittest.main()