diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c88c98 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +lakes.json diff --git a/Makefile b/Makefile index c1512a8..f22929c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: deploy deploy: fetch_data - rsync -av ./ zirconium:/var/www/home.chandlerswift.com/cities/ + rsync -av ./ zirconium:/var/www/home.chandlerswift.com/lakes/ .PHONY: fetch_data fetch_data: get_data.py diff --git a/data/.gitignore b/data/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/data/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/get_data.py b/get_data.py index 8a2280c..831b4a6 100644 --- a/get_data.py +++ b/get_data.py @@ -1,5 +1,4 @@ import requests -import csv import json import zipfile import tempfile @@ -7,98 +6,41 @@ import os import subprocess import io -# https://www2.census.gov/programs-surveys/popest/technical-documentation/file-layouts/2020-2022/SUB-EST2022.pdf -INCORPORATED_PLACE = "162" +with tempfile.TemporaryDirectory() as tmpdir: + print(f"Fetching lake data", flush=True) + res = requests.get(f"https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_dnr/water_dnr_hydrography/shp_water_dnr_hydrography.zip") + print("extracting zip file", flush=True) + zipfile.ZipFile(io.BytesIO(res.content)).extractall(tmpdir) + shapefile_name = os.path.join(tmpdir, f"dnr_hydro_features_all.shp") + geojson_file_name = os.path.join(tmpdir, "out.geojson") + print("converting to geojson", flush=True) + subprocess.run(f"ogr2ogr -f GeoJSON {geojson_file_name} {shapefile_name}", shell=True) + print("loading json", flush=True) + with open(geojson_file_name) as f: + data = json.load(f) -# Get state FIPS/ANSI codes and other data -# from https://www.census.gov/library/reference/code-lists/ansi.html#states -print("Fetching states…", flush=True, end="") -res = requests.get("https://www2.census.gov/geo/docs/reference/codes2020/national_state2020.txt") -states = list(csv.DictReader(res.text.split('\n'), delimiter='|')) -# {'STATE': 'AL', 'STATEFP': '01', 'STATENS': '01779775', 'STATE_NAME': 'Alabama'} -print("done") +print("processing", flush=True) +lakes_by_name = {} # {"Marion": {"centers": [[lon, lat], ...], "area": area_in_acres}, ...} -# Find geographic centers of cities -place_locations = {} -for state in states[:51]: # Just the 50 and DC, not Guam/American Samoa/PR/etc - with tempfile.TemporaryDirectory() as tmpdir: - print(f"Fetching data for {state['STATE_NAME']}…", flush=True, end="") - res = requests.get(f"https://www2.census.gov/geo/tiger/TIGER2020/PLACE/tl_2020_{state['STATEFP']}_place.zip") - print("processing…", end="", flush=True) - zipfile.ZipFile(io.BytesIO(res.content)).extractall(tmpdir) - shapefile_name = os.path.join(tmpdir, f"tl_2020_{state['STATEFP']}_place.shp") - geojson_file_name = os.path.join(tmpdir, "out.geojson") - subprocess.run(f"ogr2ogr -f GeoJSON {geojson_file_name} {shapefile_name}", shell=True) - with open(geojson_file_name) as f: - data = json.load(f) - for feature in data['features']: - # {"type": "Feature", "properties": {"STATEFP": "01", "PLACEFP": "02260", "PLACENS": "02405163", "GEOID": "0102260", "NAME": "Ardmore", "NAMELSAD": "Ardmore town", "LSAD": "43", "CLASSFP": "C1", "PCICBSA": "N", "PCINECTA": "N", "MTFCC": "G4110", "FUNCSTAT": "A", "ALAND": 5289895, "AWATER": 21830, "INTPTLAT": "+34.9878376", "INTPTLON": "-086.8290225"}, "geometry": {"type": "Polygon", "coordinates": [[[-86.856689, 34.992046], [-86.855354, 34.992044], [-86.855101, 34.99204] - state_place = (feature['properties']['STATEFP'], feature['properties']['PLACEFP']) - lon_lat = (float(feature['properties']['INTPTLON']), float(feature['properties']['INTPTLAT'])) - place_locations[state_place] = lon_lat - print("done") - -print("Fetching population data for all states…", flush=True, end="") -res = requests.get("https://www2.census.gov/programs-surveys/popest/datasets/2020-2022/cities/totals/sub-est2022.csv") -res.raise_for_status() -print("processing…", flush=True, end="") -cities_by_state = {} -for line in csv.DictReader(res.content.decode('utf-8-sig').split('\n')): - if line['SUMLEV'] != INCORPORATED_PLACE: +for feature in data['features']: + if feature['properties']['sub_flag'] == 'Y': continue + name = feature['properties']['map_label'] # or pw_basin_n or pw_parent_ or...?? + if not name: # many lakes with null name + continue + if name == "Unnamed": + continue + if name not in lakes_by_name: + lakes_by_name[name] = {"centers": [], "area": 0} + lakes_by_name[name]["centers"].append([feature['properties']['INSIDE_X'], feature['properties']['INSIDE_Y']]) + lakes_by_name[name]["area"] += feature['properties']['acres'] - if not line['STNAME'] in cities_by_state: - cities_by_state[line['STNAME']] = [] +lakes = [] +for name, lake in lakes_by_name.items(): + lake["name"] = name + lakes.append(lake) - try: - loc = place_locations[(line['STATE'], line['PLACE'])] - except KeyError: - # TODO: why do these happen? Currently these: - # WARN: KeyError for ('17', '10373') - # WARN: KeyError for ('17', '31991') - # WARN: KeyError for ('27', '13708') - # WARN: KeyError for ('36', '75779') - # WARN: KeyError for ('40', '43725') - # WARN: KeyError for ('40', '49860') - # WARN: KeyError for ('48', '21031') - # WARN: KeyError for ('48', '23176') - # WARN: KeyError for ('48', '58502') - # WARN: KeyError for ('48', '73493') - # WARN: KeyError for ('55', '31525') - # WARN: KeyError for ('55', '82575') - # WARN: KeyError for ('55', '84275') - # Well, we'll just shove 'em on Null Island, I guess - loc = [0,0] - print("WARN: KeyError for", (line['STATE'], line['PLACE'])) - import time - time.sleep(0.1) - cities_by_state[line['STNAME']].append({ - "name": " ".join(line['NAME'].split(" ")[:-1]), # Remove "city" or "town" from the end - "pop": int(line['POPESTIMATE2022']), - "location": loc, - }) -print("done") +lakes.sort(key=lambda lake: lake['area'], reverse=True) -print("Writing data to disk…", flush=True, end="") -for state, cities in cities_by_state.items(): - cities.sort(key=lambda i: i["pop"], reverse=True) - - with open(f"data/{state}.json", 'w') as f: - f.write(json.dumps(cities)) - -with open(f"data/states.json", 'w') as f: - f.write(json.dumps(list(cities_by_state.keys()))) -print("done") - -# ----- MAP ----- -print("Fetching state outlines…", flush=True, end="") -CMD=""" -curl --silent --remote-name https://www2.census.gov/geo/tiger/GENZ2022/shp/cb_2022_us_state_20m.zip -unzip -q -o cb_2022_us_state_20m.zip -ogr2ogr -f GeoJSON data/states.geojson cb_2022_us_state_20m.shp -sed -i '/^"crs":/d' data/states.geojson -rm cb_2022_us_state_20m.* -""" - -subprocess.run(CMD, shell=True) -print("done") +with open(f"lakes.json", 'w') as f: + f.write(json.dumps(lakes)) diff --git a/index.html b/index.html index 00a6a99..ad7609b 100644 --- a/index.html +++ b/index.html @@ -3,35 +3,18 @@ - Name All the Cities + Name All MN's Lakes
-

Name All the Cities

-
- -
-
- - -
-
- - -
- -
-
-
+

Name All MN's Lakes

+
- +
- +
@@ -39,13 +22,12 @@ {{ message }}
-

{{ state_name }} cities

+ v-for="(required_lakes, name) in achievements" + :class="{ 'badge': true, 'me-1': true, 'text-bg-secondary': !required_lakes(lakes).every(c => c.guessed), 'text-bg-warning': required_lakes(lakes).every(c => c.guessed) }"> {{ name }} - ({{ required_cities(state_cities).filter(c => c.guessed).length }}/{{ required_cities(state_cities).length }}) + ({{ required_lakes(lakes).filter(c => c.guessed).length }}/{{ required_lakes(lakes).length }})
@@ -53,15 +35,17 @@ - + + -
Rank NamePopulationCountTotal Area