chandlerswift/name-all-cities-by-population-quiz
1
0
Fork 0
Code Issues 3 Pull requests Packages Projects Releases Wiki Activity

Fetch lat/lon for cities for eventual map

Browse source
This commit is contained in:
Chandler Swift 2024-02-29 23:19:08 -06:00
parent 542c8131a1
commit b062accb68
Signed by: chandlerswift
GPG key ID: A851D929D52FB93F
1 changed files with 64 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
get_data.py
View file

@ -1,13 +1,47 @@
import requests import requests
import csv import csv
import json import json
import zipfile
import tempfile
import os
import subprocess
import io
# https://www2.census.gov/programs-surveys/popest/technical-documentation/file-layouts/2020-2022/SUB-EST2022.pdf # https://www2.census.gov/programs-surveys/popest/technical-documentation/file-layouts/2020-2022/SUB-EST2022.pdf
INCORPORATED_PLACE = "162" INCORPORATED_PLACE = "162"
# Get state FIPS/ANSI codes and other data
# from https://www.census.gov/library/reference/code-lists/ansi.html#states
print("Fetching states…", flush=True, end="")
res = requests.get("https://www2.census.gov/geo/docs/reference/codes2020/national_state2020.txt")
states = list(csv.DictReader(res.text.split('\n'), delimiter='|'))
# {'STATE': 'AL', 'STATEFP': '01', 'STATENS': '01779775', 'STATE_NAME': 'Alabama'}
print("done")
# Find geographic centers of cities
place_locations = {}
for state in states[:51]: # Just the 50 and DC, not Guam/American Samoa/PR/etc
with tempfile.TemporaryDirectory() as tmpdir:
print(f"Fetching data for {state['STATE_NAME']}…", flush=True, end="")
res = requests.get(f"https://www2.census.gov/geo/tiger/TIGER2020/PLACE/tl_2020_{state['STATEFP']}_place.zip")
print("processing…", end="", flush=True)
zipfile.ZipFile(io.BytesIO(res.content)).extractall(tmpdir)
shapefile_name = os.path.join(tmpdir, f"tl_2020_{state['STATEFP']}_place.shp")
geojson_file_name = os.path.join(tmpdir, "out.geojson")
subprocess.run(f"ogr2ogr -f GeoJSON {geojson_file_name} {shapefile_name}", shell=True)
with open(geojson_file_name) as f:
data = json.load(f)
for feature in data['features']:
# {"type": "Feature", "properties": {"STATEFP": "01", "PLACEFP": "02260", "PLACENS": "02405163", "GEOID": "0102260", "NAME": "Ardmore", "NAMELSAD": "Ardmore town", "LSAD": "43", "CLASSFP": "C1", "PCICBSA": "N", "PCINECTA": "N", "MTFCC": "G4110", "FUNCSTAT": "A", "ALAND": 5289895, "AWATER": 21830, "INTPTLAT": "+34.9878376", "INTPTLON": "-086.8290225"}, "geometry": {"type": "Polygon", "coordinates": [[[-86.856689, 34.992046], [-86.855354, 34.992044], [-86.855101, 34.99204]
state_place = (feature['properties']['STATEFP'], feature['properties']['PLACEFP'])
lat_lon = (float(feature['properties']['INTPTLAT']), float(feature['properties']['INTPTLON']))
place_locations[state_place] = lat_lon
print("done")
print("Fetching population data for all states…", flush=True, end="")
res = requests.get("https://www2.census.gov/programs-surveys/popest/datasets/2020-2022/cities/totals/sub-est2022.csv") res = requests.get("https://www2.census.gov/programs-surveys/popest/datasets/2020-2022/cities/totals/sub-est2022.csv")
res.raise_for_status() res.raise_for_status()
print("processing…", flush=True, end="")
cities_by_state = {} cities_by_state = {}
for line in csv.DictReader(res.content.decode('utf-8-sig').split('\n')): for line in csv.DictReader(res.content.decode('utf-8-sig').split('\n')):
if line['SUMLEV'] != INCORPORATED_PLACE: if line['SUMLEV'] != INCORPORATED_PLACE:
@ -16,11 +50,36 @@ for line in csv.DictReader(res.content.decode('utf-8-sig').split('\n')):
if not line['STNAME'] in cities_by_state: if not line['STNAME'] in cities_by_state:
cities_by_state[line['STNAME']] = [] cities_by_state[line['STNAME']] = []
try:
loc = place_locations[(line['STATE'], line['PLACE'])]
except KeyError:
# TODO: why do these happen? Currently these:
# WARN: KeyError for ('17', '10373')
# WARN: KeyError for ('17', '31991')
# WARN: KeyError for ('27', '13708')
# WARN: KeyError for ('36', '75779')
# WARN: KeyError for ('40', '43725')
# WARN: KeyError for ('40', '49860')
# WARN: KeyError for ('48', '21031')
# WARN: KeyError for ('48', '23176')
# WARN: KeyError for ('48', '58502')
# WARN: KeyError for ('48', '73493')
# WARN: KeyError for ('55', '31525')
# WARN: KeyError for ('55', '82575')
# WARN: KeyError for ('55', '84275')
# Well, we'll just shove 'em on Null Island, I guess
loc = [0,0]
print("WARN: KeyError for", (line['STATE'], line['PLACE']))
import time
time.sleep(0.1)
cities_by_state[line['STNAME']].append({ cities_by_state[line['STNAME']].append({
"name": " ".join(line['NAME'].split(" ")[:-1]), # Remove "city" or "town" from the end "name": " ".join(line['NAME'].split(" ")[:-1]), # Remove "city" or "town" from the end
"pop": int(line['POPESTIMATE2022']), "pop": int(line['POPESTIMATE2022']),
"location": loc,
}) })
print("done")
print("Writing data to disk…", flush=True, end="")
for state, cities in cities_by_state.items(): for state, cities in cities_by_state.items():
cities.sort(key=lambda i: i["pop"], reverse=True) cities.sort(key=lambda i: i["pop"], reverse=True)
@ -29,11 +88,10 @@ for state, cities in cities_by_state.items():
with open(f"data/states.json", 'w') as f: with open(f"data/states.json", 'w') as f:
f.write(json.dumps(list(cities_by_state.keys()))) f.write(json.dumps(list(cities_by_state.keys())))
print("done")
# ----- MAP ----- # ----- MAP -----
print("Fetching state outlines…", flush=True, end="")
import subprocess
CMD=""" CMD="""
curl --silent --remote-name https://www2.census.gov/geo/tiger/GENZ2022/shp/cb_2022_us_state_20m.zip curl --silent --remote-name https://www2.census.gov/geo/tiger/GENZ2022/shp/cb_2022_us_state_20m.zip
unzip -q -o cb_2022_us_state_20m.zip unzip -q -o cb_2022_us_state_20m.zip
@ -43,3 +101,4 @@ rm cb_2022_us_state_20m.*
""" """
subprocess.run(CMD, shell=True) subprocess.run(CMD, shell=True)
print("done")