Switch to lake guessing

main
Chandler Swift 2024-07-06 15:25:57 -05:00
parent 4c7c98bba0
commit b428ecf7b6
Signed by: chandlerswift
GPG Key ID: A851D929D52FB93F
5 changed files with 103 additions and 261 deletions

1
.gitignore vendored 100644
View File

@ -0,0 +1 @@
lakes.json

View File

@ -1,6 +1,6 @@
.PHONY: deploy .PHONY: deploy
deploy: fetch_data deploy: fetch_data
rsync -av ./ zirconium:/var/www/home.chandlerswift.com/cities/ rsync -av ./ zirconium:/var/www/home.chandlerswift.com/lakes/
.PHONY: fetch_data .PHONY: fetch_data
fetch_data: get_data.py fetch_data: get_data.py

2
data/.gitignore vendored
View File

@ -1,2 +0,0 @@
*
!.gitignore

View File

@ -1,5 +1,4 @@
import requests import requests
import csv
import json import json
import zipfile import zipfile
import tempfile import tempfile
@ -7,98 +6,41 @@ import os
import subprocess import subprocess
import io import io
# https://www2.census.gov/programs-surveys/popest/technical-documentation/file-layouts/2020-2022/SUB-EST2022.pdf with tempfile.TemporaryDirectory() as tmpdir:
INCORPORATED_PLACE = "162" print(f"Fetching lake data", flush=True)
res = requests.get(f"https://resources.gisdata.mn.gov/pub/gdrs/data/pub/us_mn_state_dnr/water_dnr_hydrography/shp_water_dnr_hydrography.zip")
print("extracting zip file", flush=True)
zipfile.ZipFile(io.BytesIO(res.content)).extractall(tmpdir)
shapefile_name = os.path.join(tmpdir, f"dnr_hydro_features_all.shp")
geojson_file_name = os.path.join(tmpdir, "out.geojson")
print("converting to geojson", flush=True)
subprocess.run(f"ogr2ogr -f GeoJSON {geojson_file_name} {shapefile_name}", shell=True)
print("loading json", flush=True)
with open(geojson_file_name) as f:
data = json.load(f)
# Get state FIPS/ANSI codes and other data print("processing", flush=True)
# from https://www.census.gov/library/reference/code-lists/ansi.html#states lakes_by_name = {} # {"Marion": {"centers": [[lon, lat], ...], "area": area_in_acres}, ...}
print("Fetching states…", flush=True, end="")
res = requests.get("https://www2.census.gov/geo/docs/reference/codes2020/national_state2020.txt")
states = list(csv.DictReader(res.text.split('\n'), delimiter='|'))
# {'STATE': 'AL', 'STATEFP': '01', 'STATENS': '01779775', 'STATE_NAME': 'Alabama'}
print("done")
# Find geographic centers of cities for feature in data['features']:
place_locations = {} if feature['properties']['sub_flag'] == 'Y':
for state in states[:51]: # Just the 50 and DC, not Guam/American Samoa/PR/etc
with tempfile.TemporaryDirectory() as tmpdir:
print(f"Fetching data for {state['STATE_NAME']}", flush=True, end="")
res = requests.get(f"https://www2.census.gov/geo/tiger/TIGER2020/PLACE/tl_2020_{state['STATEFP']}_place.zip")
print("processing…", end="", flush=True)
zipfile.ZipFile(io.BytesIO(res.content)).extractall(tmpdir)
shapefile_name = os.path.join(tmpdir, f"tl_2020_{state['STATEFP']}_place.shp")
geojson_file_name = os.path.join(tmpdir, "out.geojson")
subprocess.run(f"ogr2ogr -f GeoJSON {geojson_file_name} {shapefile_name}", shell=True)
with open(geojson_file_name) as f:
data = json.load(f)
for feature in data['features']:
# {"type": "Feature", "properties": {"STATEFP": "01", "PLACEFP": "02260", "PLACENS": "02405163", "GEOID": "0102260", "NAME": "Ardmore", "NAMELSAD": "Ardmore town", "LSAD": "43", "CLASSFP": "C1", "PCICBSA": "N", "PCINECTA": "N", "MTFCC": "G4110", "FUNCSTAT": "A", "ALAND": 5289895, "AWATER": 21830, "INTPTLAT": "+34.9878376", "INTPTLON": "-086.8290225"}, "geometry": {"type": "Polygon", "coordinates": [[[-86.856689, 34.992046], [-86.855354, 34.992044], [-86.855101, 34.99204]
state_place = (feature['properties']['STATEFP'], feature['properties']['PLACEFP'])
lon_lat = (float(feature['properties']['INTPTLON']), float(feature['properties']['INTPTLAT']))
place_locations[state_place] = lon_lat
print("done")
print("Fetching population data for all states…", flush=True, end="")
res = requests.get("https://www2.census.gov/programs-surveys/popest/datasets/2020-2022/cities/totals/sub-est2022.csv")
res.raise_for_status()
print("processing…", flush=True, end="")
cities_by_state = {}
for line in csv.DictReader(res.content.decode('utf-8-sig').split('\n')):
if line['SUMLEV'] != INCORPORATED_PLACE:
continue continue
name = feature['properties']['map_label'] # or pw_basin_n or pw_parent_ or...??
if not name: # many lakes with null name
continue
if name == "Unnamed":
continue
if name not in lakes_by_name:
lakes_by_name[name] = {"centers": [], "area": 0}
lakes_by_name[name]["centers"].append([feature['properties']['INSIDE_X'], feature['properties']['INSIDE_Y']])
lakes_by_name[name]["area"] += feature['properties']['acres']
if not line['STNAME'] in cities_by_state: lakes = []
cities_by_state[line['STNAME']] = [] for name, lake in lakes_by_name.items():
lake["name"] = name
lakes.append(lake)
try: lakes.sort(key=lambda lake: lake['area'], reverse=True)
loc = place_locations[(line['STATE'], line['PLACE'])]
except KeyError:
# TODO: why do these happen? Currently these:
# WARN: KeyError for ('17', '10373')
# WARN: KeyError for ('17', '31991')
# WARN: KeyError for ('27', '13708')
# WARN: KeyError for ('36', '75779')
# WARN: KeyError for ('40', '43725')
# WARN: KeyError for ('40', '49860')
# WARN: KeyError for ('48', '21031')
# WARN: KeyError for ('48', '23176')
# WARN: KeyError for ('48', '58502')
# WARN: KeyError for ('48', '73493')
# WARN: KeyError for ('55', '31525')
# WARN: KeyError for ('55', '82575')
# WARN: KeyError for ('55', '84275')
# Well, we'll just shove 'em on Null Island, I guess
loc = [0,0]
print("WARN: KeyError for", (line['STATE'], line['PLACE']))
import time
time.sleep(0.1)
cities_by_state[line['STNAME']].append({
"name": " ".join(line['NAME'].split(" ")[:-1]), # Remove "city" or "town" from the end
"pop": int(line['POPESTIMATE2022']),
"location": loc,
})
print("done")
print("Writing data to disk…", flush=True, end="") with open(f"lakes.json", 'w') as f:
for state, cities in cities_by_state.items(): f.write(json.dumps(lakes))
cities.sort(key=lambda i: i["pop"], reverse=True)
with open(f"data/{state}.json", 'w') as f:
f.write(json.dumps(cities))
with open(f"data/states.json", 'w') as f:
f.write(json.dumps(list(cities_by_state.keys())))
print("done")
# ----- MAP -----
print("Fetching state outlines…", flush=True, end="")
CMD="""
curl --silent --remote-name https://www2.census.gov/geo/tiger/GENZ2022/shp/cb_2022_us_state_20m.zip
unzip -q -o cb_2022_us_state_20m.zip
ogr2ogr -f GeoJSON data/states.geojson cb_2022_us_state_20m.shp
sed -i '/^"crs":/d' data/states.geojson
rm cb_2022_us_state_20m.*
"""
subprocess.run(CMD, shell=True)
print("done")

View File

@ -3,35 +3,18 @@
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Name All the Cities</title> <title>Name All MN's Lakes</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous"> <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
</head> </head>
<body class="h-100 d-flex flex-column"> <body class="h-100 d-flex flex-column">
<div v-scope @vue:mounted="mounted" class="container vh-100"> <div v-scope @vue:mounted="mounted" class="container vh-100">
<h1>Name All the Cities</h1> <h1>Name All MN's Lakes</h1>
<div class="mb-3"> <div class="row">
<label class="form-label">State:</label>
<div class="btn-toolbar">
<div class="input-group me-3 flex-grow-1">
<select class="form-select" v-model="state_name" :disabled="launched">
<option value="" selected>Select a state…</option>
<option v-for="state in states">{{ state }}</option>
</select>
<button class="btn btn-primary" type="button" @click="launch" :disabled="launched || !state_name">Play</button>
</div>
<div class="btn-group">
<button class="btn btn-secondary" type="button" @click="save" :disabled="!launched">Save</button>
<label for="restore-input" class="btn btn-success" :class="{disabled: launched}">Restore</label>
</div>
<input id="restore-input" type="file" @input="restore" style="display: none;"></button>
</div>
</div>
<div class="row" v-show="launched">
<div class="col-4"> <div class="col-4">
<div class="mb-3"> <div class="mb-3">
<label for="city-guess" class="form-label">City:</label> <label for="lake-guess" class="form-label">Lake:</label>
<div class="input-group"> <div class="input-group">
<input class="form-control" id="city-guess" v-model="city_guess" @keyup.enter="guess"> <input class="form-control" id="lake-guess" v-model="lake_guess" @keyup.enter="guess">
<button class="btn btn-primary" type="button" @click="guess">Guess</button> <button class="btn btn-primary" type="button" @click="guess">Guess</button>
</div> </div>
</div> </div>
@ -39,13 +22,12 @@
{{ message }} {{ message }}
</div> </div>
<h3>{{ state_name }} cities</h3>
<div> <div>
<span <span
v-for="(required_cities, name) in achievements" v-for="(required_lakes, name) in achievements"
:class="{ 'badge': true, 'me-1': true, 'text-bg-secondary': !required_cities(state_cities).every(c => c.guessed), 'text-bg-warning': required_cities(state_cities).every(c => c.guessed) }"> :class="{ 'badge': true, 'me-1': true, 'text-bg-secondary': !required_lakes(lakes).every(c => c.guessed), 'text-bg-warning': required_lakes(lakes).every(c => c.guessed) }">
{{ name }} {{ name }}
({{ required_cities(state_cities).filter(c => c.guessed).length }}/{{ required_cities(state_cities).length }}) ({{ required_lakes(lakes).filter(c => c.guessed).length }}/{{ required_lakes(lakes).length }})
</span> </span>
</div> </div>
<table class="table"> <table class="table">
@ -53,15 +35,17 @@
<tr> <tr>
<th>Rank</th> <th>Rank</th>
<th>Name</th> <th>Name</th>
<th>Population</th> <th>Count</th>
<th>Total Area</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
<template v-for="(city, rank) in state_cities"> <template v-for="(lake, rank) in lakes">
<tr v-show="city.guessed"> <tr v-show="lake.guessed">
<td>{{ rank + 1 }}</td> <td>{{ rank + 1 }}</td>
<td>{{ city.name }}</td> <td>{{ lake.name }}</td>
<td>{{ city.pop }}</td> <td>{{ lake.centers.length }}</td>
<td>{{ lake.area.toFixed(3) }}</td>
</tr> </tr>
</template> </template>
</tbody> </tbody>
@ -70,136 +54,68 @@
<div class="col-8"> <div class="col-8">
<canvas id="canvas" class="mx-3 my-auto" width="800" height="600"></canvas> <canvas id="canvas" class="mx-3 my-auto" width="800" height="600"></canvas>
<div> <div>
<input type="checkbox" id="show-unguessed-checkbox" v-model="show_unguessed_cities" @change="draw"> <input type="checkbox" id="show-unguessed-checkbox" v-model="show_unguessed_lakes" @change="draw">
<label for="show-unguessed-checkbox">Show unguessed cities</label> <label for="show-unguessed-checkbox">Show unguessed lakes</label>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
<footer class="footer mt-auto py-3 text-center text-muted"> <footer class="footer mt-auto py-3 text-center text-muted">
Created by <a href="https://chandlerswift.com">Chandler Swift</a> using 2020 US Census data | Created by <a href="https://chandlerswift.com">Chandler Swift</a> using data from MN DNR |
<a href="https://git.chandlerswift.com/chandlerswift/name-all-cities-by-population-quiz">Source</a> <a href="https://git.chandlerswift.com/chandlerswift/name-all-lakes-by-population-quiz">Source</a>
(<a href="https://git.chandlerswift.com/chandlerswift/name-all-cities-by-population-quiz/src/branch/main/LICENSE">GPL3</a>) (<a href="https://git.chandlerswift.com/chandlerswift/name-all-lakes-by-population-quiz/src/branch/main/LICENSE">GPL3</a>)
</footer> </footer>
<script type="module"> <script type="module">
import { createApp } from 'https://unpkg.com/petite-vue?module'; import { createApp } from 'https://unpkg.com/petite-vue?module';
const response = await fetch(`lakes.json`);
const lakes = await response.json();
createApp({ createApp({
launched: false, lakes: lakes,
shapes_request: null, bounds: null,
states: null, lake_guess: "",
state_name: null,
state_cities: [],
state_shape: null,
state_mercator_adjusted_bounds: null,
simplified_cities: null,
city_guess: "",
message: "", message: "",
show_unguessed_cities: true, show_unguessed_lakes: true,
achievements: { achievements: {
"Top Five": cities => cities.slice(0, 5), "Top Five": lakes => lakes.slice(0, 5),
"Top Ten": cities => cities.slice(0, 10), "Top Ten": lakes => lakes.slice(0, 10),
"Top Twenty": cities => cities.slice(0, 20), "Top Twenty": lakes => lakes.slice(0, 20),
"Top Fifty": cities => cities.slice(0, 50), "Top Fifty": lakes => lakes.slice(0, 50),
"Top Hundred": cities => cities.slice(0, 100), "Top Hundred": lakes => lakes.slice(0, 100),
"Every Single One": cities => cities, "Every Single One": lakes => lakes,
"All above 100k": cities => cities.filter(city => city.pop >= 100000), "All above 1M acres": lakes => lakes.filter(lake => lake.area >= 1000000),
"All above 50k": cities => cities.filter(city => city.pop >= 50000), "All above 100k acres": lakes => lakes.filter(lake => lake.area >= 100000),
"All above 25k": cities => cities.filter(city => city.pop >= 25000), "All above 50k acres": lakes => lakes.filter(lake => lake.area >= 50000),
"All above 25k acres": lakes => lakes.filter(lake => lake.area >= 25000),
"With >100 lakes": lakes => lakes.filter(lake => lake.centers.length >= 100),
"With >50 lakes": lakes => lakes.filter(lake => lake.centers.length >= 50),
"With >25 lakes": lakes => lakes.filter(lake => lake.centers.length >= 25),
}, },
async mounted() { async mounted() {
// Fire off this request ASAP let all_centers = [];
this.shapes_request = fetch("data/states.geojson"); for (let lake of lakes) {
all_centers.push(...lake.centers);
// Check if there's a parameter in the URL already
this.state_name = (new URLSearchParams(window.location.search)).get('state');
if (this.state_name) {
this.launch();
} else {
const response = await fetch("data/states.json");
this.states = await response.json();
} }
},
async launch() {
this.launched = true;
const response = await fetch(`data/${this.state_name}.json`);
this.state_cities = await response.json();
this.simplified_cities = this.state_cities.map(city => simplify(city.name));
const shapes_response = await this.shapes_request;
const shapes_data = await shapes_response.json();
const shape = shapes_data.features.find(f => f.properties.NAME.toLowerCase() == this.state_name.toLowerCase());
if (shape.geometry.type == "MultiPolygon") {
this.state_shape = shape.geometry.coordinates.map(a => a.flat());
} else { // Polygon
this.state_shape = shape.geometry.coordinates;
}
this.state_mercator_adjusted_bounds = find_bounds(this.state_shape.flat().map(mercator));
this.draw();
},
async save() {
const data = {
state: this.state_name,
cities: this.state_cities,
time: new Date,
};
// https://stackoverflow.com/a/30800715
const dataStr = "data:text/json;charset=utf-8," + encodeURIComponent(JSON.stringify(data));
const downloadAnchorNode = document.createElement('a');
downloadAnchorNode.setAttribute("href", dataStr);
const safeStateName = this.state_name.replace(/[^A-Za-z]/, '-');
const datestring = `${data.time.getFullYear()}-${data.time.getMonth() + 1}-${data.time.getDate()}`;
downloadAnchorNode.setAttribute("download", `name-all-cities-${safeStateName}-${datestring}.json`);
document.body.appendChild(downloadAnchorNode); // required for firefox
downloadAnchorNode.click();
downloadAnchorNode.remove();
},
async restore(event) {
this.launched = true;
// Sure would be nice if the API were just designed this way!
const read_file = async (file) => new Promise((resolve) => {
const fileReader = new FileReader();
fileReader.onload = (e) => resolve(fileReader.result);
fileReader.readAsText(file);
});
const save_data = await read_file(event.target.files[0]);
const save = JSON.parse(save_data, (k, v) => k == 'time' ? new Date(v) : v);
this.state_cities = save.cities;
this.simplified_cities = this.state_cities.map(city => simplify(city.name));
this.state_name = save.state;
this.message = `Restored game from ${save.time.toLocaleString()} with ${save.cities.filter(c => c.guessed).length }/${ save.cities.length} cities guessed`;
// The rest of this function is duplicated from launch(), above
const shapes_response = await this.shapes_request;
const shapes_data = await shapes_response.json();
const shape = shapes_data.features.find(f => f.properties.NAME.toLowerCase() == this.state_name.toLowerCase());
if (shape.geometry.type == "MultiPolygon") {
this.state_shape = shape.geometry.coordinates.map(a => a.flat());
} else { // Polygon
this.state_shape = shape.geometry.coordinates;
}
this.state_mercator_adjusted_bounds = find_bounds(this.state_shape.flat().map(mercator));
this.bounds = find_bounds(all_centers);
console.log(this.lakes[0].name);
this.simplified_lakes = this.lakes.map(lake => simplify(lake.name));
this.draw(); this.draw();
}, },
guess() { guess() {
const rank = this.simplified_cities.indexOf(simplify(this.city_guess)) const rank = this.simplified_lakes.indexOf(simplify(this.lake_guess))
if (rank >= 0) { if (rank >= 0) {
const city = this.state_cities[rank]; const lake = this.lakes[rank];
if (!city.guessed) { if (!lake.guessed) {
city.guessed = true; lake.guessed = true;
this.message = `${city.name} (population ${city.pop}) is the ${ordinal_suffix_of(rank + 1)} most populated city in ${this.state_name}.`; this.message = `${lake.name} (total area of ${lake.centers.length} lakes ${lake.area.toFixed(3)} acres) is the ${ordinal_suffix_of(rank + 1)} largest area set of lakes in MN.`;
this.city_guess = ""; this.lake_guess = "";
this.draw(); this.draw();
} else { } else {
this.message = `Already guessed ${city.name} (population ${city.pop}, rank ${rank}).`; this.message = `Already guessed ${lake.name} (${lake.area.toFixed(3)} acres on ${lake.centers.length} lakes, rank ${rank + 1}).`;
this.city_guess = ""; this.lake_guess = "";
} }
} }
}, },
@ -208,7 +124,7 @@
const ctx = canvas.getContext("2d"); const ctx = canvas.getContext("2d");
ctx.clearRect(0, 0, canvas.width, canvas.height); ctx.clearRect(0, 0, canvas.width, canvas.height);
const [minx, maxx, miny, maxy] = this.state_mercator_adjusted_bounds; const [minx, maxx, miny, maxy] = this.bounds;
const height_scale = (canvas.height - 8) / (maxy - miny); const height_scale = (canvas.height - 8) / (maxy - miny);
const width_scale = (canvas.width - 8) / (maxx - minx); const width_scale = (canvas.width - 8) / (maxx - minx);
const scale = Math.min(height_scale, width_scale); const scale = Math.min(height_scale, width_scale);
@ -217,31 +133,24 @@
const x_offset = minx - (canvas.width / scale - (maxx - minx)) / 2; const x_offset = minx - (canvas.width / scale - (maxx - minx)) / 2;
function transform(pt) { function transform(pt) {
pt = mercator(pt)
return [scale * (pt[0] - x_offset), canvas.height - (scale * (pt[1] - y_offset))]; return [scale * (pt[0] - x_offset), canvas.height - (scale * (pt[1] - y_offset))];
} }
for (let state_outline_segment of this.state_shape) { for (let lake of this.lakes) {
ctx.beginPath(); for (let center of lake.centers) {
ctx.moveTo(...transform(state_outline_segment[0])) if (lake.guessed) {
for (let coord of state_outline_segment.slice(1)) { ctx.beginPath();
ctx.lineTo(...transform(coord)); const c = transform(center)
} ctx.fillStyle = "black";
ctx.stroke(); ctx.arc(c[0], c[1], 2, 0, 2*Math.PI, true);
} ctx.fill();
for (let city of this.state_cities) { } else if (this.show_unguessed_lakes) {
if (city.guessed) { ctx.beginPath();
ctx.beginPath(); const c = transform(center)
const c = transform(city.location) ctx.fillStyle = "lightgray";
ctx.fillStyle = "black"; ctx.arc(c[0], c[1], 2, 0, 2*Math.PI, true);
ctx.arc(c[0], c[1], 2, 0, 2*Math.PI, true); ctx.fill();
ctx.fill(); }
} else if (this.show_unguessed_cities) {
ctx.beginPath();
const c = transform(city.location)
ctx.fillStyle = "lightgray";
ctx.arc(c[0], c[1], 2, 0, 2*Math.PI, true);
ctx.fill();
} }
} }
} }
@ -265,14 +174,6 @@
const simplify = s => s.toLowerCase().replace(/[^a-z]/g, ''); const simplify = s => s.toLowerCase().replace(/[^a-z]/g, '');
function mercator(pt) {
const radians_from_eq = Math.abs(Math.PI/180 * pt[1]);
// https://en.wikipedia.org/wiki/Mercator_projection#Derivation
const y_radians = Math.log(Math.tan(Math.PI/4 + radians_from_eq/2));
const y_degrees = 180/Math.PI * y_radians;
return [pt[0], y_degrees];
}
// returns minx, maxx, miny, maxy // returns minx, maxx, miny, maxy
function find_bounds(coords) { function find_bounds(coords) {
return [ return [