From 7957523c3ce674e06d9f92caca81a282997c2d1c Mon Sep 17 00:00:00 2001 From: Chandler Swift Date: Tue, 25 Jul 2023 19:12:30 -0500 Subject: [PATCH] Distribute data with layers instead of data/ dir This was originally done to make the gitignoring easier, but ended up being somewhat more complex when trying to include files, so they're moving out closer to the point of use. --- .gitignore | 2 ++ Makefile | 9 ++++++++- data/get_all_data.sh | 10 ---------- layers/amtrak/get_data.sh | 2 +- layers/amtrak/layer.js | 8 ++++---- layers/bikepacking/get_data.sh | 2 +- layers/bikepacking/layer.js | 2 +- layers/chains/culvers/get_data.py | 2 +- layers/chains/culvers/layer.js | 8 ++++---- layers/chains/krispy-kreme/get_data.py | 2 +- layers/chains/krispy-kreme/layer.js | 8 ++++---- layers/chains/kwik-trip/get_data.py | 3 +-- layers/chains/kwik-trip/layer.js | 8 ++++---- layers/chains/menards/get_data.py | 2 +- layers/chains/menards/layer.js | 8 ++++---- layers/chains/milwaukee-burger-company/get_data.py | 6 +++--- layers/chains/milwaukee-burger-company/layer.js | 2 +- layers/chains/waffle-house/get_data.py | 2 +- layers/chains/waffle-house/layer.js | 8 ++++---- layers/nhl-arenas/get_data.py | 2 +- layers/nhl-arenas/layer.js | 4 ++-- 21 files changed, 49 insertions(+), 51 deletions(-) delete mode 100755 data/get_all_data.sh diff --git a/.gitignore b/.gitignore index f06235c..2fbb441 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ node_modules dist +*.geojson +*.shp diff --git a/Makefile b/Makefile index 39a13bd..e45b7d2 100644 --- a/Makefile +++ b/Makefile @@ -14,4 +14,11 @@ build: # TODO: add get_data as a prereq # this file much more often. Maybe as things stabilize! .PHONY: get_data get_data: - ./data/get_all_data.sh + @find ./ -name "get_data.*" \! -executable | grep . >/dev/null \ + && echo "Non-executable files found; consider running:" \ + && find ./ -name "get_data.*" \! -executable -exec echo " " chmod +x {} \; \ + && exit 1 \ + || true + + @find ./ -name "get_data.*" -executable \ + -exec sh -c 'cd $$(dirname {}) && echo processing {}; ./$$(basename {}) | sed --unbuffered "s/^/ /"' \; diff --git a/data/get_all_data.sh b/data/get_all_data.sh deleted file mode 100755 index ab6465b..0000000 --- a/data/get_all_data.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -cd $(dirname $0) - -find ../ -name "get_data.*" \! -executable | grep . >/dev/null \ - && echo "Non-executable files found; consider running:" \ - && find ../ -name "get_data.*" \! -executable -exec echo chmod +x {} \; \ - && exit 1 - -find ../ -name "get_data.*" -executable -exec sh -c "echo processing {}; {} | sed --unbuffered 's/^/ /'" \; diff --git a/layers/amtrak/get_data.sh b/layers/amtrak/get_data.sh index b905eab..be7b597 100755 --- a/layers/amtrak/get_data.sh +++ b/layers/amtrak/get_data.sh @@ -1,3 +1,3 @@ #!/bin/sh -curl --silent --output amtrak-data.geojson https://maps.amtrak.com/services/MapDataService/stations/nationalRoute +curl --silent --output data.geojson https://maps.amtrak.com/services/MapDataService/stations/nationalRoute diff --git a/layers/amtrak/layer.js b/layers/amtrak/layer.js index 20cdfa7..d87fe4c 100644 --- a/layers/amtrak/layer.js +++ b/layers/amtrak/layer.js @@ -4,19 +4,19 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style, Stroke} from 'ol/style.js'; -import amtrakURL from '/data/amtrak-data.geojson?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? -const amtrak_colors = '0,83,126'; // from their website's cookie banner, and other places +const colors = '0,83,126'; // from their website's cookie banner, and other places const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: amtrakURL, + url: url, format: new GeoJSON, }), style: function(feature, resolution){ return new Style({ stroke: new Stroke({ - color: `rgba(${amtrak_colors},${Math.min(1, Math.pow(resolution/10, 1/4))})`, + color: `rgba(${colors},${Math.min(1, Math.pow(resolution/10, 1/4))})`, width: 10/Math.pow(resolution, 1/4), }) }); diff --git a/layers/bikepacking/get_data.sh b/layers/bikepacking/get_data.sh index 4dbaf10..2782a1e 100755 --- a/layers/bikepacking/get_data.sh +++ b/layers/bikepacking/get_data.sh @@ -2,4 +2,4 @@ # from https://ridewithgps.com/embeds?type=region&id=37 # also https://ridewithgps.com/route_set_overviews/37.json?include_details=1 -curl --silent --output bikepacking-data.geojson https://ridewithgps.com/cos/route_set_overviews/simplified_route/37.json +curl --silent --output data.geojson https://ridewithgps.com/cos/route_set_overviews/simplified_route/37.json diff --git a/layers/bikepacking/layer.js b/layers/bikepacking/layer.js index 0d2feda..c973a31 100644 --- a/layers/bikepacking/layer.js +++ b/layers/bikepacking/layer.js @@ -4,7 +4,7 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style, Stroke} from 'ol/style.js'; -import url from '/data/bikepacking-data.geojson?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? const colors = '0,0,0'; // from their website's cookie banner, and other places diff --git a/layers/chains/culvers/get_data.py b/layers/chains/culvers/get_data.py index 9d34184..4b322f1 100755 --- a/layers/chains/culvers/get_data.py +++ b/layers/chains/culvers/get_data.py @@ -66,5 +66,5 @@ geojson = { "features": stores, } -with open("culvers-data.geojson", "w") as f: +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) diff --git a/layers/chains/culvers/layer.js b/layers/chains/culvers/layer.js index 4e26b2b..af140b4 100644 --- a/layers/chains/culvers/layer.js +++ b/layers/chains/culvers/layer.js @@ -5,18 +5,18 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import culversURL from '/data/culvers-data.geojson?url'; // TODO: remove `?url`? -import pinURL from './pin.svg?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? +import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: culversURL, + url: url, format: new GeoJSON, }), style: new Style({ image: new Icon({ anchor: [0.5, 1], - src: pinURL, + src: pin, }), }), }); diff --git a/layers/chains/krispy-kreme/get_data.py b/layers/chains/krispy-kreme/get_data.py index fe7872a..e3206fc 100755 --- a/layers/chains/krispy-kreme/get_data.py +++ b/layers/chains/krispy-kreme/get_data.py @@ -31,7 +31,7 @@ geojson = { "features": stores, } -with open("krispy-kreme-data.geojson", "w") as f: +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) print(f"{len(locations)} locations found") diff --git a/layers/chains/krispy-kreme/layer.js b/layers/chains/krispy-kreme/layer.js index b5ffbaa..af140b4 100644 --- a/layers/chains/krispy-kreme/layer.js +++ b/layers/chains/krispy-kreme/layer.js @@ -5,18 +5,18 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import waffleHouseURL from '/data/krispy-kreme-data.geojson?url'; // TODO: remove `?url`? -import pinURL from './pin.svg?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? +import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: waffleHouseURL, + url: url, format: new GeoJSON, }), style: new Style({ image: new Icon({ anchor: [0.5, 1], - src: pinURL, + src: pin, }), }), }); diff --git a/layers/chains/kwik-trip/get_data.py b/layers/chains/kwik-trip/get_data.py index 237d849..97b13bf 100755 --- a/layers/chains/kwik-trip/get_data.py +++ b/layers/chains/kwik-trip/get_data.py @@ -4,7 +4,6 @@ import requests import json from bs4 import BeautifulSoup -print("Searching for Kwik Trip locations") response = requests.get('https://www.kwiktrip.com/Maps-Downloads/Store-List') # HACK HACK HACK @@ -44,5 +43,5 @@ geojson = { "features": stores, } -with open("kwik-trip-data.geojson", "w") as f: +with open("./data.geojson", "w") as f: f.write(json.dumps(geojson)) diff --git a/layers/chains/kwik-trip/layer.js b/layers/chains/kwik-trip/layer.js index e1d1b7c..af140b4 100644 --- a/layers/chains/kwik-trip/layer.js +++ b/layers/chains/kwik-trip/layer.js @@ -5,18 +5,18 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import kwikTripURL from '/data/kwik-trip-data.geojson?url'; // TODO: remove `?url`? -import pinURL from './pin.svg?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? +import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: kwikTripURL, + url: url, format: new GeoJSON, }), style: new Style({ image: new Icon({ anchor: [0.5, 1], - src: pinURL, + src: pin, }), }), }); diff --git a/layers/chains/menards/get_data.py b/layers/chains/menards/get_data.py index 7b7f08e..a98e2ac 100755 --- a/layers/chains/menards/get_data.py +++ b/layers/chains/menards/get_data.py @@ -52,7 +52,7 @@ geojson = { "features": stores, } -with open("menards-data.geojson", "w") as f: +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) print(f"{len(menardses)} locations found") diff --git a/layers/chains/menards/layer.js b/layers/chains/menards/layer.js index 41c7e85..af140b4 100644 --- a/layers/chains/menards/layer.js +++ b/layers/chains/menards/layer.js @@ -5,18 +5,18 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import menardsURL from '/data/menards-data.geojson?url'; // TODO: remove `?url`? -import pinURL from './pin.svg?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? +import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: menardsURL, + url: url, format: new GeoJSON, }), style: new Style({ image: new Icon({ anchor: [0.5, 1], - src: pinURL, + src: pin, }), }), }); diff --git a/layers/chains/milwaukee-burger-company/get_data.py b/layers/chains/milwaukee-burger-company/get_data.py index 31ee135..45582f1 100755 --- a/layers/chains/milwaukee-burger-company/get_data.py +++ b/layers/chains/milwaukee-burger-company/get_data.py @@ -4,8 +4,6 @@ import requests import json from bs4 import BeautifulSoup -print("Searching for Milwaukee Burger Company locations") - # Stolen from my machine, appears to work; sufficient and necessary to get # around their firewall apparently? UA={ @@ -39,5 +37,7 @@ geojson = { "features": locations, } -with open("milwaukee-burger-company-data.geojson", "w") as f: +print(len(locations), "locations found") + +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) diff --git a/layers/chains/milwaukee-burger-company/layer.js b/layers/chains/milwaukee-burger-company/layer.js index 23d2ffb..af140b4 100644 --- a/layers/chains/milwaukee-burger-company/layer.js +++ b/layers/chains/milwaukee-burger-company/layer.js @@ -5,7 +5,7 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import url from '/data/milwaukee-burger-company-data.geojson?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ diff --git a/layers/chains/waffle-house/get_data.py b/layers/chains/waffle-house/get_data.py index fa81d22..39e5998 100755 --- a/layers/chains/waffle-house/get_data.py +++ b/layers/chains/waffle-house/get_data.py @@ -35,7 +35,7 @@ geojson = { "features": stores, } -with open("waffle-house-data.geojson", "w") as f: +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) print(f"{len(locations)} locations found") diff --git a/layers/chains/waffle-house/layer.js b/layers/chains/waffle-house/layer.js index 093fadd..af140b4 100644 --- a/layers/chains/waffle-house/layer.js +++ b/layers/chains/waffle-house/layer.js @@ -5,18 +5,18 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style} from 'ol/style.js'; import Icon from 'ol/style/Icon.js'; -import waffleHouseURL from '/data/waffle-house-data.geojson?url'; // TODO: remove `?url`? -import pinURL from './pin.svg?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? +import pin from './pin.svg?url'; // TODO: remove `?url`? const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: waffleHouseURL, + url: url, format: new GeoJSON, }), style: new Style({ image: new Icon({ anchor: [0.5, 1], - src: pinURL, + src: pin, }), }), }); diff --git a/layers/nhl-arenas/get_data.py b/layers/nhl-arenas/get_data.py index e83d365..5459f20 100755 --- a/layers/nhl-arenas/get_data.py +++ b/layers/nhl-arenas/get_data.py @@ -93,5 +93,5 @@ geojson = { "features": arenas, } -with open("nhl-arenas-data.geojson", "w") as f: +with open("data.geojson", "w") as f: f.write(json.dumps(geojson)) diff --git a/layers/nhl-arenas/layer.js b/layers/nhl-arenas/layer.js index 434ac9c..f8b85d8 100644 --- a/layers/nhl-arenas/layer.js +++ b/layers/nhl-arenas/layer.js @@ -4,13 +4,13 @@ import GeoJSON from 'ol/format/GeoJSON.js'; import {Style, Stroke, Circle, Fill} from 'ol/style.js'; -import arenaURL from '/data/nhl-arenas-data.geojson?url'; // TODO: remove `?url`? +import url from './data.geojson?url'; // TODO: remove `?url`? import visitedArenas from './visited.js' const vectorLayer = new VectorLayer({ source: new VectorSource({ - url: arenaURL, + url: url, format: new GeoJSON, }), // TODO: use '✓' and '✗' (or maybe '✔' and '✘') (from https://en.wikipedia.org/wiki/List_of_Unicode_characters#Dingbats)