Move chains to their own directory
This commit is contained in:
parent
f3f7a21645
commit
c74edcb8c0
21 changed files with 50 additions and 40 deletions
58
layers/chains/menards/get_data.py
Executable file
58
layers/chains/menards/get_data.py
Executable file
|
|
@ -0,0 +1,58 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import re
|
||||
import json
|
||||
|
||||
# Stolen from my machine, appears to work; sufficient and necessary to get
|
||||
# around their firewall apparently? Last time this didn't work, so I wonder if
|
||||
# their protections get stronger as I make more requests. Hopefully not!
|
||||
UA={
|
||||
"User-Agent": 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0'
|
||||
}
|
||||
|
||||
res = requests.get('https://www.menards.com/store-details/locator.html', headers=UA)
|
||||
|
||||
content = res.text
|
||||
|
||||
# # For debugging, only fetch the above once; then cache it:
|
||||
# with open('temp.txt', 'w') as f:
|
||||
# f.write(content)
|
||||
|
||||
# ...and re-open it each time
|
||||
# with open('temp.txt') as f:
|
||||
# content = f.read()
|
||||
try:
|
||||
raw_initial_store_data = re.search(r'data-initial-stores="([^"]*)"', content)[1]
|
||||
except:
|
||||
print(content)
|
||||
raise SystemExit
|
||||
|
||||
menardses = json.loads(raw_initial_store_data.replace(""", '"'))
|
||||
|
||||
stores = []
|
||||
for menards in menardses:
|
||||
stores.append({
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [float(menards['longitude']), float(menards['latitude'])], # yes, [lon, lat] since it's [x, y]
|
||||
},
|
||||
"properties": {
|
||||
'address': menards['street'].title(),
|
||||
'city': menards['city'].title(),
|
||||
'state': menards['state'],
|
||||
'zip': menards['zip'],
|
||||
'website': f"https://www.menards.com/store-details/store.html?store={menards['number']}",
|
||||
},
|
||||
})
|
||||
|
||||
geojson = {
|
||||
"type": "FeatureCollection",
|
||||
"features": stores,
|
||||
}
|
||||
|
||||
with open("menards-data.geojson", "w") as f:
|
||||
f.write(json.dumps(geojson))
|
||||
|
||||
print(f"{len(menardses)} locations found")
|
||||
Loading…
Add table
Add a link
Reference in a new issue