2023-07-04 00:08:39 -05:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
|
|
import requests
|
|
|
|
import re
|
|
|
|
import json
|
|
|
|
|
|
|
|
# Stolen from my machine, appears to work; sufficient and necessary to get
|
|
|
|
# around their firewall apparently? Last time this didn't work, so I wonder if
|
|
|
|
# their protections get stronger as I make more requests. Hopefully not!
|
|
|
|
UA={
|
|
|
|
"User-Agent": 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0'
|
|
|
|
}
|
|
|
|
|
|
|
|
res = requests.get('https://www.menards.com/store-details/locator.html', headers=UA)
|
|
|
|
|
|
|
|
content = res.text
|
|
|
|
|
|
|
|
# # For debugging, only fetch the above once; then cache it:
|
|
|
|
# with open('temp.txt', 'w') as f:
|
|
|
|
# f.write(content)
|
|
|
|
|
|
|
|
# ...and re-open it each time
|
|
|
|
# with open('temp.txt') as f:
|
|
|
|
# content = f.read()
|
|
|
|
try:
|
|
|
|
raw_initial_store_data = re.search(r'data-initial-stores="([^"]*)"', content)[1]
|
|
|
|
except:
|
|
|
|
print(content)
|
|
|
|
raise SystemExit
|
|
|
|
|
|
|
|
menardses = json.loads(raw_initial_store_data.replace(""", '"'))
|
|
|
|
|
|
|
|
stores = []
|
|
|
|
for menards in menardses:
|
|
|
|
stores.append({
|
|
|
|
"type": "Feature",
|
|
|
|
"geometry": {
|
|
|
|
"type": "Point",
|
|
|
|
"coordinates": [float(menards['longitude']), float(menards['latitude'])], # yes, [lon, lat] since it's [x, y]
|
|
|
|
},
|
|
|
|
"properties": {
|
|
|
|
'address': menards['street'].title(),
|
|
|
|
'city': menards['city'].title(),
|
|
|
|
'state': menards['state'],
|
|
|
|
'zip': menards['zip'],
|
|
|
|
'website': f"https://www.menards.com/store-details/store.html?store={menards['number']}",
|
|
|
|
},
|
|
|
|
})
|
|
|
|
|
|
|
|
geojson = {
|
|
|
|
"type": "FeatureCollection",
|
|
|
|
"features": stores,
|
|
|
|
}
|
|
|
|
|
2023-07-25 19:12:30 -05:00
|
|
|
with open("data.geojson", "w") as f:
|
2023-07-04 00:08:39 -05:00
|
|
|
f.write(json.dumps(geojson))
|
|
|
|
|
|
|
|
print(f"{len(menardses)} locations found")
|