95 lines
9.2 KiB
Python
95 lines
9.2 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
|
||
|
|
import requests
|
||
|
|
import json, re
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
|
||
|
|
menufy_urls = [
|
||
|
|
"https://www.gilibertos1.com/",
|
||
|
|
"https://www.gilibertosno2.com/",
|
||
|
|
"https://www.gilibertos3.com/",
|
||
|
|
"https://www.gilibertos5.com/",
|
||
|
|
"https://www.gilibertos7.com/",
|
||
|
|
"https://www.gilibertos11.com/",
|
||
|
|
"https://www.gilbertos12.com/",
|
||
|
|
]
|
||
|
|
|
||
|
|
locations = []
|
||
|
|
for menufy_url in menufy_urls:
|
||
|
|
res = requests.get(menufy_url)
|
||
|
|
data = {}
|
||
|
|
for attr in ["lat", "lng", "name", "address"]:
|
||
|
|
data[attr] = re.search(fr'^\s*var _location{attr.title()} = "?(.*?)"?;', res.content.decode(), re.MULTILINE)[1]
|
||
|
|
|
||
|
|
locations.append({
|
||
|
|
"type": "Feature",
|
||
|
|
"geometry": {
|
||
|
|
"type": "Point",
|
||
|
|
"coordinates": [float(data['lng']), float(data['lat'])], # yes, [lon, lat] since it's [x, y]
|
||
|
|
},
|
||
|
|
"properties": {
|
||
|
|
'name': data['name'],
|
||
|
|
'address': data['address'],
|
||
|
|
'website': menufy_url,
|
||
|
|
},
|
||
|
|
})
|
||
|
|
|
||
|
|
print(locations)
|
||
|
|
|
||
|
|
# curl gilibertosno4.com \
|
||
|
|
# | <extract big lump of json> from javascript script \
|
||
|
|
# | jq -r \
|
||
|
|
# | tail -c +4 \
|
||
|
|
# | jq '.[3].children[1][3].children[3].locations[] | {address: .address, hours: .hours, name: .name, contact: .contact, coordinates: .coordinates}' > tmp.json
|
||
|
|
|
||
|
|
data="""[
|
||
|
|
{"address":{"city":"St Cloud","full":"2301 W Division St #100, St Cloud, MN 56301, USA","state":"MN","street":"2301 W Division St #100","zip":"56301","unit":null},"hours":{"delivery":{"sunday":{"hours":[],"open":true},"monday":{"hours":[],"open":true},"tuesday":{"hours":[],"open":true},"wednesday":{"hours":[],"open":true},"thursday":{"hours":[],"open":true},"friday":{"hours":[],"open":true},"saturday":{"hours":[],"open":true}},"pickup":{"sunday":{"hours":[],"open":true},"monday":{"hours":[],"open":true},"tuesday":{"hours":[],"open":true},"wednesday":{"hours":[],"open":true},"thursday":{"hours":[],"open":true},"friday":{"hours":[],"open":true},"saturday":{"hours":[],"open":true}},"store":{"sunday":{"hours":[],"open":true},"monday":{"hours":[],"open":true},"tuesday":{"hours":[],"open":true},"wednesday":{"hours":[],"open":true},"thursday":{"hours":[],"open":true},"friday":{"hours":[],"open":true},"saturday":{"hours":[],"open":true}}},"name":"GILIBERTO'S #4","contact":{"email":"alvarezcoconi@gmail.com","phone":"+13202535729"},"coordinates":{"latitude":45.5535297,"longitude":-94.1857337}},
|
||
|
|
{"address":{"city":"Willmar","full":"602 1st St S e, Willmar, MN 56201, USA","state":"MN","street":"602 1st St S Suite E","zip":"56201","unit":null},"hours":{"delivery":{"sunday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true}},"pickup":{"sunday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true}},"store":{"sunday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"12:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"1:00 AM","start":"6:00 AM"}],"open":true}}},"name":"GILIBERTO'S #6","contact":{"email":"alvarezcoconi@gmail.com","phone":"+19702106534"},"coordinates":{"latitude":45.1183051,"longitude":-95.04330569999999}},
|
||
|
|
{"address":{"city":"Mankato","full":"625 Madison Ave, Mankato, MN 56001, USA","state":"MN","street":"625 Madison Ave","zip":"56001","unit":null},"hours":{"delivery":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}},"pickup":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}},"store":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}}},"name":"GILIBERTO'S #10","contact":{"email":"alvarezcoconi@gmail.com","phone":"+19702106534"},"coordinates":{"latitude":44.1709202,"longitude":-93.9886599}},
|
||
|
|
{"address":{"city":"Owatonna","full":"1232 S Oak Ave, Owatonna, MN 55060, USA","state":"MN","street":"1232 S Oak Ave","zip":"55060","unit":null},"hours":{"delivery":{"sunday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true}},"pickup":{"sunday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true}},"store":{"sunday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"monday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"tuesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"wednesday":{"hours":[{"end":"2:00 AM","start":"6:00 AM"}],"open":true},"thursday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"friday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true},"saturday":{"hours":[{"end":"3:00 AM","start":"6:00 AM"}],"open":true}}},"name":"GILIBERTO'S #13","contact":{"email":"alvarezcoconi@gmail.com","phone":"+19702106534"},"coordinates":{"latitude":44.0704912,"longitude":-93.2283946}},
|
||
|
|
{"address":{"city":"Rochester","full":"5550 US-52, Rochester, MN 55901, USA","state":"MN","street":"5550 US-52","zip":"55901","unit":null},"hours":{"delivery":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}},"pickup":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}},"store":{"sunday":{"open":true,"hours":[]},"monday":{"open":true,"hours":[]},"tuesday":{"open":true,"hours":[]},"wednesday":{"open":true,"hours":[]},"thursday":{"open":true,"hours":[]},"friday":{"open":true,"hours":[]},"saturday":{"open":true,"hours":[]}}},"name":"GILIBERTO'S #14","contact":{"email":"alvarezcoconi@gmail.com","phone":"+19702106534"},"coordinates":{"latitude":44.0808551,"longitude":-92.506073}}
|
||
|
|
]
|
||
|
|
"""
|
||
|
|
|
||
|
|
for mn_loc in json.loads(data):
|
||
|
|
locations.append({
|
||
|
|
"type": "Feature",
|
||
|
|
"geometry": {
|
||
|
|
"type": "Point",
|
||
|
|
"coordinates": [mn_loc['coordinates']['longitude'], mn_loc['coordinates']['latitude']], # yes, [lon, lat] since it's [x, y]
|
||
|
|
},
|
||
|
|
"properties": {
|
||
|
|
'name': mn_loc['name'],
|
||
|
|
'address': mn_loc['address'],
|
||
|
|
'hours': mn_loc['hours'],
|
||
|
|
'contact': mn_loc['contact'],
|
||
|
|
'website': "https://gilibertosno4.com/",
|
||
|
|
},
|
||
|
|
})
|
||
|
|
|
||
|
|
# Screw it, we'll do this one manually
|
||
|
|
locations.append({
|
||
|
|
"type": "Feature",
|
||
|
|
"geometry": {
|
||
|
|
"type": "Point",
|
||
|
|
"coordinates": [-93.5041166,41.6533301], # yes, [lon, lat] since it's [x, y]
|
||
|
|
},
|
||
|
|
"properties": {
|
||
|
|
'name': 'Gilibertos No. 9',
|
||
|
|
'address': '400 34th Avenue SW, Altoona, IA 50009',
|
||
|
|
'hours': mn_loc['hours'],
|
||
|
|
'phone': '515-967-8966',
|
||
|
|
'website': "https://gilibertos.net/",
|
||
|
|
},
|
||
|
|
})
|
||
|
|
|
||
|
|
geojson = {
|
||
|
|
"type": "FeatureCollection",
|
||
|
|
"features": locations,
|
||
|
|
}
|
||
|
|
|
||
|
|
print(len(locations), "locations found")
|
||
|
|
|
||
|
|
with open("data.geojson", "w") as f:
|
||
|
|
f.write(json.dumps(geojson))
|