chandlerswift/lawrencedeerclub
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Multithread satellite imagery downloading

Browse source
This commit is contained in:
Chandler Swift 2025-01-02 23:14:29 -06:00
parent fb1d3947b7
commit 068a0e908b
Signed by: chandlerswift
GPG key ID: A851D929D52FB93F
1 changed files with 29 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
scripts/download_satellite.py
View file

@ -6,18 +6,23 @@
import os import os
import urllib.request import urllib.request
from PIL import Image from PIL import Image
from concurrent.futures import ThreadPoolExecutor
MAX_ZOOM=20 MAX_ZOOM=20
current, total = 0, (4 ** (MAX_ZOOM - 11) - 1)//3 def download_all():
current, total = 0, (4 ** (MAX_ZOOM - 11) - 1)//3
for z in range(12, MAX_ZOOM+1): with ThreadPoolExecutor() as executor:
for z in range(12, MAX_ZOOM + 1):
scale = 2 ** (z - 12) scale = 2 ** (z - 12)
start_x = 985 * scale start_x = 985 * scale
start_y = 1432 * scale start_y = 1432 * scale
for x in range(start_x, start_x + scale): for x in range(start_x, start_x + scale):
for y in range(start_y, start_y + scale): for y in range(start_y, start_y + scale):
executor.submit(download_image, z, x, y, total, current)
current += 1 current += 1
def download_image(z, x, y, total, current):
progress = f"[{current:0{len(str(total))}d}/{total}]" progress = f"[{current:0{len(str(total))}d}/{total}]"
url = f"https://svc.pictometry.com/Image/D2B06344-7A2D-5BD0-FC89-DFDDC9888C41/wmts/PICT-MNITAS23-EDVs7UTOVt/default/GoogleMapsCompatible/{z}/{x}/{y}.png" url = f"https://svc.pictometry.com/Image/D2B06344-7A2D-5BD0-FC89-DFDDC9888C41/wmts/PICT-MNITAS23-EDVs7UTOVt/default/GoogleMapsCompatible/{z}/{x}/{y}.png"
file = os.path.normpath(os.path.join(os.path.dirname(__file__), f"../satellite/{z}/{x}/{y}.png")) file = os.path.normpath(os.path.join(os.path.dirname(__file__), f"../satellite/{z}/{x}/{y}.png"))
@ -26,9 +31,11 @@ for z in range(12, MAX_ZOOM+1):
try: # does it already exist and look good? try: # does it already exist and look good?
Image.open(file).verify() # CRC check on PNGs; nothing on other file types? https://pillow.readthedocs.io/en/latest/_modules/PIL/PngImagePlugin.html#ChunkStream.verify Image.open(file).verify() # CRC check on PNGs; nothing on other file types? https://pillow.readthedocs.io/en/latest/_modules/PIL/PngImagePlugin.html#ChunkStream.verify
print(f"{progress} {file} already exists; skipping") print(f"{progress} {file} already exists; skipping")
continue return
except OSError as e: except OSError as e:
print(f"{progress} replacing corrupt {file}") print(f"{progress} replacing corrupt {file}")
else: else:
print(f"{progress} downloading {file}") print(f"{progress} downloading {file}")
urllib.request.urlretrieve(url, file) urllib.request.urlretrieve(url, file)
download_all()