chandlerswift/lawrencedeerclub
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Multithread satellite imagery downloading

Browse source
This commit is contained in:
Chandler Swift 2025-01-02 23:14:29 -06:00
parent fb1d3947b7
commit 068a0e908b
Signed by: chandlerswift
GPG key ID: A851D929D52FB93F
1 changed files with 29 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
scripts/download_satellite.py
View file

@ -6,29 +6,36 @@
import os import os
import urllib.request import urllib.request
from PIL import Image from PIL import Image
from concurrent.futures import ThreadPoolExecutor
MAX_ZOOM=20 MAX_ZOOM=20
current, total = 0, (4 ** (MAX_ZOOM - 11) - 1)//3 def download_all():
current, total = 0, (4 ** (MAX_ZOOM - 11) - 1)//3
with ThreadPoolExecutor() as executor:
for z in range(12, MAX_ZOOM + 1):
scale = 2 ** (z - 12)
start_x = 985 * scale
start_y = 1432 * scale
for x in range(start_x, start_x + scale):
for y in range(start_y, start_y + scale):
executor.submit(download_image, z, x, y, total, current)
current += 1
for z in range(12, MAX_ZOOM+1): def download_image(z, x, y, total, current):
scale = 2 ** (z - 12) progress = f"[{current:0{len(str(total))}d}/{total}]"
start_x = 985 * scale url = f"https://svc.pictometry.com/Image/D2B06344-7A2D-5BD0-FC89-DFDDC9888C41/wmts/PICT-MNITAS23-EDVs7UTOVt/default/GoogleMapsCompatible/{z}/{x}/{y}.png"
start_y = 1432 * scale file = os.path.normpath(os.path.join(os.path.dirname(__file__), f"../satellite/{z}/{x}/{y}.png"))
for x in range(start_x, start_x + scale): os.makedirs(os.path.dirname(file), exist_ok=True)
for y in range(start_y, start_y + scale): if os.path.isfile(file):
current += 1 try: # does it already exist and look good?
progress = f"[{current:0{len(str(total))}d}/{total}]" Image.open(file).verify() # CRC check on PNGs; nothing on other file types? https://pillow.readthedocs.io/en/latest/_modules/PIL/PngImagePlugin.html#ChunkStream.verify
url = f"https://svc.pictometry.com/Image/D2B06344-7A2D-5BD0-FC89-DFDDC9888C41/wmts/PICT-MNITAS23-EDVs7UTOVt/default/GoogleMapsCompatible/{z}/{x}/{y}.png" print(f"{progress} {file} already exists; skipping")
file = os.path.normpath(os.path.join(os.path.dirname(__file__), f"../satellite/{z}/{x}/{y}.png")) return
os.makedirs(os.path.dirname(file), exist_ok=True) except OSError as e:
if os.path.isfile(file): print(f"{progress} replacing corrupt {file}")
try: # does it already exist and look good? else:
Image.open(file).verify() # CRC check on PNGs; nothing on other file types? https://pillow.readthedocs.io/en/latest/_modules/PIL/PngImagePlugin.html#ChunkStream.verify print(f"{progress} downloading {file}")
print(f"{progress} {file} already exists; skipping") urllib.request.urlretrieve(url, file)
continue
except OSError as e: download_all()
print(f"{progress} replacing corrupt {file}")
else:
print(f"{progress} downloading {file}")
urllib.request.urlretrieve(url, file)