From fb1d3947b76d1e0c3049c71db64da90625512c41 Mon Sep 17 00:00:00 2001 From: Chandler Swift Date: Thu, 2 Jan 2025 22:34:29 -0600 Subject: [PATCH] Add basic detection/recovery of corrupt partial downloads --- scripts/download_satellite.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/download_satellite.py b/scripts/download_satellite.py index c7622f8..d0f083c 100755 --- a/scripts/download_satellite.py +++ b/scripts/download_satellite.py @@ -1,10 +1,11 @@ #!/usr/bin/env nix-shell -#!nix-shell --quiet -p python3 -i python +#!nix-shell --quiet -p python312 -p python312Packages.pillow -i python # https://svc.pictometry.com/Image/D2B06344-7A2D-5BD0-FC89-DFDDC9888C41/wmts?SERVICE=WMTS&REQUEST=GetCapabilities import os import urllib.request +from PIL import Image MAX_ZOOM=20 @@ -22,7 +23,12 @@ for z in range(12, MAX_ZOOM+1): file = os.path.normpath(os.path.join(os.path.dirname(__file__), f"../satellite/{z}/{x}/{y}.png")) os.makedirs(os.path.dirname(file), exist_ok=True) if os.path.isfile(file): - print(f"{progress} {file} already exists; skipping") - continue - print(f"{progress} downloading {file}") + try: # does it already exist and look good? + Image.open(file).verify() # CRC check on PNGs; nothing on other file types? https://pillow.readthedocs.io/en/latest/_modules/PIL/PngImagePlugin.html#ChunkStream.verify + print(f"{progress} {file} already exists; skipping") + continue + except OSError as e: + print(f"{progress} replacing corrupt {file}") + else: + print(f"{progress} downloading {file}") urllib.request.urlretrieve(url, file)