ID

031bba6

date

2024-05-08 06:08:24+00:00

author

Alex Chan <alex@alexwlchan.net>

parent

cec70fd

message

Also handle retrying curl in this script

changed files

1 file, 52 additions, 8 deletions

• web/rcurl

Changed files

web/rcurl (238) → web/rcurl (1032)

diff --git a/web/rcurl b/web/rcurl
index 800970b..758ebb7 100755
--- a/web/rcurl
+++ b/web/rcurl
@@ -1,14 +1,58 @@
 #!/usr/bin/env bash
-# r(esumable) curl.
+# Download large files with r(esumable) curl.
 #
-# This just calls curl with a couple of flags that allow it to resume
-# an in-progress download.
+# This is for large files that can't necessarily be downloaded in one go --
+# it will call curl with a set of flags that allow it to resume an
+# in-progress download, and call it repeatedly until it's done.
 
 set -o errexit
 set -o nounset
 
-curl \
-  --location \
-  --remote-name \
-  --continue-at - \
-  "$1"
+if (( $# == 1 ))
+then
+  url="$1"
+else
+  echo "Usage: $0 URL" >&2
+  exit 1
+fi
+
+url="$1"
+
+filename=$(
+  curl \
+    --location \
+    --remote-name \
+    --silent \
+    --write-out '%{filename_effective}' \
+    --head \
+    "$url"
+)
+remote_size=$(
+  curl \
+    --location \
+    --head \
+    --silent \
+    --output /dev/null \
+    --write-out '%header{content-length}' \
+    "$url"
+)
+
+while true
+do
+  curl \
+    --location \
+    --remote-name \
+    --continue-at - \
+    "$url" || true
+
+  downloaded_size=$(stat -f%z "$filename")
+
+  if (( downloaded_size == remote_size ))
+  then
+    echo "Download complete!"
+    exit 0
+  else
+    echo "*** Download failed before it completed, retrying in 10 seconds..."
+    sleep 10
+  fi
+done