ID

168f33c

date

2023-09-06 06:47:32+00:00

author

Alex Chan <alex@alexwlchan.net>

parent

98db4e4

message

avoid hitting S3's rate limits

changed files

1 file, 3 additions, 1 deletion

• aws/s3rm.py

Changed files

aws/s3rm.py (2429) → aws/s3rm.py (2534)

diff --git a/aws/s3rm.py b/aws/s3rm.py
index 7272990..f92c440 100755
--- a/aws/s3rm.py
+++ b/aws/s3rm.py
@@ -59,7 +59,9 @@ def delete_objects(sess, iterator):
     try:
         for batch, _ in concurrently(
             handler=delete_batch,
-            inputs=more_itertools.chunked(iterator, 1000)
+            inputs=more_itertools.chunked(iterator, 1000),
+            # Note: if you go too fast, you get a SlowDown error from S3.
+            max_concurrency=3
         ):
             total_deleted_count += len(batch)
             total_deleted_size += sum(s3_obj['Size'] for s3_obj in batch)