ID

ed83408

date

2023-10-12 08:08:12+00:00

author

Alex Chan <alex@alexwlchan.net>

parent

4751b87

message

Remove a bunch of Wellcome-specific AWS wrappers

changed files

20 files, 9 additions, 195 deletions

• aws/README.md
• aws/_ensure_aws_credentials_are_fresh
• aws/aws
• aws/bulk_sns_publish.py
• aws/download_sqs_messages
• aws/dynamols
• aws/login_ecr_public
• aws/s3_unfreeze
• aws/s3hash
• aws/s3ls
• aws/s3rm
• aws/s3tree
• aws/sqs_stats
• wellcome/README.md
• wellcome/bag
• wellcome/logs
• wellcome/screenshots/bag.png
• wellcome/screenshots/logs.png
• wellcome/ssh_to_archivematica
• wellcome/wellcome

Changed files

aws/README.md (5415) → aws/README.md (5469)

diff --git a/aws/README.md b/aws/README.md
index 70e396f..5b91958 100644
--- a/aws/README.md
+++ b/aws/README.md
@@ -13,7 +13,7 @@ These are scripts to do stuff in AWS.
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/bulk_sns_publish"><code>bulk_sns_publish</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/bulk_sns_publish.py"><code>bulk_sns_publish.py</code></a>
   </dt>
   <dd>
     a tool for publishing lots of messages to SNS, using the <code>PublishBatch</code> API.
@@ -21,7 +21,7 @@ These are scripts to do stuff in AWS.
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/download_sqs_messages"><code>download_sqs_messages</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/download_sqs_messages.py"><code>download_sqs_messages.py</code></a>
   </dt>
   <dd>
     a tool for downloading lots of messages from SQS, using the <code>ReceiveMessage</code> API.
@@ -29,28 +29,28 @@ These are scripts to do stuff in AWS.
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/dynamols"><code>dynamols</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/dynamols.py"><code>dynamols.py</code></a>
   </dt>
   <dd>
     print the items in a DynamoDB table, one item per line
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3_unfreeze"><code>s3_unfreeze</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3_unfreeze.py"><code>s3_unfreeze.py</code></a>
   </dt>
   <dd>
     takes a list of S3 URIs as input, and either restores those objects from Glacier or reports the status of an in-progress restoration
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3hash"><code>s3hash <S3_URI> [--algorithm=<ALGO>]</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3hash.py"><code>s3hash.py <S3_URI> [--algorithm=<ALGO>]</code></a>
   </dt>
   <dd>
     get the checksum/hash of an object in S3
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3ls"><code>s3ls</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3ls.py"><code>s3ls.py</code></a>
   </dt>
   <dd>
     list objects from an S3 prefix using the <code>ListObjectsV2</code> API, and print them as JSON to stdout.
@@ -63,7 +63,7 @@ …</code></pre></p>
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3rm"><code>s3rm</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3rm.py"><code>s3rm.py</code></a>
   </dt>
   <dd>
     delete objects from an S3 prefix.
@@ -71,7 +71,7 @@ …</code></pre></p>
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3tree"><code>s3tree</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3tree.py"><code>s3tree.py</code></a>
   </dt>
   <dd>
     show a tree-like view of objects and folders in an S3 prefix.
@@ -80,7 +80,7 @@ …</code></pre></p>
   </dd>
 
   <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/sqs_stats"><code>sqs_stats</code></a>
+    <a href="https://github.com/alexwlchan/scripts/blob/main/aws/sqs_stats.py"><code>sqs_stats.py</code></a>
   </dt>
   <dd>
     prints a summary of messages visible on our SQS queues.

aws/_ensure_aws_credentials_are_fresh (1396) → aws/_ensure_aws_credentials_are_fresh (0)

diff --git a/aws/_ensure_aws_credentials_are_fresh b/aws/_ensure_aws_credentials_are_fresh
deleted file mode 100755
index 06540b9..0000000
--- a/aws/_ensure_aws_credentials_are_fresh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# To get AWS credentials, I use aws-azure-login, which use Azure SSO
-# to get temporary credentials.
-# See https://github.com/wellcomecollection/platform-infrastructure/blob/main/accounts/docs/cli-credentials.md
-#
-# This script runs in front of a couple of wrappers that require AWS access
-# (e.g. tfi, tfp) to ensure that I have fresh credentials, which saves me
-# trying something which will obviously fail, refreshing creds, then trying
-# a second time. Everything goes a little smoother!
-
-set -o errexit
-set -o nounset
-
-function get_aws_credentials_expiry_timestamp() {
-  timestamp=$(
-    grep aws_expiration ~/.aws/credentials \
-      | tr '=' ' ' \
-      | awk '{print $2}'
-  )
-
-  unix_timestamp=$(python3 -c "
-from datetime import *
-d = datetime.strptime('$timestamp', '%Y-%m-%dT%H:%M:%S.%fZ')
-print(int(d.timestamp()))
-  ")
-
-  echo "$unix_timestamp"
-}
-
-EXPIRY_TIMESTAMP=$(get_aws_credentials_expiry_timestamp)
-CURRENT_TIMESTAMP=$(date +%s)
-
-if (( CURRENT_TIMESTAMP - 100 >= EXPIRY_TIMESTAMP ))
-then
-  echo "AWS credentials have expired, auto-refreshing..."
-  AWS_PROFILE="" aws-azure-login --no-prompt
-fi
-
-EXPIRY_TIMESTAMP=$(get_aws_credentials_expiry_timestamp)
-CURRENT_TIMESTAMP=$(date +%s)
-
-if (( CURRENT_TIMESTAMP - 100 >= EXPIRY_TIMESTAMP ))
-then
-  echo "AWS credentials are still expired, throwing to a GUI prompt"
-  AWS_PROFILE="" aws-azure-login --mode=gui
-fi

aws/aws (376) → aws/aws (0)

diff --git a/aws/aws b/aws/aws
deleted file mode 100755
index 469c5f0..0000000
--- a/aws/aws
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-# This provides a thin wrapper around the AWS CLI, and prefaces it by
-# ensuring I have up-to-date credentials.
-#
-# It relies on:
-#
-#   1. This script being before the regular AWS CLI in my $PATH
-#   2. Having the AWS CLI installed in the same dir across machines
-#
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-
-python3 -m awscli "$@"

aws/bulk_sns_publish (4614) → aws/bulk_sns_publish.py (4614)

diff --git a/aws/bulk_sns_publish b/aws/bulk_sns_publish.py
similarity index 100%
rename from aws/bulk_sns_publish
rename to aws/bulk_sns_publish.py

aws/download_sqs_messages (116) → aws/download_sqs_messages (0)

diff --git a/aws/download_sqs_messages b/aws/download_sqs_messages
deleted file mode 100755
index 765fd8e..0000000
--- a/aws/download_sqs_messages
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-download_sqs_messages.py "$@"

aws/dynamols (103) → aws/dynamols (0)

diff --git a/aws/dynamols b/aws/dynamols
deleted file mode 100755
index c50b9e5..0000000
--- a/aws/dynamols
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-dynamols.py "$@"

aws/login_ecr_public (248) → aws/login_ecr_public (213)

diff --git a/aws/login_ecr_public b/aws/login_ecr_public
index 0fb1dc4..611f151 100755
--- a/aws/login_ecr_public
+++ b/aws/login_ecr_public
@@ -4,7 +4,5 @@ set -o errexit
 set -o nounset
 set -o verbose
 
-_ensure_aws_credentials_are_fresh
-
 AWS_PROFILE=experience-dev aws ecr-public \
   --region=us-east-1 get-login-password | docker login --username AWS --password-stdin public.ecr.aws

aws/s3_unfreeze (106) → aws/s3_unfreeze (0)

diff --git a/aws/s3_unfreeze b/aws/s3_unfreeze
deleted file mode 100755
index 1a49f0f..0000000
--- a/aws/s3_unfreeze
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-s3_unfreeze.py "$@"

aws/s3hash (101) → aws/s3hash (0)

diff --git a/aws/s3hash b/aws/s3hash
deleted file mode 100755
index 97bed45..0000000
--- a/aws/s3hash
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-s3hash.py "$@"

aws/s3ls (99) → aws/s3ls (0)

diff --git a/aws/s3ls b/aws/s3ls
deleted file mode 100755
index 4b3dfbd..0000000
--- a/aws/s3ls
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-s3ls.py "$@"

aws/s3rm (99) → aws/s3rm (0)

diff --git a/aws/s3rm b/aws/s3rm
deleted file mode 100755
index 69b9b43..0000000
--- a/aws/s3rm
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-s3rm.py "$@"

aws/s3tree (101) → aws/s3tree (0)

diff --git a/aws/s3tree b/aws/s3tree
deleted file mode 100755
index 63289f9..0000000
--- a/aws/s3tree
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-s3tree.py "$@"

aws/sqs_stats (104) → aws/sqs_stats (0)

diff --git a/aws/sqs_stats b/aws/sqs_stats
deleted file mode 100755
index 9cfe276..0000000
--- a/aws/sqs_stats
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-sqs_stats.py "$@"

wellcome/README.md (1457) → wellcome/README.md (0)

diff --git a/wellcome/README.md b/wellcome/README.md
deleted file mode 100644
index 6b6bad2..0000000
--- a/wellcome/README.md
+++ /dev/null
@@ -1,37 +0,0 @@
-# wellcome
-
-These are scripts that are highly specific to my work at [Wellcome Collection].
-It's unlikely these would be of any use to somebody not at Wellcome (except as a learning tool).
-
-[Wellcome Collection]: https://wellcomecollection.org/
-
-## The individual scripts
-
-<dl>
-  <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/wellcome/bag"><code>bag</code></a>
-  </dt>
-  <dd>
-    prints a tree view of a bag in the <a href="https://github.com/wellcomecollection/storage-service">Wellcome storage service</a>.
-    This is a thin wrapper around my <a href="https://github.com/alexwlchan/scripts/blob/main/aws/s3tree.py"><code>s3tree</code> script</a>.
-    <img src="screenshots/bag.png">
-  </dd>
-
-  <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/wellcome/logs"><code>logs</code></a>
-  </dt>
-  <dd>
-    open an app’s logs in our shared logging cluster.
-    This queries the logging cluster to find all the possible app names, then offers me a searchable list.
-    When I select an item, it opens a query for that app’s logs in our logging system.
-    <img src="screenshots/logs.png">
-  </dd>
-
-
-  <dt>
-    <a href="https://github.com/alexwlchan/scripts/blob/main/wellcome/ssh_to_archivematica"><code>ssh_to_archivematica</code></a>
-  </dt>
-  <dd>
-    ssh into the container hosts we use to run our <a href="https://github.com/wellcomecollection/archivematica-infrastructure">Archivematica instances</a>.
-  </dd>
-</dl>

wellcome/bag (106) → wellcome/bag (0)

diff --git a/wellcome/bag b/wellcome/bag
deleted file mode 100755
index a2d1bda..0000000
--- a/wellcome/bag
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-s3tree "s3://wellcomecollection-storage/digitised/$@"

wellcome/logs (158) → wellcome/logs (0)

diff --git a/wellcome/logs b/wellcome/logs
deleted file mode 100755
index 2dce463..0000000
--- a/wellcome/logs
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-python3 ~/repos/platform-infrastructure/logging/open_logging_cluster.py

wellcome/screenshots/bag.png (514043) → wellcome/screenshots/bag.png (0)

diff --git a/wellcome/screenshots/bag.png b/wellcome/screenshots/bag.png
deleted file mode 100644
index d4ec061..0000000
Binary files a/wellcome/screenshots/bag.png and /dev/null differ

wellcome/screenshots/logs.png (646328) → wellcome/screenshots/logs.png (0)

diff --git a/wellcome/screenshots/logs.png b/wellcome/screenshots/logs.png
deleted file mode 100644
index 98008c3..0000000
Binary files a/wellcome/screenshots/logs.png and /dev/null differ

wellcome/ssh_to_archivematica (158) → wellcome/ssh_to_archivematica (0)

diff --git a/wellcome/ssh_to_archivematica b/wellcome/ssh_to_archivematica
deleted file mode 100755
index b5a6432..0000000
--- a/wellcome/ssh_to_archivematica
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-set -o errexit
-set -o nounset
-
-_ensure_aws_credentials_are_fresh
-
-~/repos/archivematica-infrastructure/scripts/ssh_to_archivematica "$@"

wellcome/wellcome (317) → wellcome/wellcome (0)

diff --git a/wellcome/wellcome b/wellcome/wellcome
deleted file mode 100755
index 4020962..0000000
--- a/wellcome/wellcome
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-# This is a shortcut for 'mv' that I use for filing documents.
-#
-# I have a 'wellcome' directory that's divided into years where I put
-# work documents; running `wellcome nameofdocument.pdf` will move it
-# to the appropriate folder for the current year.
-
-_file_by_year "~/Documents/wellcome" "$@"