add a script for fast lookups in the logging cluster
- ID
425d245- date
2023-05-28 14:29:11+00:00- author
Alex Chan <alex@alexwlchan.net>- parent
9bee71f- message
add a script for fast lookups in the logging cluster closes #2- changed files
5 files, 193 additions, 10 deletions
Changed files
python/pip_freeze (2377) → python/pip_freeze (2579)
diff --git a/python/pip_freeze b/python/pip_freeze
index d37bcf0..a1e2197 100755
--- a/python/pip_freeze
+++ b/python/pip_freeze
@@ -34,7 +34,7 @@ import sys
def get_freeze_string(library_name):
- if library_name in {"os", "re", "subprocess", "sys", "tempfile"}:
+ if library_name in {"json", "os", "re", "subprocess", "sys", "tempfile"}:
return None
try:
@@ -56,19 +56,24 @@ if __name__ == "__main__":
lines = []
for line in open(infile):
- m = re.match(r"^import (?P<library_name>[a-zA-Z]+)\n$", line)
+ m1 = re.match(r"^import (?P<library_name>[a-zA-Z0-9]+)\n$", line)
+ m2 = re.match(r"^from (?P<library_name>[a-zA-Z0-9]+) import [^#]*$", line)
- if m is None:
+ if m1 is None and m2 is None:
lines.append(line)
- else:
- library_name = m.group("library_name")
+ continue
+
+ if m1 is not None:
+ library_name = m1.group("library_name")
+ elif m2 is not None:
+ library_name = m2.group("library_name")
- freeze = get_freeze_string(library_name)
+ freeze = get_freeze_string(library_name)
- if freeze:
- lines.append(f"import {library_name} # {freeze}\n")
- else:
- lines.append(line)
+ if freeze:
+ lines.append(f"{line.rstrip()} # {freeze}\n")
+ else:
+ lines.append(line)
# Note: the original implementation wrote the modified script to
# a temporary file first, then os.rename-d that over the original.
wellcome/README.md (1038) → wellcome/README.md (1457)
diff --git a/wellcome/README.md b/wellcome/README.md
index 269795b..6b6bad2 100644
--- a/wellcome/README.md
+++ b/wellcome/README.md
@@ -18,6 +18,17 @@ It's unlikely these would be of any use to somebody not at Wellcome (except as a
</dd>
<dt>
+ <a href="https://github.com/alexwlchan/scripts/blob/main/wellcome/logs"><code>logs</code></a>
+ </dt>
+ <dd>
+ open an app’s logs in our shared logging cluster.
+ This queries the logging cluster to find all the possible app names, then offers me a searchable list.
+ When I select an item, it opens a query for that app’s logs in our logging system.
+ <img src="screenshots/logs.png">
+ </dd>
+
+
+ <dt>
<a href="https://github.com/alexwlchan/scripts/blob/main/wellcome/ssh_to_archivematica"><code>ssh_to_archivematica</code></a>
</dt>
<dd>
wellcome/logs (0) → wellcome/logs (110)
diff --git a/wellcome/logs b/wellcome/logs
new file mode 100755
index 0000000..565fc1b
--- /dev/null
+++ b/wellcome/logs
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+
+_ensure_aws_credentials_are_fresh
+open_logging_cluster.py
wellcome/open_logging_cluster.py (0) → wellcome/open_logging_cluster.py (6208)
diff --git a/wellcome/open_logging_cluster.py b/wellcome/open_logging_cluster.py
new file mode 100755
index 0000000..c7ca09f
--- /dev/null
+++ b/wellcome/open_logging_cluster.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""
+Open an app’s logs in the logging cluster.
+
+This script will:
+
+1. Query the logging cluster for all ECS service/cluster names and all
+ Lambda function names
+
+2. Offer them to the user as a pick list
+
+3. When the user selects an item, open Kibana in their web browser with
+ appropriate filters for the selected item
+
+"""
+
+import json
+import os
+import sys
+import webbrowser
+
+import boto3 # boto3==1.24.85
+import httpx # httpx==0.24.1
+from iterfzf import iterfzf # iterfzf==0.5.0.20.0
+
+
+# Where to cache data between runs.
+#
+# Because running the ES query is moderately slow, the script caches
+# lookups here -- this allows the user to start selecting an item before
+# the latest data is loaded from Elasticsearch.
+CACHE_FILE = os.path.join(os.environ["HOME"], ".logging_cluster.json")
+
+
+def get_aws_session(*, role_arn):
+ sts_client = boto3.client("sts")
+ assumed_role_object = sts_client.assume_role(
+ RoleArn=role_arn, RoleSessionName="AssumeRoleSession1"
+ )
+ credentials = assumed_role_object["Credentials"]
+
+ return boto3.Session(
+ aws_access_key_id=credentials["AccessKeyId"],
+ aws_secret_access_key=credentials["SecretAccessKey"],
+ aws_session_token=credentials["SessionToken"],
+ )
+
+
+def get_secret_string(sess, **kwargs):
+ """
+ Look up a SecretString from Secrets Manager, and return the string.
+ """
+ secrets = sess.client("secretsmanager")
+
+ resp = secrets.get_secret_value(**kwargs)
+
+ return resp["SecretString"]
+
+
+def get_logging_options_from_es():
+ sess = get_aws_session(role_arn="arn:aws:iam::760097843905:role/platform-developer")
+
+ logging_config = {
+ key: get_secret_string(sess, SecretId=f"shared/logging/es_{key}")
+ for key in ("host", "port", "user", "pass")
+ }
+
+ endpoint = f"https://{logging_config['host']}:{logging_config['port']}"
+
+ resp = httpx.request(
+ "GET",
+ f"{endpoint}/service-logs-*/_search",
+ auth=(logging_config["user"], logging_config["pass"]),
+ json={
+ "size": 0,
+ "aggs": {
+ "ecs_services": {
+ "terms": {"field": "service_name.keyword", "size": 1000},
+ "aggs": {
+ "ecs_cluster": {
+ "terms": {"field": "ecs_cluster.keyword", "size": 1}
+ }
+ },
+ },
+ "lambdas": {"terms": {"field": "service.keyword", "size": 100}},
+ },
+ },
+ ).json()
+
+ result = {}
+
+ for bucket in resp["aggregations"]["ecs_services"]["buckets"]:
+ service_name = bucket["key"]
+ cluster_name = bucket["ecs_cluster"]["buckets"][0]["key"]
+ label = f"{service_name} ({cluster_name})"
+
+ result[cluster_name] = {
+ "type": "ecs_cluster",
+ "cluster_name": cluster_name,
+ }
+
+ result[label] = {
+ "type": "ecs_service",
+ "service_name": service_name,
+ "cluster_name": cluster_name,
+ }
+
+ for bucket in resp["aggregations"]["lambdas"]["buckets"]:
+ function_name = bucket["key"]
+ result[function_name] = {
+ "type": "lambda",
+ "function_name": function_name,
+ }
+
+ return result
+
+
+def get_logging_options():
+ try:
+ with open(CACHE_FILE) as infile:
+ cached_entries = json.load(infile)
+ except FileNotFoundError:
+ cached_entries = {}
+
+ yield from cached_entries.items()
+
+ new_entries = get_logging_options_from_es()
+
+ for k, v in new_entries.items():
+ if k not in cached_entries:
+ yield (k, v)
+
+ updated_entries = {**cached_entries, **new_entries}
+
+ with open(CACHE_FILE, "w") as outfile:
+ outfile.write(json.dumps(updated_entries, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ choice = iterfzf(label for (label, _) in get_logging_options())
+
+ if choice is None:
+ sys.exit(0)
+
+ with open(CACHE_FILE) as infile:
+ es_info = json.load(infile)[choice]
+
+ if es_info["type"] == "ecs_service":
+ cluster_name = es_info["cluster_name"]
+ service_name = es_info["service_name"]
+
+ url = f"https://logging.wellcomecollection.org/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))&_a=(columns:!(log),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,key:ecs_cluster,negate:!f,params:(query:{cluster_name}),type:phrase),query:(match_phrase:(ecs_cluster:{cluster_name}))),('$state':(store:appState),meta:(alias:!n,disabled:!f,index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,key:service_name,negate:!f,params:(query:{service_name}),type:phrase),query:(match_phrase:(service_name:{service_name})))),index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,interval:auto,query:(language:kuery,query:''),sort:!(!('@timestamp',desc)))"
+ elif es_info["type"] == "ecs_cluster":
+ cluster_name = es_info["cluster_name"]
+ url = f"https://logging.wellcomecollection.org/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))&_a=(columns:!(service_name,log),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,key:ecs_cluster,negate:!f,params:(query:{cluster_name}),type:phrase),query:(match_phrase:(ecs_cluster:{cluster_name})))),index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,interval:auto,query:(language:kuery,query:''),sort:!(!('@timestamp',desc)))"
+ elif es_info["type"] == "lambda":
+ function_name = es_info["function_name"].replace("/", "%2F")
+ url = f"https://logging.wellcomecollection.org/app/discover#/?_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))&_a=(columns:!(log),filters:!(('$state':(store:appState),meta:(alias:!n,disabled:!f,index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,key:service,negate:!f,params:(query:{function_name}),type:phrase),query:(match_phrase:(service:{function_name})))),index:cb5ba262-ec15-46e3-a4c5-5668d65fe21f,interval:auto,query:(language:kuery,query:''),sort:!(!('@timestamp',desc)))"
+
+ webbrowser.open(url)
wellcome/screenshots/logs.png (0) → wellcome/screenshots/logs.png (646328)
diff --git a/wellcome/screenshots/logs.png b/wellcome/screenshots/logs.png
new file mode 100644
index 0000000..98008c3
Binary files /dev/null and b/wellcome/screenshots/logs.png differ