Merge pull request #225 from alexwlchan/python-scripts
- ID
bdd1078- date
2025-04-23 07:25:11+00:00- author
Alex Chan <alex@alexwlchan.net>- parents
2ce5734,408201d- message
Merge pull request #225 from alexwlchan/python-scripts Improve my Python scripts- changed files
Changed files
config.fish (7003) → config.fish (7042)
diff --git a/config.fish b/config.fish
index b97f102..36a8aa5 100644
--- a/config.fish
+++ b/config.fish
@@ -55,6 +55,7 @@ prepend_to_path ~/repos/scripts/git
prepend_to_path ~/repos/scripts/images
prepend_to_path ~/repos/scripts/installers
prepend_to_path ~/repos/scripts/macos
+prepend_to_path ~/repos/scripts/python
prepend_to_path ~/repos/scripts/terraform
prepend_to_path ~/repos/scripts/text
prepend_to_path ~/repos/scripts/web
fish_functions/README.md (4537) → fish_functions/README.md (3898)
diff --git a/fish_functions/README.md b/fish_functions/README.md
index 936d60d..3555605 100644
--- a/fish_functions/README.md
+++ b/fish_functions/README.md
@@ -105,15 +105,6 @@ cog_helpers.create_description_table(folder_name=folder_name, scripts=functions)
</dd>
<dt>
- <a href="https://github.com/alexwlchan/scripts/blob/main/fish_functions/pip_compile.fish">
- <code>pip_compile.fish</code>
- </a>
- </dt>
- <dd>
- Create requirements.txt lock files with uv
- </dd>
-
- <dt>
<a href="https://github.com/alexwlchan/scripts/blob/main/fish_functions/pip_sync.fish">
<code>pip_sync.fish</code>
</a>
@@ -123,24 +114,6 @@ cog_helpers.create_description_table(folder_name=folder_name, scripts=functions)
</dd>
<dt>
- <a href="https://github.com/alexwlchan/scripts/blob/main/fish_functions/pip_upgrade.fish">
- <code>pip_upgrade.fish</code>
- </a>
- </dt>
- <dd>
- Upgrade requirements.txt lock files with uv
- </dd>
-
- <dt>
- <a href="https://github.com/alexwlchan/scripts/blob/main/fish_functions/pyfmt.fish">
- <code>pyfmt.fish</code>
- </a>
- </dt>
- <dd>
- Run Python formatting over a directory
- </dd>
-
- <dt>
<a href="https://github.com/alexwlchan/scripts/blob/main/fish_functions/reload_fish_config.fish">
<code>reload_fish_config.fish</code>
</a>
@@ -167,4 +140,4 @@ cog_helpers.create_description_table(folder_name=folder_name, scripts=functions)
Create and activate a new virtual environment
</dd>
</dl>
-<!-- [[[end]]] (checksum: 2019ea5d74d78cc1b4f84da5403e077d) -->
\ No newline at end of file
+<!-- [[[end]]] (checksum: 25bdcd02dce6ff80dc797668b840c5bb) -->
\ No newline at end of file
fish_functions/pip_compile.fish (665) → fish_functions/pip_compile.fish (0)
diff --git a/fish_functions/pip_compile.fish b/fish_functions/pip_compile.fish
deleted file mode 100644
index c9ca008..0000000
--- a/fish_functions/pip_compile.fish
+++ /dev/null
@@ -1,13 +0,0 @@
-function pip_compile --description "Create requirements.txt lock files with uv"
- if test \( -e requirements.in \) -a \( -e overrides.txt \)
- uv pip compile requirements.in --output-file requirements.txt --override overrides.txt
- else if test \( -e requirements.in \)
- uv pip compile requirements.in --output-file requirements.txt
- end
-
- if test \( -e dev_requirements.in \) -a \( -e overrides.txt \)
- uv pip compile dev_requirements.in --output-file dev_requirements.txt --override overrides.txt
- else if test \( -e dev_requirements.in \)
- uv pip compile dev_requirements.in --output-file dev_requirements.txt
- end
-end
fish_functions/pip_sync.fish (390) → fish_functions/pip_sync.fish (713)
diff --git a/fish_functions/pip_sync.fish b/fish_functions/pip_sync.fish
index 196f7a3..16ef1f4 100644
--- a/fish_functions/pip_sync.fish
+++ b/fish_functions/pip_sync.fish
@@ -1,14 +1,24 @@
function pip_sync --description "Make a virtualenv dependencies look like requirements.txt"
- pip_compile
+
+ # Run the `pip compile` script to get a set of version pins.
+ if contains -- --upgrade $argv
+ pip_compile --upgrade
+ else
+ pip_compile
+ end
# If there isn't a virtualenv already, create one
if test -z "$VIRTUAL_ENV"
venv
end
+
+ echo ""
if test \( -e dev_requirements.txt \)
+ ~/repos/scripts/debug/print_info "-> uv pip sync dev_requirements.txt"
uv pip sync dev_requirements.txt
else if test \( -e requirements.txt \)
+ ~/repos/scripts/debug/print_info "-> uv pip sync requirements.txt"
uv pip sync requirements.txt
end
end
fish_functions/pip_upgrade.fish (706) → fish_functions/pip_upgrade.fish (0)
diff --git a/fish_functions/pip_upgrade.fish b/fish_functions/pip_upgrade.fish
deleted file mode 100644
index d1d3f4e..0000000
--- a/fish_functions/pip_upgrade.fish
+++ /dev/null
@@ -1,13 +0,0 @@
-function pip_upgrade --description "Upgrade requirements.txt lock files with uv"
- if test \( -e requirements.in \) -a \( -e overrides.txt \)
- uv pip compile requirements.in --output-file requirements.txt --override overrides.txt --upgrade
- else if test \( -e requirements.in \)
- uv pip compile requirements.in --output-file requirements.txt --upgrade
- end
-
- if test \( -e dev_requirements.in \) -a \( -e overrides.txt \)
- uv pip compile dev_requirements.in --output-file dev_requirements.txt --override overrides.txt --upgrade
- else if test \( -e dev_requirements.in \)
- uv pip compile dev_requirements.in --output-file dev_requirements.txt --upgrade
- end
-end
fish_functions/pyfmt.fish (363) → fish_functions/pyfmt.fish (0)
diff --git a/fish_functions/pyfmt.fish b/fish_functions/pyfmt.fish
deleted file mode 100644
index e7306f7..0000000
--- a/fish_functions/pyfmt.fish
+++ /dev/null
@@ -1,18 +0,0 @@
-function _run_ruff
- if which ruff >/dev/null
- ruff $argv
- else
- ~/repos/scripts/.venv/bin/ruff $argv
- end
-end
-
-function pyfmt --description "Run Python formatting over a directory"
- if test (count $argv) -eq 0
- set root $PWD
- else
- set root $argv[1]
- end
-
- _run_ruff check "$root"
- _run_ruff format "$root"
-end
fish_functions/venv.fish (886) → fish_functions/venv.fish (1142)
diff --git a/fish_functions/venv.fish b/fish_functions/venv.fish
index c1ff69c..4327dd4 100644
--- a/fish_functions/venv.fish
+++ b/fish_functions/venv.fish
@@ -4,6 +4,8 @@
# creating the venv and then immediately running "pip install" without
# activating it first.
#
+# This has to be a fish function, because I'm activating the venv.
+#
# See https://alexwlchan.net/2023/fish-venv/
#
function venv --description "Create and activate a new virtual environment"
@@ -13,7 +15,7 @@ function venv --description "Create and activate a new virtual environment"
cd $(mktemp -d)
end
- echo "Creating virtual environment in "(pwd)"/.venv"
+ print_info "Creating virtual environment in "(pwd)"/.venv"
uv venv --quiet .venv
source .venv/bin/activate.fish
@@ -25,5 +27,10 @@ function venv --description "Create and activate a new virtual environment"
# Tell Time Machine that it doesn't need to both backing up the
# virtualenv directory.
- tmutil addexclusion .venv
+ #
+ # Note: this is quite slow, so we only run it if we're in my home
+ # directory -- it won't get backed up otherwise.
+ if string match -q "$HOME/*" "$PWD"
+ tmutil addexclusion .venv
+ end
end
python/README.md (0) → python/README.md (1207)
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 0000000..091f94e
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,52 @@
+# python
+
+These scripts are all for working with Python.
+
+## The individual scripts
+
+<!-- [[[cog
+
+# This adds the root of the repo to the PATH, which has cog_helpers.py
+from os.path import abspath, dirname
+import sys
+
+sys.path.append(abspath(dirname(dirname("."))))
+
+import cog_helpers
+
+folder_name = "python"
+
+scripts = [
+ {
+ "name": "pyfmt [...PATH]",
+ "description": "Format Python files with ruff.",
+ },
+ {
+ "name": "pip_compile (--upgrade)",
+ "description": "Compile any `requirements.in` files into a list of exact versions in `requirements.txt`.",
+ },
+]
+
+cog_helpers.create_description_table(folder_name=folder_name, scripts=scripts)
+
+]]]-->
+<dl>
+ <dt>
+ <a href="https://github.com/alexwlchan/scripts/blob/main/python/pyfmt">
+ <code>pyfmt [...PATH]</code>
+ </a>
+ </dt>
+ <dd>
+ Format Python files with ruff.
+ </dd>
+
+ <dt>
+ <a href="https://github.com/alexwlchan/scripts/blob/main/python/pip_compile">
+ <code>pip_compile (--upgrade)</code>
+ </a>
+ </dt>
+ <dd>
+ Compile any `requirements.in` files into a list of exact versions in `requirements.txt`.
+ </dd>
+</dl>
+<!-- [[[end]]] (checksum: 3e15017c8de1c144defe8f18a08d38b1) -->
python/pip_compile (0) → python/pip_compile (1278)
diff --git a/python/pip_compile b/python/pip_compile
new file mode 100755
index 0000000..ef18e0e
--- /dev/null
+++ b/python/pip_compile
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""
+Compile any `requirements.in` files into a list of exact versions
+in `requirements.txt`.
+
+* If you pass the `--upgrade` flag, it will upgrade all the requirements
+ to the latest version.
+
+"""
+
+from pathlib import Path
+import shlex
+import subprocess
+import sys
+
+
+def compile_requirements_file(in_file: str, *, upgrade: bool) -> None:
+ """
+ Compile a single requirements file.
+ """
+ assert in_file.endswith(".in")
+ txt_file = in_file.replace(".in", ".txt")
+ assert in_file != txt_file
+
+ # If this `.in` file doesn't exist.
+ if not Path(in_file).exists():
+ return
+
+ # Construct the `uv pip compile` command.
+ cmd = ["uv", "pip", "compile", in_file, "--output-file", txt_file]
+
+ if upgrade:
+ cmd.append("--upgrade")
+
+ # Actually run the command, and print a debug entry for it
+ subprocess.check_call(
+ ["/Users/alexwlchan/repos/scripts/debug/print_info", f"-> {shlex.join(cmd)}"]
+ )
+ subprocess.check_call(cmd)
+
+
+if __name__ == "__main__":
+ upgrade = "--upgrade" in sys.argv
+
+ compile_requirements_file("requirements.in", upgrade=upgrade)
+ compile_requirements_file("script_requirements.in", upgrade=upgrade)
+ compile_requirements_file("dev_requirements.in", upgrade=upgrade)
python/pyfmt (0) → python/pyfmt (851)
diff --git a/python/pyfmt b/python/pyfmt
new file mode 100755
index 0000000..80e9e41
--- /dev/null
+++ b/python/pyfmt
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Lint and format Python files with `ruff`.
+#
+# This will format files in the working directory by default, or you
+# can supply one or more paths for it to check.
+
+set -o errexit
+set -o nounset
+
+
+# Work out which directory to run in:
+#
+# * if no argument is specified, use the current working directory
+# * if one or more directories are specified, use those
+#
+if (( $# == 0 ))
+then
+ ROOT="$(pwd)"
+else
+ ROOT="$@"
+fi
+
+
+
+# Work out which instance of ruff to use
+#
+# If we're in a virtualenv which has ruff installed, we use the locally
+# installed copy.
+#
+# If not, we use the `ruff` from my ~/repos/scripts virtualenv.
+if which ruff
+then
+ RUFF="$(which ruff)"
+else
+ RUFF=~/repos/scripts/.venv/bin/ruff
+fi
+
+
+print_info "ruff check"
+bash -c "$RUFF check $ROOT"
+
+echo ""
+
+print_info "ruff format"
+bash -c "$RUFF format $ROOT"
requirements.txt (3792) → requirements.txt (3360)
diff --git a/requirements.txt b/requirements.txt
index fbe01f0..1b10f9b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,34 +1,30 @@
# This file was autogenerated by uv via the following command:
# uv pip compile requirements.in --output-file requirements.txt
-aiofiles==23.2.1
+aiofiles==24.1.0
# via datasette
-anyio==4.1.0
+anyio==4.9.0
# via httpx
-argcomplete==3.2.3
+argcomplete==3.6.2
# via pipx
-asgi-csrf==0.9
+asgi-csrf==0.11
# via datasette
-asgiref==3.7.2
+asgiref==3.8.1
# via datasette
attrs==25.3.0
# via -r requirements.in
-backports-tarfile==1.2.0
- # via jaraco-context
beautifulsoup4==4.13.4
# via -r requirements.in
-boto3==1.35.26
+boto3==1.38.0
# via -r requirements.in
-botocore==1.35.26
+botocore==1.38.0
# via
# boto3
# s3transfer
-certifi==2023.11.17
+certifi==2025.1.31
# via
# httpcore
# httpx
-cffi==1.16.0
- # via cryptography
-click==8.1.7
+click==8.1.8
# via
# click-default-group
# datasette
@@ -41,27 +37,21 @@ click-default-group==1.2.4
# sqlite-utils
cogapp==3.4.1
# via -r requirements.in
-cryptography==42.0.7
- # via secretstorage
datasette==0.65.1
# via
# -r requirements.in
# datasette-render-image-tags
datasette-render-image-tags==0.1
# via -r requirements.in
-exceptiongroup==1.2.2
- # via
- # anyio
- # pytest
flexcache==0.3
# via datasette
-flexparser==0.3.1
+flexparser==0.4
# via datasette
h11==0.14.0
# via
# httpcore
# uvicorn
-httpcore==1.0.2
+httpcore==1.0.8
# via httpx
httpx==0.28.1
# via
@@ -69,36 +59,30 @@ httpx==0.28.1
# datasette
humanize==4.12.2
# via -r requirements.in
-hupper==1.12
+hupper==1.12.1
# via datasette
hyperlink==21.0.0
# via -r requirements.in
-idna==3.6
+idna==3.10
# via
# anyio
# httpx
# hyperlink
-importlib-metadata==8.6.1
- # via keyring
-iniconfig==2.0.0
+iniconfig==2.1.0
# via pytest
-itsdangerous==2.1.2
+itsdangerous==2.2.0
# via
# asgi-csrf
# datasette
-janus==1.0.0
+janus==2.0.0
# via datasette
-jaraco-classes==3.3.0
+jaraco-classes==3.4.0
# via keyring
-jaraco-context==5.3.0
+jaraco-context==6.0.1
# via keyring
-jaraco-functools==4.0.1
+jaraco-functools==4.1.0
# via keyring
-jeepney==0.8.0
- # via
- # keyring
- # secretstorage
-jinja2==3.1.3
+jinja2==3.1.6
# via datasette
jmespath==1.0.1
# via
@@ -106,17 +90,17 @@ jmespath==1.0.1
# botocore
keyring==25.6.0
# via -r requirements.in
-markupsafe==2.1.3
+markupsafe==3.0.2
# via jinja2
mergedeep==1.3.4
# via datasette
-more-itertools==10.1.0
+more-itertools==10.7.0
# via
# jaraco-classes
# jaraco-functools
naturalsort==1.5.1
# via -r requirements.in
-packaging==23.2
+packaging==25.0
# via
# pipx
# pytest
@@ -126,9 +110,11 @@ pillow==11.2.1
# pillow-heif
pillow-heif==0.22.0
# via -r requirements.in
+pip==25.0.1
+ # via datasette
pipx==1.7.1
# via -r requirements.in
-platformdirs==4.1.0
+platformdirs==4.3.7
# via
# datasette
# pipx
@@ -137,33 +123,31 @@ pluggy==1.5.0
# datasette
# pytest
# sqlite-utils
-pycparser==2.22
- # via cffi
pygments==2.19.1
# via -r requirements.in
pypdf==5.4.0
# via -r requirements.in
pytest==8.3.5
# via -r requirements.in
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
# via
# botocore
# sqlite-utils
-python-multipart==0.0.6
+python-multipart==0.0.20
# via asgi-csrf
-pyyaml==6.0.1
+pyyaml==6.0.2
# via datasette
ruff==0.11.6
# via -r requirements.in
-s3transfer==0.10.2
+s3transfer==0.12.0
# via boto3
-secretstorage==3.3.3
- # via keyring
-six==1.16.0
+setuptools==79.0.0
+ # via datasette
+six==1.17.0
# via python-dateutil
-sniffio==1.3.0
+sniffio==1.3.1
# via anyio
-soupsieve==2.5
+soupsieve==2.7
# via beautifulsoup4
sqlite-fts4==1.0.3
# via sqlite-utils
@@ -173,29 +157,20 @@ tabulate==0.9.0
# via sqlite-utils
termcolor==3.0.1
# via -r requirements.in
-tomli==2.2.1
- # via
- # pipx
- # pytest
tqdm==4.67.1
# via -r requirements.in
-typing-extensions==4.9.0
+typing-extensions==4.13.2
# via
- # asgiref
+ # anyio
# beautifulsoup4
# datasette
# flexcache
# flexparser
- # janus
- # pypdf
- # uvicorn
-urllib3==1.26.20
+urllib3==2.4.0
# via botocore
userpath==1.9.2
# via pipx
-uvicorn==0.25.0
+uvicorn==0.34.2
# via datasette
yt-dlp==2025.3.31
# via -r requirements.in
-zipp==3.21.0
- # via importlib-metadata