web: remove my last httpx dependency
- ID
75c68a8- date
2026-03-27 14:36:30+00:00- author
Alex Chan <alex@alexwlchan.net>- parent
b23dbfb- message
web: remove my last httpx dependency- changed files
3 files, 25 additions, 85 deletions
Changed files
requirements.in (264) → requirements.in (256)
diff --git a/requirements.in b/requirements.in
index 870fcb4..d51cb31 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,7 +1,6 @@
beautifulsoup4
+certifi
cogapp
-datasette
-httpx
humanize
hyperlink
iterm2
requirements.txt (3318) → requirements.txt (2281)
diff --git a/requirements.txt b/requirements.txt
index 14bb05e..50c62e0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,89 +1,44 @@
# This file was autogenerated by uv via the following command:
-# uv pip compile requirements.in --output-file requirements.txt
-aiofiles==25.1.0
- # via datasette
-anyio==4.12.1
- # via httpx
-asgi-csrf==0.11
- # via datasette
-asgiref==3.11.1
- # via datasette
+# uv pip compile requirements.in --output-file=requirements.txt --exclude-newer=P7D
beautifulsoup4==4.14.3
# via -r requirements.in
brotli==1.2.0
# via yt-dlp
-certifi==2026.1.4
+certifi==2026.2.25
# via
- # httpcore
- # httpx
+ # -r requirements.in
# requests
# yt-dlp
-charset-normalizer==3.4.4
+charset-normalizer==3.4.6
# via requests
click==8.3.1
# via
# click-default-group
- # datasette
# sqlite-utils
- # uvicorn
click-default-group==1.2.4
- # via
- # datasette
- # sqlite-utils
+ # via sqlite-utils
cogapp==3.6.0
# via -r requirements.in
-datasette==0.65.2
- # via -r requirements.in
-flexcache==0.3
- # via datasette
-flexparser==0.4
- # via datasette
-h11==0.16.0
- # via
- # httpcore
- # uvicorn
-httpcore==1.0.9
- # via httpx
-httpx==0.28.1
- # via
- # -r requirements.in
- # datasette
humanize==4.15.0
# via -r requirements.in
-hupper==1.12.1
- # via datasette
hyperlink==21.0.0
# via -r requirements.in
idna==3.11
# via
- # anyio
- # httpx
# hyperlink
# requests
iniconfig==2.3.0
# via pytest
-iterm2==2.13
+iterm2==2.14
# via -r requirements.in
-itsdangerous==2.2.0
- # via
- # asgi-csrf
- # datasette
-janus==2.0.0
- # via datasette
jaraco-classes==3.4.0
# via keyring
-jaraco-context==6.1.0
+jaraco-context==6.1.1
# via keyring
jaraco-functools==4.4.0
# via keyring
-jinja2==3.1.6
- # via datasette
keyring==25.7.0
# via -r requirements.in
-markupsafe==3.0.3
- # via jinja2
-mergedeep==1.3.4
- # via datasette
more-itertools==10.8.0
# via
# jaraco-classes
@@ -98,20 +53,15 @@ pillow==12.1.1
# via
# -r requirements.in
# pillow-heif
-pillow-heif==1.2.0
+pillow-heif==1.3.0
# via -r requirements.in
pip==26.0.1
- # via
- # datasette
- # sqlite-utils
-platformdirs==4.6.0
- # via datasette
+ # via sqlite-utils
pluggy==1.6.0
# via
- # datasette
# pytest
# sqlite-utils
-protobuf==6.33.5
+protobuf==7.34.0
# via iterm2
pycryptodomex==3.23.0
# via yt-dlp
@@ -119,22 +69,16 @@ pygments==2.19.2
# via
# -r requirements.in
# pytest
-pypdf==6.7.0
+pypdf==6.9.1
# via -r requirements.in
pytest==9.0.2
# via -r requirements.in
python-dateutil==2.9.0.post0
# via sqlite-utils
-python-multipart==0.0.22
- # via asgi-csrf
-pyyaml==6.0.3
- # via datasette
requests==2.32.5
# via yt-dlp
-ruff==0.15.0
+ruff==0.15.7
# via -r requirements.in
-setuptools==82.0.0
- # via datasette
six==1.17.0
# via python-dateutil
smartypants==2.0.2
@@ -145,30 +89,23 @@ sqlite-fts4==1.0.3
# via sqlite-utils
sqlite-utils==3.39
# via -r requirements.in
-tabulate==0.9.0
+tabulate==0.10.0
# via sqlite-utils
termcolor==3.3.0
# via -r requirements.in
tqdm==4.67.3
# via -r requirements.in
typing-extensions==4.15.0
- # via
- # anyio
- # beautifulsoup4
- # datasette
- # flexcache
- # flexparser
+ # via beautifulsoup4
urllib3==2.6.3
# via
# requests
# yt-dlp
-uvicorn==0.40.0
- # via datasette
websockets==16.0
# via
# iterm2
# yt-dlp
-yt-dlp==2026.2.4
+yt-dlp==2026.3.17
# via -r requirements.in
-yt-dlp-ejs==0.4.0
+yt-dlp-ejs==0.8.0
# via yt-dlp
web/scrape_really_useful_boxes.py (3817) → web/scrape_really_useful_boxes.py (3972)
diff --git a/web/scrape_really_useful_boxes.py b/web/scrape_really_useful_boxes.py
index b73daad..bb2f50e 100755
--- a/web/scrape_really_useful_boxes.py
+++ b/web/scrape_really_useful_boxes.py
@@ -13,10 +13,13 @@ See https://social.alexwlchan.net/@alex/111750446474991705
from collections.abc import Iterator
import time
+import ssl
from typing import TypedDict
from urllib.parse import urljoin
+import urllib.request
import bs4
+import certifi
import httpx
from sqlite_utils import Database
import tqdm
@@ -29,10 +32,11 @@ def get_soup(url: str) -> bs4.BeautifulSoup:
"""
Fetch the contents of a URL, parse it as HTML, and return the parsed soup.
"""
- resp = client.get(url)
- resp.raise_for_status()
-
- soup = bs4.BeautifulSoup(resp.text, "html.parser")
+ ssl_context = ssl.create_default_context(cafile=certifi.where())
+ with urllib.request.urlopen(url, context=ssl_context) as resp:
+ html = resp.read()
+
+ soup = bs4.BeautifulSoup(html, "html.parser")
return soup