storage: add an SQLite implementation of CacheStorage
- ID
280becb- date
2026-11-06 06:45:10+00:00- author
Alex Chan <alex@alexwlchan.net>- parent
fc136b6- message
storage: add an SQLite implementation of CacheStorage This serialises function arguments as an xxh64-hashed JSON string, which ensures consistent insertion/lookup, which has faster insert and lookup performance than using the raw JSON string as the key. Output values are serialised and deserialised using Pydantic.- changed files
4 files, 220 additions, 5 deletions
Changed files
dev_requirements.txt (568 → 869)
diff --git a/dev_requirements.txt b/dev_requirements.txt
index 86749b0..2442d89 100644
--- a/dev_requirements.txt
+++ b/dev_requirements.txt
@@ -2,6 +2,8 @@
# uv pip compile dev_requirements.in --output-file=dev_requirements.txt --exclude-newer=P7D --exclude-newer-package alexwlchan-chives=false
-e file:.
# via -r dev_requirements.in
+annotated-types==0.8.0
+ # via pydantic
coverage==7.17.0
# via -r dev_requirements.in
iniconfig==2.3.1
@@ -10,6 +12,10 @@ packaging==26.3
# via pytest
pluggy==1.7.0
# via pytest
+pydantic==2.14.2
+ # via funkycache
+pydantic-core==2.50.2
+ # via pydantic
pygments==2.21.0
# via pytest
pytest==9.2.0
@@ -18,3 +24,12 @@ ruff==0.17.3
# via -r dev_requirements.in
ty==0.0.93
# via -r dev_requirements.in
+typing-extensions==4.16.0
+ # via
+ # pydantic
+ # pydantic-core
+ # typing-inspection
+typing-inspection==0.4.4
+ # via pydantic
+xxhash==4.0.2
+ # via funkycache
pyproject.toml (1504 → 1471)
diff --git a/pyproject.toml b/pyproject.toml
index 07242c9..906fc5a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,13 +19,10 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
requires-python = ">=3.14"
-dependencies = []
+dependencies = ["pydantic", "xxhash"]
dynamic = ["version"]
license = "MIT"
-[project.optional-dependencies]
-xxhash = ["xxhash"]
-
[project.urls]
"Homepage" = "https://alexwlchan.net/projects/funkycache/"
"Changelog" = "https://alexwlchan.net/projects/funkycache/releases/"
src/funkycache/storage.py (2069 → 6427)
diff --git a/src/funkycache/storage.py b/src/funkycache/storage.py
index 8e5a473..7495dce 100644
--- a/src/funkycache/storage.py
+++ b/src/funkycache/storage.py
@@ -4,8 +4,15 @@ Backing storage for cache, which records the input/output of Python functions.
from abc import ABC, abstractmethod
from collections.abc import Mapping
+from functools import cache
import json
-from typing import Any
+from pathlib import Path
+import sqlite3
+from types import TracebackType
+from typing import Any, cast, Self
+
+from pydantic import TypeAdapter
+import xxhash
class CannotOverwriteValueException(ValueError):
@@ -76,3 +83,145 @@ class MemoryCacheStorage[V](CacheStorage[V]):
"""
enc_key = self._encode_key(key)
return self._underlying[enc_key]
+
+
+class TypedCacheStorage[V](CacheStorage[V]):
+ """
+ Cache storage that allows storing arbitrary Python types in backing
+ storage that only supports strings.
+ """
+
+ adapter: TypeAdapter[Any]
+
+ def __init__(self, *, data_type: Any) -> None:
+ self.adapter = TypeAdapter(data_type)
+
+ @abstractmethod
+ def _set_str(self, key: Mapping[str, Any], value: str) -> None:
+ """
+ Save a key-value pair in the cache, where the value is a string.
+
+ Throws an exception if you overwrite an existing key with
+ a different value.
+ """
+
+ def set(self, key: Mapping[str, Any], value: V) -> None:
+ """
+ Save a key-value pair in the cache.
+
+ Throws an exception if you overwrite an existing key with
+ a different value.
+ """
+ json_str = self.adapter.dump_json(value).decode("ascii")
+ self._set_str(key, json_str)
+
+ @abstractmethod
+ def _get_str(self, key: Mapping[str, Any]) -> str:
+ """
+ Look up the string value of a key in the cache.
+
+ Throws a KeyError if you look up a key which isn't in the cache.
+ """
+
+ def get(self, key: Mapping[str, Any]) -> V:
+ """
+ Look up the value of a key in the cache.
+
+ Throws a KeyError if you look up a key which isn't in the cache.
+ """
+ str_value = self._get_str(key)
+ return self.adapter.validate_json(str_value)
+
+
+class SQLiteCacheStorage[V](TypedCacheStorage[V]):
+ """
+ SQLite-backed storage for a cache of the input/output of
+ a single Python function.
+ """
+
+ con: sqlite3.Connection
+
+ def __init__(self, database: str | Path, *, data_type: Any):
+ self.con = sqlite3.connect(database)
+ self.con.execute("PRAGMA journal_mode = WAL;")
+ self.con.execute("PRAGMA synchronous = NORMAL;")
+
+ cursor = self.con.cursor()
+ cursor.execute("CREATE TABLE IF NOT EXISTS funkycache(key PRIMARY KEY, value)")
+ self.con.commit()
+ super().__init__(data_type=data_type)
+
+ def close(self) -> None:
+ """
+ Close the underlying SQLite connection.
+ """
+ self.con.close()
+
+ def __enter__(self) -> Self:
+ """
+ Allow using SQLiteCacheStorage as a context manager.
+ """
+ return self
+
+ def __exit__(
+ self,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> None:
+ """
+ Automatically close the connection upon exiting the context.
+ """
+ self.close()
+
+ def _encode_key(self, key: Mapping[str, Any]) -> str:
+ """
+ Encode a mapping as a string key.
+ """
+ json_key = json.dumps(key, sort_keys=True, default=str)
+ return _hash(json_key)
+
+ def _set_str(self, key: Mapping[str, Any], value: str) -> None:
+ """
+ Save a key-value pair in the cache, where the value is a string.
+
+ Throws an exception if you overwrite an existing key with
+ a different value.
+ """
+ enc_key = self._encode_key(key)
+
+ try:
+ self.con.execute("INSERT INTO funkycache VALUES (?,?)", (enc_key, value))
+ self.con.commit()
+ except sqlite3.IntegrityError as err:
+ if err.sqlite_errorname == "SQLITE_CONSTRAINT_PRIMARYKEY":
+ if self._get_str(key) == value:
+ return
+ else:
+ raise CannotOverwriteValueException(key)
+ else:
+ raise
+
+ def _get_str(self, key: Mapping[str, Any]) -> str:
+ """
+ Look up the string value of a key in the cache.
+
+ Throws a KeyError if you look up a key which isn't in the cache.
+ """
+ enc_key = self._encode_key(key)
+
+ res = self.con.execute("SELECT value FROM funkycache WHERE key=?", (enc_key,))
+
+ try:
+ (value,) = res.fetchone()
+ return cast(str, value)
+ except TypeError:
+ raise KeyError(key)
+
+
+@cache
+def _hash(s: str) -> str:
+ """
+ Hash a string to reduce it to a fixed size.
+ """
+ return "xxh64:" + xxhash.xxh64_hexdigest(s.encode("utf8"))
tests/test_storage.py (2236 → 4073)
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 626d742..c448a19 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -5,13 +5,16 @@ Tests for backend storage implementations.
from abc import ABC, abstractmethod
from collections.abc import Iterator, Mapping
from pathlib import Path
+import sqlite3
from typing import Any
+from unittest.mock import MagicMock
import pytest
from funkycache.storage import (
CacheStorage,
MemoryCacheStorage,
+ SQLiteCacheStorage,
CannotOverwriteValueException,
)
@@ -80,3 +83,54 @@ class TestMemoryCacheStorage(CacheStorageTestSuite):
Return an empty in-memory cache that returns integer values.
"""
yield MemoryCacheStorage[int]()
+
+
+class TestSQLiteCacheStorage(CacheStorageTestSuite):
+ """
+ Tests for SQLiteCacheStorage.
+ """
+
+ @pytest.fixture
+ def storage(self) -> Iterator[CacheStorage[int]]:
+ """
+ Return an empty in-memory cache that returns integer values.
+ """
+ yield SQLiteCacheStorage[int](":memory:", data_type=int)
+
+ def test_operations_persist(self, tmp_path: Path) -> None:
+ """
+ Cache operations persist over opening/closing the cache.
+ """
+ key = {"n": "one"}
+ db_path = tmp_path / "cache.db"
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ cache.set(key, value=1)
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ assert cache.get(key) == 1
+
+ def test_unexpected_error_during_set(self) -> None:
+ """
+ If we get an unexpected IntegrityError during set() that isn't about
+ overwriting an existing key, it's re-raised.
+ """
+
+ class OtherIntegrityError(sqlite3.IntegrityError):
+ sqlite_errorname = "SQLITE_CONSTRAINT_UNIQUE"
+
+ with SQLiteCacheStorage[int](":memory:", data_type=int) as cache:
+ # Close the original connection, then replace it with a mock
+ # that simulates a different IntegrityError.
+ cache.con.close()
+
+ mock_con = MagicMock()
+ mock_con.execute.side_effect = MagicMock(
+ side_effect=OtherIntegrityError("UNIQUE constraint failed")
+ )
+ cache.con = mock_con
+
+ with pytest.raises(sqlite3.IntegrityError) as exc_info:
+ cache.set(key={"n": "one"}, value=1)
+
+ assert exc_info.value.sqlite_errorname == "SQLITE_CONSTRAINT_UNIQUE"