storage/sqlite: add a fingerprint to the database file
- ID
7583cf2- date
2026-11-07 09:42:25+00:00- author
Alex Chan <alex@alexwlchan.net>- parent
d3fa9fa- message
storage/sqlite: add a fingerprint to the database file If the schema of the database changes, we can't use old databases. Since this is only a cache and all the data can be recreated, drop the old tables anytime the schema changes.- changed files
5 files, 141 additions, 14 deletions
Changed files
pyproject.toml (1471 → 1479)
diff --git a/pyproject.toml b/pyproject.toml
index 906fc5a..0a7e2d6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -28,7 +28,7 @@ license = "MIT"
"Changelog" = "https://alexwlchan.net/projects/funkycache/releases/"
[tool.setuptools.dynamic]
-version = {attr = "funkycache.__version__"}
+version = {attr = "funkycache.version.__version__"}
[tool.setuptools.packages.find]
where = ["src"]
src/funkycache/__init__.py (235 → 261)
diff --git a/src/funkycache/__init__.py b/src/funkycache/__init__.py
index 98069a0..b81e94a 100644
--- a/src/funkycache/__init__.py
+++ b/src/funkycache/__init__.py
@@ -4,6 +4,6 @@ funkycache is a function cache that tracks file dependencies.
from .manager import FunkyCache
from .storage import CannotOverwriteValueException
+from .version import __version__
-__version__ = "0.0.1"
-__all__ = ["CannotOverwriteValueException", "FunkyCache"]
+__all__ = ["CannotOverwriteValueException", "FunkyCache", "__version__"]
src/funkycache/storage.py (7577 → 9728)
diff --git a/src/funkycache/storage.py b/src/funkycache/storage.py
index 0090a28..d8b5aab 100644
--- a/src/funkycache/storage.py
+++ b/src/funkycache/storage.py
@@ -7,6 +7,7 @@ a collection of arguments to the return value.
from abc import ABC, abstractmethod
from collections.abc import Mapping
+from datetime import datetime, timezone
from functools import cache
import json
from pathlib import Path
@@ -17,6 +18,8 @@ from typing import Any, cast, Self
from pydantic import TypeAdapter
import xxhash
+from .version import __version__
+
class CannotOverwriteValueException(ValueError):
"""
@@ -160,18 +163,71 @@ class SQLiteCacheStorage[V](TypedCacheStorage[V]):
self.con.execute("PRAGMA journal_mode = WAL;")
self.con.execute("PRAGMA synchronous = NORMAL;")
+ try:
+ self._create_tables()
+ self.con.commit()
+ except Exception:
+ self.close()
+ raise
+
+ super().__init__(data_type=data_type)
+
+ def _create_tables(self) -> None:
+ """
+ Create or update the database tables.
+ """
cursor = self.con.cursor()
+ table_definitions = {
+ #
+ # The funkycache_meta table contains the fingerprint and
+ # some other metadata that might be useful for debugging.
+ "funkycache_meta": "CREATE TABLE funkycache_meta(key PRIMARY KEY, value)",
+ #
+ # Each row in funkycache contains three columns:
+ # - enc_key is the JSON-serialised, hashed key
+ # - raw_key is the JSON-serialised key (for debugging)
+ # - value is the JSON-serialised value
+ "funkycache_entries": "CREATE TABLE funkycache_entries(enc_key PRIMARY KEY, raw_key, value)", # noqa: E501
+ }
+
+ # Get a fingerprint for the current table definitions
+ cur_fingerprint = _hash(json.dumps(table_definitions, sort_keys=True))
+
+ # Check if the fingerprint in the database exists and matches
+ # the current file.
+ try:
+ cursor.execute(
+ "SELECT value FROM funkycache_meta WHERE key = 'fingerprint'"
+ )
+ except sqlite3.OperationalError as err:
+ if err.args == ("no such table: funkycache_meta",):
+ pass
+ else:
+ raise
+ else:
+ # If the stored fingerprint matches, the database is already
+ # using the correct data.
+ if result := cursor.fetchone():
+ if result[0] == cur_fingerprint:
+ return
- # Each row in the table contains three columns:
- # - enc_key is the JSON-serialised, hashed key
- # - raw_key is the JSON-serialised key (for debugging)
- # - value is the JSON-serialised value
+ # If the fingerprint is missing/wrong, drop and recreate the tables.
+ for tbl, definition in table_definitions.items():
+ cursor.execute(f"DROP TABLE IF EXISTS {tbl}")
+ cursor.execute(definition)
+
+ # Add some metadata to the funkycache_meta table
cursor.execute(
- "CREATE TABLE IF NOT EXISTS funkycache(enc_key PRIMARY KEY, raw_key, value)"
+ "INSERT INTO funkycache_meta VALUES (?,?)", ("fingerprint", cur_fingerprint)
+ )
+ cursor.execute(
+ "INSERT INTO funkycache_meta VALUES (?,?)",
+ ("funkycache_version", __version__),
+ )
+ cursor.execute(
+ "INSERT INTO funkycache_meta VALUES (?,?)",
+ ("date_created", datetime.now(tz=timezone.utc).isoformat()),
)
-
- self.con.commit()
- super().__init__(data_type=data_type)
def close(self) -> None:
"""
@@ -218,7 +274,8 @@ class SQLiteCacheStorage[V](TypedCacheStorage[V]):
try:
self.con.execute(
- "INSERT INTO funkycache VALUES (?,?,?)", (enc_key, json_key, value)
+ "INSERT INTO funkycache_entries VALUES (?,?,?)",
+ (enc_key, json_key, value),
)
self.con.commit()
except sqlite3.IntegrityError as err:
@@ -239,7 +296,7 @@ class SQLiteCacheStorage[V](TypedCacheStorage[V]):
_, enc_key = self._encode_key(key)
res = self.con.execute(
- "SELECT value FROM funkycache WHERE enc_key=?", (enc_key,)
+ "SELECT value FROM funkycache_entries WHERE enc_key=?", (enc_key,)
)
try:
src/funkycache/version.py (0 → 55)
diff --git a/src/funkycache/version.py b/src/funkycache/version.py
new file mode 100644
index 0000000..ad32e07
--- /dev/null
+++ b/src/funkycache/version.py
@@ -0,0 +1,5 @@
+"""
+The FunkyCache version.
+"""
+
+__version__ = "0.0.1"
tests/test_storage.py (4736 → 7192)
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 0789c9e..431ad02 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -4,7 +4,7 @@ Tests for backend storage implementations.
from abc import ABC, abstractmethod
from collections.abc import Generator, Mapping
-from contextlib import contextmanager
+from contextlib import closing, contextmanager
from pathlib import Path
import sqlite3
from typing import Any
@@ -153,3 +153,68 @@ class TestSQLiteCacheStorage(CacheStorageTestSuite):
cache.set(key={"n": "one"}, value=1)
assert exc_info.value.sqlite_errorname == "SQLITE_CONSTRAINT_UNIQUE"
+
+ def test_reset_if_fingerprint_is_wrong(self, tmp_path: Path) -> None:
+ """
+ If the database contains a fingerprint from an older version of
+ FunkyCache, old cache entries are dropped.
+ """
+ key = {"n": "one"}
+ db_path = tmp_path / "cache.db"
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ cache.set(key, value=1)
+
+ with closing(sqlite3.connect(db_path)) as con:
+ cur = con.cursor()
+ cur.execute(
+ "UPDATE funkycache_meta SET value=? WHERE key=?",
+ ("old-fingerprint", "fingerprint"),
+ )
+ con.commit()
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ with pytest.raises(KeyError):
+ cache.get(key)
+
+ def test_reset_if_fingerprint_is_missing(self, tmp_path: Path) -> None:
+ """
+ If the database doesn't have a fingerprint from an older version of
+ FunkyCache, old cache entries are dropped.
+ """
+ key = {"n": "one"}
+ db_path = tmp_path / "cache.db"
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ cache.set(key, value=1)
+
+ with closing(sqlite3.connect(db_path)) as con:
+ cur = con.cursor()
+ cur.execute("DELETE FROM funkycache_meta")
+ con.commit()
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ with pytest.raises(KeyError):
+ cache.get(key)
+
+ def test_error_if_meta_is_wrong_format(self, tmp_path: Path) -> None:
+ """
+ If looking up the fingerprint funkycache_meta table fails for
+ a reason other than the table missing, the error is re-raised.
+ """
+ key = {"n": "one"}
+ db_path = tmp_path / "cache.db"
+
+ with SQLiteCacheStorage[int](db_path, data_type="int") as cache:
+ cache.set(key, value=1)
+
+ # Delete the original table, then recreate it without a "key" column.
+ # Looking up by key will now fail.
+ with closing(sqlite3.connect(db_path)) as con:
+ cur = con.cursor()
+ cur.execute("DROP TABLE funkycache_meta")
+ cur.execute("CREATE TABLE funkycache_meta(value)")
+ con.commit()
+
+ with pytest.raises(sqlite3.OperationalError, match="no such column: key"):
+ SQLiteCacheStorage[int](db_path, data_type="int")