magnus/btcdata-archive
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement verified weekly archives and SQLite inventory reporting

Browse Source
This commit is contained in:
magnus
2026-10-03 19:42:42 +02:00
commit 885f618e66
36 changed files with 3077 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/compose.yaml
+25
View File
@@ -0,0 +1,25 @@
# Disposable tests ONLY. Known test credentials are intentionally not production secrets.
services:
postgres:
image: postgres:18.0-bookworm
environment:
POSTGRES_PASSWORD: disposable-test
ports: ["127.0.0.1:15432:5432"]
tmpfs: ["/var/lib/postgresql"]
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 2s
timeout: 3s
retries: 30
rustfs:
image: rustfs/rustfs:1.0.0
environment:
RUSTFS_ACCESS_KEY: disposable-test
RUSTFS_SECRET_KEY: disposable-test-secret
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_OBS_LOG_DIRECTORY: /logs
command: ["/data"]
ports: ["127.0.0.1:19000:9000"]
tmpfs:
- /data:uid=10001,gid=10001
- /logs:uid=10001,gid=10001
tests/integration/conftest.py
+76
View File
@@ -0,0 +1,76 @@
"""Opt-in integration infrastructure. Always creates NEW disposable resources."""
import os
from pathlib import Path
from types import SimpleNamespace
import uuid
import psycopg
from psycopg import sql
import pytest
from btcdata_archive.postgres import Postgres
from btcdata_archive.object_store import ObjectStore
from btcdata_archive.schema import TABLES
@pytest.fixture
def postgres_infra(tmp_path):
if not os.environ.get('TEST_PGHOST'):
pytest.skip('set TEST_PGHOST for disposable PostgreSQL infrastructure')
suffix = uuid.uuid4().hex
database, role, password = 'archive_test_' + suffix, 'archive_test_' + suffix, uuid.uuid4().hex
admin_args = {'host': os.environ['TEST_PGHOST'], 'port': int(os.environ.get('TEST_PGPORT', '5432')),
'user': os.environ.get('TEST_PGUSER', 'postgres'), 'password': os.environ.get('TEST_PGPASSWORD', 'disposable-test'),
'dbname': 'postgres'}
config = SimpleNamespace(pg={**admin_args, 'dbname': database, 'user': role, 'password': password},
endpoint=os.environ.get('TEST_S3_ENDPOINT_URL'), access_key=os.environ.get('TEST_S3_ACCESS_KEY', 'disposable-test'),
secret_key=os.environ.get('TEST_S3_SECRET_KEY', 'disposable-test-secret'),
region='us-east-1', bucket='archive-test-' + suffix, prefix='btcprod/weeks/', work_dir=tmp_path, batch_rows=2)
root = psycopg.connect(**admin_args, autocommit=True)
admin = pg = None
try:
root.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(database)))
root.execute(sql.SQL('CREATE ROLE {} LOGIN PASSWORD {}').format(sql.Identifier(role), sql.Literal(password)))
admin = psycopg.connect(**{**admin_args, 'dbname': database}, autocommit=True)
for table in TABLES:
cols = sql.SQL(',').join(sql.SQL('{} {} {}').format(sql.Identifier(name), sql.SQL(typ),
sql.SQL('PRIMARY KEY' if name == table.pk else '' if nullable else 'NOT NULL'))
for name, typ, nullable in table.columns)
admin.execute(sql.SQL('CREATE TABLE public.{} ({})').format(sql.Identifier(table.name), cols))
setup = (Path(__file__).resolve().parents[2] / 'sql/setup.sql').read_text()
setup = setup.replace(':"DBNAME"', sql.Identifier(database).as_string()).replace(':"runtime_role"', sql.Identifier(role).as_string())
admin.execute(setup)
pg = Postgres(config)
yield SimpleNamespace(config=config, pg=pg, admin=admin)
finally:
if pg:
pg.close()
if admin:
admin.close()
root.execute(sql.SQL('DROP DATABASE IF EXISTS {} WITH (FORCE)').format(sql.Identifier(database)))
root.execute(sql.SQL('DROP ROLE IF EXISTS {}').format(sql.Identifier(role)))
root.close()
@pytest.fixture
def infra(postgres_infra):
if not os.environ.get('TEST_S3_ENDPOINT_URL'):
pytest.skip('set TEST_S3_ENDPOINT_URL for disposable RustFS infrastructure')
config = postgres_infra.config
store = ObjectStore(config)
bucket_created = False
instance = SimpleNamespace(config=config, pg=postgres_infra.pg, store=store, admin=postgres_infra.admin)
try:
store.client.create_bucket(Bucket=config.bucket)
bucket_created = True
store.client.put_bucket_versioning(Bucket=config.bucket, VersioningConfiguration={'Status': 'Enabled'})
yield instance
finally:
instance.pg.close()
if bucket_created:
# Test-only bucket with random unique name. Never use runtime cleanup in production.
for page in store.client.get_paginator('list_object_versions').paginate(Bucket=config.bucket):
items = [{'Key': v['Key'], 'VersionId': v['VersionId']} for v in page.get('Versions', []) + page.get('DeleteMarkers', [])]
if items:
store.client.delete_objects(Bucket=config.bucket, Delete={'Objects': items})
store.client.delete_bucket(Bucket=config.bucket)
tests/integration/test_archive.py
+194
View File
@@ -0,0 +1,194 @@
from datetime import datetime, timezone, timedelta
from decimal import Decimal
from pathlib import Path
import os
import subprocess
import sys
from psycopg import sql
import pytest
from btcdata_archive.archive import run
from btcdata_archive.postgres import Postgres
from btcdata_archive.schema import TABLES, encode_row, decode_row
from btcdata_archive.sqlite_archive import connect, validate
from btcdata_archive.weeks import boundaries, WEEK
pytestmark = pytest.mark.integration
CUTOFF = boundaries(datetime.now(timezone.utc))[1]
START = CUTOFF - 3 * WEEK
def seed(infra):
for offset, pk in [(0, 1), (14, 3), (21, 4)]:
instant = START + timedelta(days=offset, microseconds=123456)
infra.admin.execute('INSERT INTO binance_price VALUES (%s,%s,%s)', (pk, Decimal('1234567890.12345678'), instant))
infra.admin.execute("INSERT INTO strike_price VALUES (%s,'{\"a\":12345678901234567890.12345678901234567890,\"null\":null}'::jsonb,%s,NULL)", (pk, instant))
infra.admin.execute('INSERT INTO kraken_minute VALUES (%s,1.00000001,2,3,4,5,1234567890123456.12345678,7,%s)', (int(instant.timestamp()), instant))
def invoke(infra, mode='archive-and-prune', max_weeks=None):
return run(infra.pg, infra.store, infra.config, CUTOFF, mode, max_weeks)
def test_export_gap_precision_cutoff_restore(infra, tmp_path):
seed(infra)
original = {}
for table in TABLES:
projection = ','.join(n + '::text' if typ == 'jsonb' else n for n, typ, _ in table.columns)
original[table.name] = [encode_row(table, row) for row in infra.admin.execute(f'SELECT {projection} FROM public.{table.name} ORDER BY {table.pk}')]
# Exactly at cutoff must stay, regardless of current week.
infra.admin.execute('INSERT INTO binance_price VALUES (5,5,%s)', (CUTOFF,))
invoke(infra, 'verify-only')
assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 4
assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='verified'").fetchone()[0] == 3
invoke(infra)
assert infra.admin.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(4,), (5,)]
assert infra.admin.execute('SELECT count(*) FROM kraken_minute').fetchone()[0] == 1
for offset in (0, 7, 14):
start = START + timedelta(days=offset)
key = infra.config.prefix + start.date().isoformat() + '.sqlite'
head = infra.store.head(key)
path = tmp_path / ('restored-' + str(offset) + '.sqlite')
infra.store.download(key, head['VersionId'], path)
meta = validate(path, start, infra.config.pg['dbname'], 2)
assert all(v['count'] == (0 if offset == 7 else 1) for v in meta['tables'].values())
# Restore every column through the documented type mapping into isolated PostgreSQL tables.
db = connect(path, readonly=True)
for table in TABLES:
restored = 'restored_' + str(offset) + '_' + table.name
infra.admin.execute(sql.SQL('CREATE TABLE {} (LIKE public.{})').format(sql.Identifier(restored), sql.Identifier(table.name)))
rows = db.execute(f'SELECT * FROM {table.name} ORDER BY {table.pk}').fetchall()
for row in rows:
placeholders = ','.join('%s::jsonb' if typ == 'jsonb' else '%s' for _, typ, _ in table.columns)
infra.admin.execute(sql.SQL(f'INSERT INTO {{}} VALUES ({placeholders})').format(sql.Identifier(restored)), decode_row(table, row))
projection = ','.join(n + '::text' if typ == 'jsonb' else n for n, typ, _ in table.columns)
roundtrip = [encode_row(table, r) for r in infra.admin.execute(sql.SQL(f'SELECT {projection} FROM {{}} ORDER BY {table.pk}').format(sql.Identifier(restored)))]
assert roundtrip == rows
assert all(r in original[table.name] for r in rows)
db.close()
assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='complete'").fetchone()[0] == 3
# Application latest-value reads still work with the retained rows.
assert infra.admin.execute('SELECT id FROM binance_price ORDER BY recorded_at DESC LIMIT 1').fetchone()[0] == 4
def test_schema_mismatch(infra):
seed(infra)
infra.admin.execute('ALTER TABLE strike_price ADD COLUMN unexpected text')
with pytest.raises(RuntimeError, match='migration required'):
invoke(infra)
assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3
def test_concurrent_lock_and_loss(infra, monkeypatch):
seed(infra)
second = Postgres(infra.config)
infra.pg.lock()
try:
with pytest.raises(RuntimeError, match='advisory lock'):
second.lock()
finally:
second.close()
# Closing the coordinating session after upload must prevent ANY deletion.
upload = infra.store.upload
def lose_lock(*args):
result = upload(*args)
infra.pg.close()
return result
monkeypatch.setattr(infra.store, 'upload', lose_lock)
# Don't acquire a second lock on the same session in run.
monkeypatch.setattr(infra.pg, 'lock', lambda: None)
with pytest.raises(Exception):
invoke(infra)
assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3
def test_changed_and_late_rows(infra, monkeypatch):
seed(infra)
upload = infra.store.upload
def change(*args):
version = upload(*args)
infra.admin.execute('UPDATE binance_price SET price=9 WHERE id=1')
infra.admin.execute('INSERT INTO binance_price VALUES (2,2,%s)', (START,))
return version
monkeypatch.setattr(infra.store, 'upload', change)
with pytest.raises(RuntimeError, match='reconciliation required'):
invoke(infra)
assert infra.admin.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(1,), (2,), (3,), (4,)]
# Other unchanged archived rows were safely pruned; their data must survive the merge.
assert infra.admin.execute('SELECT count(*) FROM strike_price').fetchone()[0] == 2
monkeypatch.setattr(infra.store, 'upload', upload)
invoke(infra)
key = infra.config.prefix + START.date().isoformat() + '.sqlite'
path = infra.config.work_dir / 'late.sqlite'
infra.store.download(key, infra.store.head(key)['VersionId'], path)
db = connect(path, readonly=True)
assert db.execute('SELECT id,price FROM binance_price ORDER BY id').fetchall() == [(1,'9.00000000'), (2,'2.00000000')]
assert db.execute('SELECT count(*) FROM strike_price').fetchone()[0] == 1
db.close()
@pytest.mark.parametrize('boundary', ['uploaded', 'verified', 'pruning', 'deletion_batch', 'complete'])
def test_crash_restart_boundaries(infra, monkeypatch, boundary):
seed(infra)
ledger, upload = infra.pg.ledger, infra.store.upload
fired = False
def crash_ledger(start, status, *args, **kwargs):
nonlocal fired
result = ledger(start, status, *args, **kwargs)
target = 'pruning' if boundary == 'deletion_batch' else boundary
if not fired and status == target and (boundary != 'deletion_batch' or kwargs.get('deletions')):
fired = True
raise KeyboardInterrupt()
return result
def crash_upload(*args):
result = upload(*args)
raise KeyboardInterrupt()
monkeypatch.setattr(infra.pg, 'ledger', crash_ledger)
if boundary == 'uploaded':
monkeypatch.setattr(infra.store, 'upload', crash_upload)
with pytest.raises(KeyboardInterrupt):
invoke(infra)
infra.pg.close()
infra.pg = Postgres(infra.config)
monkeypatch.setattr(infra.store, 'upload', upload)
invoke(infra)
key = infra.config.prefix + START.date().isoformat() + '.sqlite'
path = infra.config.work_dir / 'resumed.sqlite'
infra.store.download(key, infra.store.head(key)['VersionId'], path)
assert all(s['count'] == 1 for s in validate(path, START, infra.config.pg['dbname'], 2)['tables'].values())
assert infra.admin.execute('SELECT id FROM binance_price').fetchall() == [(4,)]
@pytest.mark.parametrize('operation', ['upload', 'download'])
def test_network_failure_preserves_data(infra, monkeypatch, operation):
seed(infra)
def fail(*args):
raise OSError('network unavailable')
monkeypatch.setattr(infra.store, operation, fail)
with pytest.raises(OSError):
invoke(infra)
assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3
def test_missing_delete_privilege(infra):
seed(infra)
infra.admin.execute(sql.SQL('REVOKE DELETE ON binance_price FROM {}').format(sql.Identifier(infra.config.pg['user'])))
with pytest.raises(Exception):
invoke(infra)
assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3
def test_wrapper_all_weeks_and_bound(infra):
seed(infra)
c = infra.config
env = {**os.environ, 'PGHOST': c.pg['host'], 'PGPORT': str(c.pg['port']), 'PGDATABASE': c.pg['dbname'],
'PGUSER': c.pg['user'], 'PGPASSWORD': c.pg['password'], 'S3_ENDPOINT_URL': c.endpoint,
'AWS_ACCESS_KEY_ID': c.access_key, 'AWS_SECRET_ACCESS_KEY': c.secret_key, 'ARCHIVE_BUCKET': c.bucket,
'ARCHIVE_WORK_DIR': str(c.work_dir), 'PATH': str(Path(sys.executable).parent) + ':' + os.environ['PATH']}
script = str(Path(__file__).resolve().parents[2] / 'scripts/archive-old-weeks.sh')
first = subprocess.run([script, '--mode', 'verify-only', '--max-weeks', '1'], env=env, capture_output=True, text=True)
assert first.returncode == 0, first.stdout
second = subprocess.run([script, '--mode', 'archive-and-prune'], env=env, capture_output=True, text=True)
assert second.returncode == 0, second.stdout
assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='complete'").fetchone()[0] == 3
tests/integration/test_postgres.py
+88
View File
@@ -0,0 +1,88 @@
"""PostgreSQL-only regressions; never connects using the production environment."""
from datetime import datetime, timezone
import psycopg
import pytest
from btcdata_archive import sqlite_archive
from btcdata_archive.schema import TABLES
from btcdata_archive.weeks import WEEK
pytestmark = pytest.mark.integration
START = datetime(2026, 2, 16, tzinfo=timezone.utc)
def test_ledger_progress_preserves_verification_before_insert_check(postgres_infra):
pg, admin = postgres_infra.pg, postgres_infra.admin
pg.lock()
key, version, checksum = 'test/week.sqlite', 'verified-version', 'a' * 64
counts = {'binance_price': 3, 'strike_price': 0, 'kraken_minute': 0}
pg.ledger(START, 'exporting', key)
pg.ledger(START, 'verified', key, version, checksum, counts)
# These progress-only calls used to fail INSERT's CHECK before ON CONFLICT.
pg.ledger(START, 'pruning', key)
pg.ledger(START, 'pruning', key, deletions={'binance_price': 2})
pg.ledger(START, 'complete', key, deletions={'binance_price': 3})
row = admin.execute('SELECT status,object_version_id,file_sha256,row_counts,deletion_counts,completion_time IS NOT NULL FROM btcdata_archive.weeks WHERE week_start=%s', (START,)).fetchone()
assert row == ('complete', version, checksum, counts, {'binance_price': 3}, True)
pg.ledger(START, 'failed', key, error='test interruption')
pg.ledger(START, 'exporting', key)
pg.ledger(START, 'verified', key, 'new-version', 'b' * 64, counts)
pg.ledger(START, 'pruning', key)
assert admin.execute('SELECT object_version_id,file_sha256,completion_time FROM btcdata_archive.weeks').fetchone() == ('new-version', 'b' * 64, None)
def test_ledger_still_rejects_unverified_pruning(postgres_infra):
pg = postgres_infra.pg
pg.lock()
with pytest.raises(psycopg.errors.CheckViolation):
pg.ledger(START, 'pruning', 'test/week.sqlite')
pg.ledger(START, 'exporting', 'test/week.sqlite')
with pytest.raises(psycopg.errors.CheckViolation):
pg.ledger(START, 'complete', 'test/week.sqlite')
assert postgres_infra.admin.execute('SELECT status FROM btcdata_archive.weeks').fetchone() == ('exporting',)
def test_typed_pruning_changed_rows_and_recovery(postgres_infra, tmp_path):
pg, admin, config = postgres_infra.pg, postgres_infra.admin, postgres_infra.config
pg.lock()
pg.check_schema()
admin.execute('INSERT INTO binance_price VALUES (1,1.00000001,%s)', (START,))
admin.execute('INSERT INTO strike_price VALUES (1,%s::jsonb,%s,NULL)', ('{"a":12345678901234567890.12345678901234567890}', START))
admin.execute('INSERT INTO kraken_minute VALUES (%s,1.00000001,2,3,4,5,1234567890123456.12345678,7,%s)', (int(START.timestamp()), START.replace(microsecond=123456)))
path = tmp_path / 'verified.sqlite'
sqlite_archive.create(path)
key = 'test/week.sqlite'
def export_and_record(version):
pg.ledger(START, 'exporting', key)
pg.export(path, START)
meta = sqlite_archive.finalize(path, START, config.pg['dbname'], START, 2)
pg.ledger(START, 'verified', key, version, sqlite_archive.file_hash(path),
{name: stats['count'] for name, stats in meta['tables'].items()})
pg.ledger(START, 'pruning', key)
def progress(deleted):
pg.ledger(START, 'pruning', key, deletions=deleted)
export_and_record('test-v1')
# Source mutation and late insertion must remain; unchanged rows can be deleted.
admin.execute('UPDATE binance_price SET price=9 WHERE id=1')
admin.execute("INSERT INTO strike_price VALUES (2,'null'::jsonb,%s,NULL)", (START,))
with pytest.raises(RuntimeError, match='reconciliation required'):
pg.prune(path, START, START + WEEK, progress)
assert admin.execute('SELECT id FROM binance_price').fetchall() == [(1,)]
assert admin.execute('SELECT id FROM strike_price').fetchall() == [(2,)]
assert admin.execute('SELECT count(*) FROM kraken_minute').fetchone() == (0,)
pg.ledger(START, 'failed', key, error='test reconciliation')
export_and_record('test-v2')
deleted = pg.prune(path, START, START + WEEK, progress)
pg.ledger(START, 'complete', key, deletions=deleted)
assert all(pg.counts(START, START + WEEK)[t.name] == 0 for t in TABLES)
db = sqlite_archive.connect(path, readonly=True)
try:
assert db.execute('SELECT id,price FROM binance_price').fetchall() == [(1, '9.00000000')]
assert db.execute('SELECT id,rates,quote_sats FROM strike_price ORDER BY id').fetchall() == [(1, '{"a":12345678901234567890.12345678901234567890}', None), (2, 'null', None)]
assert db.execute('SELECT count(*) FROM kraken_minute').fetchone() == (1,)
finally:
db.close()