Implement verified weekly archives and SQLite inventory reporting
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
import io
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from botocore.response import StreamingBody
|
||||
from botocore.stub import Stubber
|
||||
|
||||
from btcdata_archive.errors import SafetyError
|
||||
from btcdata_archive.object_store import ObjectStore
|
||||
from btcdata_archive.postgres import Postgres
|
||||
from btcdata_archive.schema import TABLES
|
||||
|
||||
|
||||
def store():
|
||||
return ObjectStore(SimpleNamespace(endpoint='http://127.0.0.1:19000', access_key='test', secret_key='test', region='us-east-1', bucket='test'))
|
||||
|
||||
|
||||
def test_versioning_enforced():
|
||||
backend = store()
|
||||
with Stubber(backend.client) as stub:
|
||||
stub.add_response('get_bucket_versioning', {'Status':'Suspended'}, {'Bucket':'test'})
|
||||
with pytest.raises(SafetyError, match='versioning'):
|
||||
backend.require_versioning()
|
||||
|
||||
|
||||
def test_head_missing_not_forbidden():
|
||||
backend = store()
|
||||
with Stubber(backend.client) as stub:
|
||||
stub.add_client_error('head_object', 'NoSuchKey', http_status_code=404, expected_params={'Bucket':'test','Key':'week.sqlite'})
|
||||
assert backend.head('week.sqlite') is None
|
||||
stub.add_client_error('head_object', 'AccessDenied', http_status_code=403, expected_params={'Bucket':'test','Key':'week.sqlite'})
|
||||
with pytest.raises(Exception):
|
||||
backend.head('week.sqlite')
|
||||
|
||||
|
||||
def test_download_exact_version(tmp_path):
|
||||
backend = store()
|
||||
body = StreamingBody(io.BytesIO(b'archive'), 7)
|
||||
with Stubber(backend.client) as stub:
|
||||
stub.add_response('get_object', {'VersionId':'v1','Body':body}, {'Bucket':'test','Key':'week.sqlite','VersionId':'v1'})
|
||||
path = tmp_path / 'archive'
|
||||
backend.download('week.sqlite','v1',path)
|
||||
assert path.read_bytes() == b'archive'
|
||||
stub.add_response('get_object', {'VersionId':'v2','Body':StreamingBody(io.BytesIO(b'bad'), 3)}, {'Bucket':'test','Key':'week.sqlite','VersionId':'v1'})
|
||||
with pytest.raises(SafetyError, match='wrong version'):
|
||||
backend.download('week.sqlite','v1',path)
|
||||
|
||||
|
||||
def test_upload_captures_completion_version(tmp_path):
|
||||
backend = store()
|
||||
path = tmp_path / 'archive'
|
||||
path.write_bytes(b'archive')
|
||||
with Stubber(backend.client) as stub:
|
||||
stub.add_response('get_bucket_versioning', {'Status':'Enabled'}, {'Bucket':'test'})
|
||||
stub.add_response('create_multipart_upload', {'UploadId':'upload'}, {'Bucket':'test','Key':'week.sqlite','ContentType':'application/vnd.sqlite3','Metadata':{'sha256':'digest'}})
|
||||
stub.add_response('upload_part', {'ETag':'not-a-checksum'}, {'Bucket':'test','Key':'week.sqlite','UploadId':'upload','PartNumber':1,'Body':b'archive'})
|
||||
stub.add_response('complete_multipart_upload', {'VersionId':'verified-version'}, {'Bucket':'test','Key':'week.sqlite','UploadId':'upload','MultipartUpload':{'Parts':[{'PartNumber':1,'ETag':'not-a-checksum'}]}})
|
||||
assert backend.upload('week.sqlite',path,'digest') == 'verified-version'
|
||||
|
||||
|
||||
class Result:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
def fetchone(self):
|
||||
return self.rows[0]
|
||||
def fetchall(self):
|
||||
return self.rows
|
||||
|
||||
|
||||
class SchemaConnection:
|
||||
def __init__(self, change=None):
|
||||
self.change = change
|
||||
def execute(self, query, params):
|
||||
table = next(t for t in TABLES if params[0] == 'public.' + t.name)
|
||||
if 'relkind' in query:
|
||||
return Result([('r',False)])
|
||||
if 'pg_trigger' in query or 'confdeltype' in query:
|
||||
return Result([(0,)])
|
||||
if 'format_type' in query:
|
||||
columns = list(table.columns)
|
||||
if self.change == 'added':
|
||||
columns.append(('extra','text',True))
|
||||
elif self.change == 'removed':
|
||||
columns.pop()
|
||||
elif self.change == 'type':
|
||||
columns[0] = ('id','integer',False)
|
||||
return Result(columns)
|
||||
return Result([([table.pk if self.change != 'pk' else 'wrong'],)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize('change', [None,'added','removed','type','pk'])
|
||||
def test_source_schema_fail_closed(change):
|
||||
pg = Postgres.__new__(Postgres)
|
||||
pg.conn = SchemaConnection(change)
|
||||
if change is None:
|
||||
pg.check_schema()
|
||||
else:
|
||||
with pytest.raises(SafetyError, match='migration required'):
|
||||
pg.check_schema()
|
||||
@@ -0,0 +1,169 @@
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from decimal import Decimal
|
||||
import json
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from btcdata_archive import sqlite_archive as archive
|
||||
from btcdata_archive.cli import parser, main
|
||||
from btcdata_archive.config import Config, load_env_file
|
||||
from btcdata_archive.errors import SafetyError
|
||||
from btcdata_archive.schema import TABLES, Digest, encode_row, decode_row, canonical_json
|
||||
from btcdata_archive.weeks import boundaries, week_start, weeks, timestamp
|
||||
|
||||
UTC = timezone.utc
|
||||
START = datetime(2026, 8, 31, tzinfo=UTC)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('instant,anchor,cutoff', [
|
||||
('2026-10-05T00:00:00+00:00', '2026-10-05', '2026-09-07'),
|
||||
('2026-10-04T23:59:59+00:00', '2026-09-28', '2026-08-31'),
|
||||
('2027-01-01T12:00:00+00:00', '2026-12-28', '2026-11-30'),
|
||||
('2024-02-29T12:00:00+00:00', '2024-02-26', '2024-01-29'),
|
||||
('2026-03-29T03:30:00+02:00', '2026-03-23', '2026-02-23'),
|
||||
])
|
||||
def test_boundaries(instant, anchor, cutoff):
|
||||
result = boundaries(datetime.fromisoformat(instant))
|
||||
assert [d.date().isoformat() for d in result] == [anchor, cutoff]
|
||||
|
||||
|
||||
def test_week_intervals():
|
||||
assert list(weeks(START + timedelta(days=2), START + timedelta(days=21))) == [START, START + timedelta(days=7), START + timedelta(days=14)]
|
||||
assert list(weeks(None, START)) == []
|
||||
assert list(weeks(START, START)) == []
|
||||
with pytest.raises(ValueError):
|
||||
week_start(datetime(2026, 1, 1))
|
||||
assert TABLES[0].retention == TABLES[1].retention == 'recorded_at'
|
||||
assert TABLES[2].retention == 'ts'
|
||||
|
||||
|
||||
def test_exact_roundtrips(tmp_path):
|
||||
path = tmp_path / 'week.sqlite'
|
||||
archive.create(path)
|
||||
values = [
|
||||
(2**63 - 1, Decimal('1234567890.12345678'), START.replace(microsecond=123456)),
|
||||
(2**63 - 2, '{"z":null,"b":12345678901234567890.12345678901234567890,"a":[1,true]}', START, None),
|
||||
(int(START.timestamp()), *[Decimal('0.00000001')] * 6, 0, START.replace(microsecond=999999)),
|
||||
]
|
||||
db = archive.connect(path)
|
||||
for table, row in zip(TABLES, values):
|
||||
encoded = encode_row(table, row)
|
||||
archive.upsert(db, table, [encoded])
|
||||
assert encode_row(table, decode_row(table, encoded)) == encoded
|
||||
db.commit()
|
||||
db.close()
|
||||
meta = archive.finalize(path, START, 'btcprod', START, 1)
|
||||
assert all(s['count'] == 1 for s in meta['tables'].values())
|
||||
db = archive.connect(path, readonly=True)
|
||||
rates = db.execute('SELECT rates, quote_sats FROM strike_price').fetchone()
|
||||
assert rates[1] is None
|
||||
assert '12345678901234567890.12345678901234567890' in rates[0]
|
||||
assert json.loads(rates[0])['z'] is None
|
||||
db.close()
|
||||
assert not list(tmp_path.glob('*-wal'))
|
||||
assert timestamp(START.replace(microsecond=1)).endswith('.000001Z')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('price', ['1.000000001', '10000000000.00000000'])
|
||||
def test_decimal_type_limits(tmp_path, price):
|
||||
path = tmp_path / 'week.sqlite'
|
||||
archive.create(path)
|
||||
db = archive.connect(path)
|
||||
archive.upsert(db, TABLES[0], [(1, price, timestamp(START))])
|
||||
db.commit()
|
||||
db.close()
|
||||
with pytest.raises(SafetyError, match='precision/scale'):
|
||||
archive.finalize(path, START, 'btcprod', START, 2)
|
||||
|
||||
|
||||
def test_json_and_digest():
|
||||
assert canonical_json('{"b": 1.234567890123456789, "a": null}') == '{"a":null,"b":1.234567890123456789}'
|
||||
a, b = Digest(), Digest()
|
||||
a.add((1, 'a,b', None))
|
||||
b.add((1, 'a', 'b'))
|
||||
assert a.result() != b.result()
|
||||
|
||||
|
||||
def test_empty_and_corruption(tmp_path):
|
||||
path = tmp_path / 'week.sqlite'
|
||||
archive.create(path)
|
||||
assert all(v['count'] == 0 for v in archive.finalize(path, START, 'btcprod', START, 2)['tables'].values())
|
||||
with pytest.raises(ValueError, match='metadata'):
|
||||
archive.validate(path, START, 'other', 2)
|
||||
db = archive.connect(path)
|
||||
db.execute('ALTER TABLE strike_price ADD COLUMN extra TEXT')
|
||||
db.commit()
|
||||
db.close()
|
||||
with pytest.raises(ValueError, match='schema'):
|
||||
archive.validate(path, START, 'btcprod', 2)
|
||||
path.write_bytes(b'corrupt')
|
||||
with pytest.raises(sqlite3.DatabaseError):
|
||||
archive.validate(path, START, 'btcprod', 2)
|
||||
|
||||
|
||||
def test_outside_week_and_checksum(tmp_path):
|
||||
path = tmp_path / 'week.sqlite'
|
||||
archive.create(path)
|
||||
db = archive.connect(path)
|
||||
archive.upsert(db, TABLES[0], [(1, '1.00000000', timestamp(START))])
|
||||
db.commit()
|
||||
db.close()
|
||||
archive.finalize(path, START, 'btcprod', START, 2)
|
||||
db = archive.connect(path)
|
||||
db.execute("UPDATE binance_price SET price='2.00000000'")
|
||||
db.commit()
|
||||
db.close()
|
||||
with pytest.raises(ValueError, match='checksum'):
|
||||
archive.validate(path, START, 'btcprod', 2)
|
||||
db = archive.connect(path)
|
||||
db.execute('UPDATE binance_price SET recorded_at=?', (timestamp(START + timedelta(days=7)),))
|
||||
db.commit()
|
||||
db.close()
|
||||
with pytest.raises(ValueError, match='outside week'):
|
||||
archive.finalize(path, START, 'btcprod', START, 2)
|
||||
|
||||
|
||||
def test_literal_env_file(tmp_path, monkeypatch):
|
||||
path = tmp_path / '.env'
|
||||
path.write_text("# comment\nTEST_ARCHIVE_VALUE='space # $value'\nTEST_ARCHIVE_KEEP=file\nTEST_ARCHIVE_EMPTY=\nTEST_ARCHIVE_LITERAL='$(touch should-not-exist)'\n")
|
||||
for key in ('TEST_ARCHIVE_VALUE', 'TEST_ARCHIVE_KEEP', 'TEST_ARCHIVE_EMPTY', 'TEST_ARCHIVE_LITERAL'):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
# Register environment restoration even for variables created by the loader.
|
||||
monkeypatch.setenv(key, '')
|
||||
monkeypatch.delenv(key)
|
||||
monkeypatch.setenv('TEST_ARCHIVE_KEEP', 'environment')
|
||||
load_env_file(path)
|
||||
import os
|
||||
assert os.environ['TEST_ARCHIVE_VALUE'] == 'space # $value'
|
||||
assert os.environ['TEST_ARCHIVE_KEEP'] == 'environment'
|
||||
assert os.environ['TEST_ARCHIVE_EMPTY'] == ''
|
||||
assert os.environ['TEST_ARCHIVE_LITERAL'] == '$(touch should-not-exist)'
|
||||
path.write_text("TEST_ARCHIVE_VALUE='private-unclosed-value\n")
|
||||
with pytest.raises(SafetyError) as error:
|
||||
load_env_file(path)
|
||||
assert 'private-unclosed-value' not in str(error.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('args', [['run'], ['run', '--mode', 'other'], ['run', '--mode', 'verify-only', '--max-weeks', '0'], ['run', '--mode', 'verify-only', '--max-weeks', '-1']])
|
||||
def test_cli_rejects(args):
|
||||
with pytest.raises(SystemExit):
|
||||
parser().parse_args(args)
|
||||
|
||||
|
||||
def test_config_and_safe_error(monkeypatch, capsys):
|
||||
monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False)
|
||||
for key in ('PGHOST','PGUSER','PGPASSWORD','S3_ENDPOINT_URL','AWS_ACCESS_KEY_ID','AWS_SECRET_ACCESS_KEY'):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
assert main(['run', '--mode', 'verify-only']) == 1
|
||||
assert 'missing configuration' in capsys.readouterr().out
|
||||
for key in ('PGHOST','PGUSER','PGPASSWORD'):
|
||||
monkeypatch.setenv(key, 'test')
|
||||
assert Config.from_env(need_s3=False).endpoint is None
|
||||
with pytest.raises(SafetyError):
|
||||
Config.from_env()
|
||||
monkeypatch.setenv('S3_ENDPOINT_URL', 'http://user:password@example.com')
|
||||
monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test')
|
||||
monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test')
|
||||
with pytest.raises(SafetyError):
|
||||
Config.from_env()
|
||||
@@ -0,0 +1,157 @@
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from btcdata_archive.config import Config
|
||||
from btcdata_archive.errors import SafetyError
|
||||
from btcdata_archive import report, sqlite_archive
|
||||
from btcdata_archive.schema import TABLES
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, objects):
|
||||
self.objects = objects
|
||||
self.client = self
|
||||
self.downloads = []
|
||||
|
||||
def get_paginator(self, operation):
|
||||
assert operation == 'list_objects_v2'
|
||||
return self
|
||||
|
||||
def paginate(self, **kwargs):
|
||||
# Multiple pages; counts must include every object, not only the first page.
|
||||
for key, data in self.objects.items():
|
||||
yield {'Contents': [{'Key': key, 'Size': len(data)}]}
|
||||
|
||||
def head(self, key):
|
||||
data = self.objects[key]
|
||||
return {'VersionId': 'pinned-version', 'ContentLength': len(data),
|
||||
'Metadata': {'sha256': hashlib.sha256(data).hexdigest()}}
|
||||
|
||||
def download(self, key, version, path):
|
||||
assert version == 'pinned-version'
|
||||
# Previously downloaded report files must already have been removed.
|
||||
assert not list(path.parent.parent.glob('btcdata-report-*/archive.sqlite'))
|
||||
self.downloads.append(key)
|
||||
path.write_bytes(self.objects[key])
|
||||
|
||||
|
||||
def archived_file(path, populated=True):
|
||||
sqlite_archive.create(path)
|
||||
db = sqlite_archive.connect(path)
|
||||
if populated:
|
||||
sqlite_archive.upsert(db, TABLES[0], [(1, '1.00000000', '2026-02-16T00:00:00.000000Z'),
|
||||
(2, '2.00000000', '2026-02-16T00:00:00.000000Z')])
|
||||
sqlite_archive.upsert(db, TABLES[1], [(1, 'null', '2026-02-16T00:00:00.000000Z', None)])
|
||||
# Deliberately incorrect metadata; reporting must count the actual SQLite rows.
|
||||
db.execute('INSERT INTO metadata VALUES (?,?)', ('tables', '{"binance_price":{"count":99999}}'))
|
||||
db.commit()
|
||||
db.close()
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def system(tmp_path):
|
||||
first = archived_file(tmp_path / 'first.sqlite')
|
||||
second = archived_file(tmp_path / 'second.sqlite', populated=False)
|
||||
config = SimpleNamespace(bucket='test', prefix='btcprod/weeks/', work_dir=tmp_path / 'work')
|
||||
store = Store({'btcprod/weeks/first.sqlite': first, 'btcprod/weeks/second.sqlite': second,
|
||||
'btcprod/weeks/readme.txt': b'not sqlite'})
|
||||
return config, store
|
||||
|
||||
|
||||
def test_report_actual_sqlite_counts_pagination_cleanup(system):
|
||||
config, store = system
|
||||
result = list(report.files(store, config))
|
||||
assert result[0][2] == {'binance_price': 2, 'strike_price': 1, 'kraken_minute': 0}
|
||||
assert result[1][2] == dict.fromkeys(report.TABLE_NAMES, 0)
|
||||
assert result[2] == ('btcprod/weeks/readme.txt', 10, None)
|
||||
assert store.downloads == ['btcprod/weeks/first.sqlite', 'btcprod/weeks/second.sqlite']
|
||||
assert not list(config.work_dir.iterdir())
|
||||
|
||||
|
||||
def test_report_output_and_totals(system, capsys):
|
||||
config, store = system
|
||||
report.show(store, config)
|
||||
output = capsys.readouterr().out
|
||||
assert all(name in output for name in report.TABLE_NAMES)
|
||||
assert 'first.sqlite' in output and 'second.sqlite' in output and 'readme.txt' in output
|
||||
assert 'TOTAL' in output and '3 files' in output
|
||||
assert '99999' not in output
|
||||
total_line = next(line for line in output.splitlines() if line.startswith('TOTAL'))
|
||||
assert total_line.split()[-3:] == ['2', '1', '0']
|
||||
|
||||
|
||||
def test_report_config_requires_no_postgres(monkeypatch):
|
||||
monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False)
|
||||
for key in ('PGHOST','PGUSER','PGPASSWORD'):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
monkeypatch.setenv('PGPORT', 'not a port')
|
||||
monkeypatch.setenv('S3_ENDPOINT_URL', 'http://127.0.0.1:9000')
|
||||
monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test')
|
||||
monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test')
|
||||
assert Config.from_env(need_pg=False).pg == {}
|
||||
|
||||
|
||||
def test_report_main_without_postgres(system, monkeypatch):
|
||||
config, store = system
|
||||
monkeypatch.setattr(report.Config, 'from_env', lambda **kwargs: config if kwargs == {'need_pg': False} else pytest.fail('PG requested'))
|
||||
monkeypatch.setattr(report, 'ObjectStore', lambda config: store)
|
||||
# Importing this reporting module in isolation must not even load psycopg.
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
result = subprocess.run([sys.executable, '-c',
|
||||
'import sys; import btcdata_archive.report; assert "psycopg" not in sys.modules; assert "btcdata_archive.postgres" not in sys.modules'],
|
||||
env={**os.environ, 'PYTHONPATH': str(root / 'src')}, capture_output=True)
|
||||
assert result.returncode == 0
|
||||
assert report.main([]) == 0
|
||||
|
||||
|
||||
def test_report_corrupt_download_cleans_up(system):
|
||||
config, store = system
|
||||
store.objects['btcprod/weeks/first.sqlite'] = b'not a database'
|
||||
with pytest.raises(Exception):
|
||||
list(report.files(store, config))
|
||||
assert not list(config.work_dir.iterdir())
|
||||
|
||||
|
||||
def test_report_checksum_mismatch(system, monkeypatch):
|
||||
config, store = system
|
||||
head = store.head
|
||||
def wrong_checksum(key):
|
||||
result = head(key)
|
||||
result['Metadata']['sha256'] = '0' * 64
|
||||
return result
|
||||
monkeypatch.setattr(store, 'head', wrong_checksum)
|
||||
with pytest.raises(SafetyError, match='SHA-256'):
|
||||
list(report.files(store, config))
|
||||
assert not list(config.work_dir.iterdir())
|
||||
|
||||
|
||||
def test_report_disk_space(system, monkeypatch):
|
||||
config, store = system
|
||||
monkeypatch.setattr(shutil, 'disk_usage', lambda path: SimpleNamespace(free=0))
|
||||
with pytest.raises(SafetyError, match='storage'):
|
||||
list(report.files(store, config))
|
||||
assert not store.downloads
|
||||
|
||||
|
||||
def test_report_missing_table(tmp_path):
|
||||
path = tmp_path / 'bad.sqlite'
|
||||
sqlite_archive.create(path)
|
||||
db = sqlite_archive.connect(path)
|
||||
db.execute('DROP TABLE strike_price')
|
||||
db.commit()
|
||||
db.close()
|
||||
with pytest.raises(SafetyError, match='required table'):
|
||||
report.sqlite_counts(path)
|
||||
|
||||
|
||||
def test_report_empty_listing(capsys, tmp_path):
|
||||
report.show(Store({}), SimpleNamespace(bucket='test',prefix='btcprod/weeks/',work_dir=tmp_path))
|
||||
assert '0 files' in capsys.readouterr().out
|
||||
@@ -0,0 +1,213 @@
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
import shutil
|
||||
import subprocess
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from btcdata_archive.archive import run
|
||||
from btcdata_archive import sqlite_archive
|
||||
from btcdata_archive.schema import TABLES
|
||||
from btcdata_archive.weeks import timestamp
|
||||
from test_contract import START
|
||||
|
||||
|
||||
class PG:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.data = {START: [(1, '1.00000000', timestamp(START))], START + timedelta(days=14): [(3, '3.00000000', timestamp(START + timedelta(days=14)))]}
|
||||
self.states = {}
|
||||
self.events = []
|
||||
self.locked = False
|
||||
self.change = False
|
||||
|
||||
def lock(self):
|
||||
self.locked = True
|
||||
|
||||
def alive(self):
|
||||
if not self.locked:
|
||||
raise RuntimeError('session lost')
|
||||
|
||||
def check_schema(self):
|
||||
pass
|
||||
|
||||
def first_week(self, cutoff):
|
||||
return min([s for s, data in self.data.items() if data and s < cutoff] + [s for s, state in self.states.items() if state != 'complete'], default=None)
|
||||
|
||||
def counts(self, start, end):
|
||||
return {t.name: len(self.data.get(start, [])) if t == TABLES[0] else 0 for t in TABLES}
|
||||
|
||||
def archived_version(self, start):
|
||||
return None
|
||||
|
||||
def estimated_bytes(self, start):
|
||||
return 1024
|
||||
|
||||
def ledger(self, start, status, *args, **kwargs):
|
||||
self.alive()
|
||||
self.events.append((start, status))
|
||||
self.states[start] = status
|
||||
|
||||
def export(self, path, start):
|
||||
self.alive()
|
||||
db = sqlite_archive.connect(path)
|
||||
sqlite_archive.upsert(db, TABLES[0], self.data.get(start, []))
|
||||
db.commit()
|
||||
db.close()
|
||||
|
||||
def prune(self, path, start, cutoff, progress):
|
||||
self.alive()
|
||||
if self.change:
|
||||
self.data[start][0] = (1, '9.00000000', timestamp(start))
|
||||
self.change = False
|
||||
db = sqlite_archive.connect(path, readonly=True)
|
||||
archived = set(db.execute('SELECT * FROM binance_price'))
|
||||
db.close()
|
||||
old = self.data.get(start, [])
|
||||
self.data[start] = [r for r in old if r not in archived]
|
||||
progress({'binance_price': len(old) - len(self.data[start])})
|
||||
if self.data[start]:
|
||||
raise RuntimeError('reconciliation required')
|
||||
return {'binance_price': len(old)}
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self):
|
||||
self.objects = {}
|
||||
self.fail = None
|
||||
self.pg = None
|
||||
|
||||
def require_versioning(self):
|
||||
if self.fail == 'versioning':
|
||||
raise RuntimeError('disabled')
|
||||
|
||||
def head(self, key):
|
||||
if key not in self.objects:
|
||||
return None
|
||||
version, data, checksum = self.objects[key][-1]
|
||||
return {'VersionId': version, 'ContentLength': len(data), 'Metadata': {'sha256': checksum}}
|
||||
|
||||
def upload(self, key, path, checksum):
|
||||
if self.fail == 'upload':
|
||||
raise OSError('sensitive password')
|
||||
version = str(len(self.objects.get(key, [])) + 1)
|
||||
self.objects.setdefault(key, []).append((version, path.read_bytes(), checksum))
|
||||
if self.fail == 'lock_loss':
|
||||
self.pg.locked = False
|
||||
return version
|
||||
|
||||
def download(self, key, version, path):
|
||||
if self.fail == 'download':
|
||||
raise OSError('sensitive payload')
|
||||
data = next(data for v, data, sha in self.objects[key] if v == version)
|
||||
path.write_bytes(data if self.fail != 'checksum' else data + b'bad')
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def system(tmp_path):
|
||||
config = SimpleNamespace(prefix='btcprod/weeks/', work_dir=tmp_path, pg={'dbname': 'btcprod'}, batch_rows=2)
|
||||
pg, store = PG(config), Store()
|
||||
store.pg = pg
|
||||
return config, pg, store
|
||||
|
||||
|
||||
def test_sequential_gap_and_fixed_cutoff(system):
|
||||
config, pg, store = system
|
||||
cutoff = START + timedelta(days=21)
|
||||
run(pg, store, config, cutoff, 'archive-and-prune')
|
||||
assert [s for s, state in pg.events if state == 'exporting'] == [START + timedelta(days=d) for d in (0,7,14)]
|
||||
assert len(store.objects) == 3
|
||||
assert all(not values for values in pg.data.values())
|
||||
for i, (s, state) in enumerate(pg.events):
|
||||
if state == 'exporting' and s != START:
|
||||
assert pg.events[i-1][1] == 'complete'
|
||||
assert not list(config.work_dir.iterdir())
|
||||
|
||||
|
||||
def test_verify_only_bounded_resume_preserves_old_rows(system, tmp_path):
|
||||
config, pg, store = system
|
||||
cutoff = START + timedelta(days=21)
|
||||
run(pg, store, config, cutoff, 'verify-only', 1)
|
||||
assert pg.states[START] == 'verified'
|
||||
assert pg.data[START]
|
||||
run(pg, store, config, cutoff, 'archive-and-prune', 1)
|
||||
assert not pg.data[START]
|
||||
pg.data[START] = [(2, '2.00000000', timestamp(START))]
|
||||
run(pg, store, config, cutoff, 'archive-and-prune', 1)
|
||||
key = config.prefix + START.date().isoformat() + '.sqlite'
|
||||
path = tmp_path / 'restore.sqlite'
|
||||
store.download(key, store.head(key)['VersionId'], path)
|
||||
db = sqlite_archive.connect(path, readonly=True)
|
||||
assert db.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(1,), (2,)]
|
||||
db.close()
|
||||
assert len(store.objects[key]) == 3
|
||||
|
||||
|
||||
@pytest.mark.parametrize('failure', ['upload', 'download', 'checksum', 'versioning', 'lock_loss'])
|
||||
def test_failure_never_prunes_unverified_or_advances(system, failure, capsys):
|
||||
config, pg, store = system
|
||||
store.fail = failure
|
||||
with pytest.raises(Exception):
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune')
|
||||
assert pg.data[START]
|
||||
assert not any(s > START for s, state in pg.events)
|
||||
assert 'sensitive' not in capsys.readouterr().out
|
||||
assert not list(config.work_dir.iterdir())
|
||||
|
||||
|
||||
def test_changed_row_reconciliation(system):
|
||||
config, pg, store = system
|
||||
pg.change = True
|
||||
with pytest.raises(RuntimeError):
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune')
|
||||
assert pg.states[START] == 'failed'
|
||||
assert pg.data[START][0][1] == '9.00000000'
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune', 1)
|
||||
assert not pg.data[START]
|
||||
|
||||
|
||||
def test_full_disk(system, monkeypatch):
|
||||
config, pg, store = system
|
||||
monkeypatch.setattr(shutil, 'disk_usage', lambda p: SimpleNamespace(free=0))
|
||||
with pytest.raises(RuntimeError, match='storage'):
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune')
|
||||
assert pg.data[START]
|
||||
assert not store.objects
|
||||
|
||||
|
||||
def test_missing_previously_verified_object(system, monkeypatch):
|
||||
config, pg, store = system
|
||||
monkeypatch.setattr(pg, 'archived_version', lambda start: 'previous-version')
|
||||
with pytest.raises(RuntimeError, match='operator recovery'):
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune')
|
||||
assert pg.data[START]
|
||||
assert not store.objects
|
||||
|
||||
|
||||
def test_empty(system):
|
||||
config, pg, store = system
|
||||
pg.data.clear()
|
||||
run(pg, store, config, START + timedelta(days=21), 'archive-and-prune')
|
||||
assert not store.objects
|
||||
|
||||
|
||||
def test_wrapper_interpreter_override_and_arguments(tmp_path):
|
||||
interpreter = tmp_path / 'python-test'
|
||||
interpreter.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\nexit 17\n')
|
||||
interpreter.chmod(0o755)
|
||||
script = Path('scripts/archive-old-weeks.sh').resolve()
|
||||
result = subprocess.run([str(script), '--mode', 'verify-only', '--max-weeks', '1'],
|
||||
cwd=tmp_path, env={**os.environ, 'PYTHON': str(interpreter)},
|
||||
capture_output=True, text=True)
|
||||
assert result.returncode == 17
|
||||
assert result.stdout.splitlines() == ['-m', 'btcdata_archive', 'run', '--mode', 'verify-only', '--max-weeks', '1']
|
||||
|
||||
|
||||
def test_wrapper_requires_mode():
|
||||
import sys
|
||||
env = {**os.environ, 'PATH': str(Path(sys.executable).parent) + ':' + os.environ['PATH']}
|
||||
result = subprocess.run(['scripts/archive-old-weeks.sh'], env=env, capture_output=True, text=True)
|
||||
assert result.returncode == 2
|
||||
assert '--mode' in result.stderr
|
||||
Reference in New Issue
Block a user