158 lines
6.1 KiB
Python
158 lines
6.1 KiB
Python
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
from btcdata_archive.config import Config
|
|
from btcdata_archive.errors import SafetyError
|
|
from btcdata_archive import report, sqlite_archive
|
|
from btcdata_archive.schema import TABLES
|
|
|
|
|
|
class Store:
|
|
def __init__(self, objects):
|
|
self.objects = objects
|
|
self.client = self
|
|
self.downloads = []
|
|
|
|
def get_paginator(self, operation):
|
|
assert operation == 'list_objects_v2'
|
|
return self
|
|
|
|
def paginate(self, **kwargs):
|
|
# Multiple pages; counts must include every object, not only the first page.
|
|
for key, data in self.objects.items():
|
|
yield {'Contents': [{'Key': key, 'Size': len(data)}]}
|
|
|
|
def head(self, key):
|
|
data = self.objects[key]
|
|
return {'VersionId': 'pinned-version', 'ContentLength': len(data),
|
|
'Metadata': {'sha256': hashlib.sha256(data).hexdigest()}}
|
|
|
|
def download(self, key, version, path):
|
|
assert version == 'pinned-version'
|
|
# Previously downloaded report files must already have been removed.
|
|
assert not list(path.parent.parent.glob('btcdata-report-*/archive.sqlite'))
|
|
self.downloads.append(key)
|
|
path.write_bytes(self.objects[key])
|
|
|
|
|
|
def archived_file(path, populated=True):
|
|
sqlite_archive.create(path)
|
|
db = sqlite_archive.connect(path)
|
|
if populated:
|
|
sqlite_archive.upsert(db, TABLES[0], [(1, '1.00000000', '2026-02-16T00:00:00.000000Z'),
|
|
(2, '2.00000000', '2026-02-16T00:00:00.000000Z')])
|
|
sqlite_archive.upsert(db, TABLES[1], [(1, 'null', '2026-02-16T00:00:00.000000Z', None)])
|
|
# Deliberately incorrect metadata; reporting must count the actual SQLite rows.
|
|
db.execute('INSERT INTO metadata VALUES (?,?)', ('tables', '{"binance_price":{"count":99999}}'))
|
|
db.commit()
|
|
db.close()
|
|
return path.read_bytes()
|
|
|
|
|
|
@pytest.fixture
|
|
def system(tmp_path):
|
|
first = archived_file(tmp_path / 'first.sqlite')
|
|
second = archived_file(tmp_path / 'second.sqlite', populated=False)
|
|
config = SimpleNamespace(bucket='test', prefix='btcprod/weeks/', work_dir=tmp_path / 'work')
|
|
store = Store({'btcprod/weeks/first.sqlite': first, 'btcprod/weeks/second.sqlite': second,
|
|
'btcprod/weeks/readme.txt': b'not sqlite'})
|
|
return config, store
|
|
|
|
|
|
def test_report_actual_sqlite_counts_pagination_cleanup(system):
|
|
config, store = system
|
|
result = list(report.files(store, config))
|
|
assert result[0][2] == {'binance_price': 2, 'strike_price': 1, 'kraken_minute': 0}
|
|
assert result[1][2] == dict.fromkeys(report.TABLE_NAMES, 0)
|
|
assert result[2] == ('btcprod/weeks/readme.txt', 10, None)
|
|
assert store.downloads == ['btcprod/weeks/first.sqlite', 'btcprod/weeks/second.sqlite']
|
|
assert not list(config.work_dir.iterdir())
|
|
|
|
|
|
def test_report_output_and_totals(system, capsys):
|
|
config, store = system
|
|
report.show(store, config)
|
|
output = capsys.readouterr().out
|
|
assert all(name in output for name in report.TABLE_NAMES)
|
|
assert 'first.sqlite' in output and 'second.sqlite' in output and 'readme.txt' in output
|
|
assert 'TOTAL' in output and '3 files' in output
|
|
assert '99999' not in output
|
|
total_line = next(line for line in output.splitlines() if line.startswith('TOTAL'))
|
|
assert total_line.split()[-3:] == ['2', '1', '0']
|
|
|
|
|
|
def test_report_config_requires_no_postgres(monkeypatch):
|
|
monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False)
|
|
for key in ('PGHOST','PGUSER','PGPASSWORD'):
|
|
monkeypatch.delenv(key, raising=False)
|
|
monkeypatch.setenv('PGPORT', 'not a port')
|
|
monkeypatch.setenv('S3_ENDPOINT_URL', 'http://127.0.0.1:9000')
|
|
monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test')
|
|
monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test')
|
|
assert Config.from_env(need_pg=False).pg == {}
|
|
|
|
|
|
def test_report_main_without_postgres(system, monkeypatch):
|
|
config, store = system
|
|
monkeypatch.setattr(report.Config, 'from_env', lambda **kwargs: config if kwargs == {'need_pg': False} else pytest.fail('PG requested'))
|
|
monkeypatch.setattr(report, 'ObjectStore', lambda config: store)
|
|
# Importing this reporting module in isolation must not even load psycopg.
|
|
root = Path(__file__).resolve().parents[2]
|
|
result = subprocess.run([sys.executable, '-c',
|
|
'import sys; import btcdata_archive.report; assert "psycopg" not in sys.modules; assert "btcdata_archive.postgres" not in sys.modules'],
|
|
env={**os.environ, 'PYTHONPATH': str(root / 'src')}, capture_output=True)
|
|
assert result.returncode == 0
|
|
assert report.main([]) == 0
|
|
|
|
|
|
def test_report_corrupt_download_cleans_up(system):
|
|
config, store = system
|
|
store.objects['btcprod/weeks/first.sqlite'] = b'not a database'
|
|
with pytest.raises(Exception):
|
|
list(report.files(store, config))
|
|
assert not list(config.work_dir.iterdir())
|
|
|
|
|
|
def test_report_checksum_mismatch(system, monkeypatch):
|
|
config, store = system
|
|
head = store.head
|
|
def wrong_checksum(key):
|
|
result = head(key)
|
|
result['Metadata']['sha256'] = '0' * 64
|
|
return result
|
|
monkeypatch.setattr(store, 'head', wrong_checksum)
|
|
with pytest.raises(SafetyError, match='SHA-256'):
|
|
list(report.files(store, config))
|
|
assert not list(config.work_dir.iterdir())
|
|
|
|
|
|
def test_report_disk_space(system, monkeypatch):
|
|
config, store = system
|
|
monkeypatch.setattr(shutil, 'disk_usage', lambda path: SimpleNamespace(free=0))
|
|
with pytest.raises(SafetyError, match='storage'):
|
|
list(report.files(store, config))
|
|
assert not store.downloads
|
|
|
|
|
|
def test_report_missing_table(tmp_path):
|
|
path = tmp_path / 'bad.sqlite'
|
|
sqlite_archive.create(path)
|
|
db = sqlite_archive.connect(path)
|
|
db.execute('DROP TABLE strike_price')
|
|
db.commit()
|
|
db.close()
|
|
with pytest.raises(SafetyError, match='required table'):
|
|
report.sqlite_counts(path)
|
|
|
|
|
|
def test_report_empty_listing(capsys, tmp_path):
|
|
report.show(Store({}), SimpleNamespace(bucket='test',prefix='btcprod/weeks/',work_dir=tmp_path))
|
|
assert '0 files' in capsys.readouterr().out
|