from pathlib import Path from types import SimpleNamespace import hashlib import os import shutil import subprocess import sys import pytest from btcdata_archive.config import Config from btcdata_archive.errors import SafetyError from btcdata_archive import report, sqlite_archive from btcdata_archive.schema import TABLES class Store: def __init__(self, objects): self.objects = objects self.client = self self.downloads = [] def get_paginator(self, operation): assert operation == 'list_objects_v2' return self def paginate(self, **kwargs): # Multiple pages; counts must include every object, not only the first page. for key, data in self.objects.items(): yield {'Contents': [{'Key': key, 'Size': len(data)}]} def head(self, key): data = self.objects[key] return {'VersionId': 'pinned-version', 'ContentLength': len(data), 'Metadata': {'sha256': hashlib.sha256(data).hexdigest()}} def download(self, key, version, path): assert version == 'pinned-version' # Previously downloaded report files must already have been removed. assert not list(path.parent.parent.glob('btcdata-report-*/archive.sqlite')) self.downloads.append(key) path.write_bytes(self.objects[key]) def archived_file(path, populated=True): sqlite_archive.create(path) db = sqlite_archive.connect(path) if populated: sqlite_archive.upsert(db, TABLES[0], [(1, '1.00000000', '2026-02-16T00:00:00.000000Z'), (2, '2.00000000', '2026-02-16T00:00:00.000000Z')]) sqlite_archive.upsert(db, TABLES[1], [(1, 'null', '2026-02-16T00:00:00.000000Z', None)]) # Deliberately incorrect metadata; reporting must count the actual SQLite rows. db.execute('INSERT INTO metadata VALUES (?,?)', ('tables', '{"binance_price":{"count":99999}}')) db.commit() db.close() return path.read_bytes() @pytest.fixture def system(tmp_path): first = archived_file(tmp_path / 'first.sqlite') second = archived_file(tmp_path / 'second.sqlite', populated=False) config = SimpleNamespace(bucket='test', prefix='btcprod/weeks/', work_dir=tmp_path / 'work') store = Store({'btcprod/weeks/first.sqlite': first, 'btcprod/weeks/second.sqlite': second, 'btcprod/weeks/readme.txt': b'not sqlite'}) return config, store def test_report_actual_sqlite_counts_pagination_cleanup(system): config, store = system result = list(report.files(store, config)) assert result[0][2] == {'binance_price': 2, 'strike_price': 1, 'kraken_minute': 0} assert result[1][2] == dict.fromkeys(report.TABLE_NAMES, 0) assert result[2] == ('btcprod/weeks/readme.txt', 10, None) assert store.downloads == ['btcprod/weeks/first.sqlite', 'btcprod/weeks/second.sqlite'] assert not list(config.work_dir.iterdir()) def test_report_output_and_totals(system, capsys): config, store = system report.show(store, config) output = capsys.readouterr().out assert all(name in output for name in report.TABLE_NAMES) assert 'first.sqlite' in output and 'second.sqlite' in output and 'readme.txt' in output assert 'TOTAL' in output and '3 files' in output assert '99999' not in output total_line = next(line for line in output.splitlines() if line.startswith('TOTAL')) assert total_line.split()[-3:] == ['2', '1', '0'] def test_report_config_requires_no_postgres(monkeypatch): monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False) for key in ('PGHOST','PGUSER','PGPASSWORD'): monkeypatch.delenv(key, raising=False) monkeypatch.setenv('PGPORT', 'not a port') monkeypatch.setenv('S3_ENDPOINT_URL', 'http://127.0.0.1:9000') monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test') monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test') assert Config.from_env(need_pg=False).pg == {} def test_report_main_without_postgres(system, monkeypatch): config, store = system monkeypatch.setattr(report.Config, 'from_env', lambda **kwargs: config if kwargs == {'need_pg': False} else pytest.fail('PG requested')) monkeypatch.setattr(report, 'ObjectStore', lambda config: store) # Importing this reporting module in isolation must not even load psycopg. root = Path(__file__).resolve().parents[2] result = subprocess.run([sys.executable, '-c', 'import sys; import btcdata_archive.report; assert "psycopg" not in sys.modules; assert "btcdata_archive.postgres" not in sys.modules'], env={**os.environ, 'PYTHONPATH': str(root / 'src')}, capture_output=True) assert result.returncode == 0 assert report.main([]) == 0 def test_report_corrupt_download_cleans_up(system): config, store = system store.objects['btcprod/weeks/first.sqlite'] = b'not a database' with pytest.raises(Exception): list(report.files(store, config)) assert not list(config.work_dir.iterdir()) def test_report_checksum_mismatch(system, monkeypatch): config, store = system head = store.head def wrong_checksum(key): result = head(key) result['Metadata']['sha256'] = '0' * 64 return result monkeypatch.setattr(store, 'head', wrong_checksum) with pytest.raises(SafetyError, match='SHA-256'): list(report.files(store, config)) assert not list(config.work_dir.iterdir()) def test_report_disk_space(system, monkeypatch): config, store = system monkeypatch.setattr(shutil, 'disk_usage', lambda path: SimpleNamespace(free=0)) with pytest.raises(SafetyError, match='storage'): list(report.files(store, config)) assert not store.downloads def test_report_missing_table(tmp_path): path = tmp_path / 'bad.sqlite' sqlite_archive.create(path) db = sqlite_archive.connect(path) db.execute('DROP TABLE strike_price') db.commit() db.close() with pytest.raises(SafetyError, match='required table'): report.sqlite_counts(path) def test_report_empty_listing(capsys, tmp_path): report.show(Store({}), SimpleNamespace(bucket='test',prefix='btcprod/weeks/',work_dir=tmp_path)) assert '0 files' in capsys.readouterr().out