Implement verified weekly archives and SQLite inventory reporting

tests/unit/test_report.py

@@ -0,0 +1,157 @@
+from pathlib import Path
+from types import SimpleNamespace
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+from btcdata_archive.config import Config
+from btcdata_archive.errors import SafetyError
+from btcdata_archive import report, sqlite_archive
+from btcdata_archive.schema import TABLES
+
+
+class Store:
+    def __init__(self, objects):
+        self.objects = objects
+        self.client = self
+        self.downloads = []
+
+    def get_paginator(self, operation):
+        assert operation == 'list_objects_v2'
+        return self
+
+    def paginate(self, **kwargs):
+        # Multiple pages; counts must include every object, not only the first page.
+        for key, data in self.objects.items():
+            yield {'Contents': [{'Key': key, 'Size': len(data)}]}
+
+    def head(self, key):
+        data = self.objects[key]
+        return {'VersionId': 'pinned-version', 'ContentLength': len(data),
+                'Metadata': {'sha256': hashlib.sha256(data).hexdigest()}}
+
+    def download(self, key, version, path):
+        assert version == 'pinned-version'
+        # Previously downloaded report files must already have been removed.
+        assert not list(path.parent.parent.glob('btcdata-report-*/archive.sqlite'))
+        self.downloads.append(key)
+        path.write_bytes(self.objects[key])
+
+
+def archived_file(path, populated=True):
+    sqlite_archive.create(path)
+    db = sqlite_archive.connect(path)
+    if populated:
+        sqlite_archive.upsert(db, TABLES[0], [(1, '1.00000000', '2026-02-16T00:00:00.000000Z'),
+                                            (2, '2.00000000', '2026-02-16T00:00:00.000000Z')])
+        sqlite_archive.upsert(db, TABLES[1], [(1, 'null', '2026-02-16T00:00:00.000000Z', None)])
+    # Deliberately incorrect metadata; reporting must count the actual SQLite rows.
+    db.execute('INSERT INTO metadata VALUES (?,?)', ('tables', '{"binance_price":{"count":99999}}'))
+    db.commit()
+    db.close()
+    return path.read_bytes()
+
+
+@pytest.fixture
+def system(tmp_path):
+    first = archived_file(tmp_path / 'first.sqlite')
+    second = archived_file(tmp_path / 'second.sqlite', populated=False)
+    config = SimpleNamespace(bucket='test', prefix='btcprod/weeks/', work_dir=tmp_path / 'work')
+    store = Store({'btcprod/weeks/first.sqlite': first, 'btcprod/weeks/second.sqlite': second,
+                   'btcprod/weeks/readme.txt': b'not sqlite'})
+    return config, store
+
+
+def test_report_actual_sqlite_counts_pagination_cleanup(system):
+    config, store = system
+    result = list(report.files(store, config))
+    assert result[0][2] == {'binance_price': 2, 'strike_price': 1, 'kraken_minute': 0}
+    assert result[1][2] == dict.fromkeys(report.TABLE_NAMES, 0)
+    assert result[2] == ('btcprod/weeks/readme.txt', 10, None)
+    assert store.downloads == ['btcprod/weeks/first.sqlite', 'btcprod/weeks/second.sqlite']
+    assert not list(config.work_dir.iterdir())
+
+
+def test_report_output_and_totals(system, capsys):
+    config, store = system
+    report.show(store, config)
+    output = capsys.readouterr().out
+    assert all(name in output for name in report.TABLE_NAMES)
+    assert 'first.sqlite' in output and 'second.sqlite' in output and 'readme.txt' in output
+    assert 'TOTAL' in output and '3 files' in output
+    assert '99999' not in output
+    total_line = next(line for line in output.splitlines() if line.startswith('TOTAL'))
+    assert total_line.split()[-3:] == ['2', '1', '0']
+
+
+def test_report_config_requires_no_postgres(monkeypatch):
+    monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False)
+    for key in ('PGHOST','PGUSER','PGPASSWORD'):
+        monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv('PGPORT', 'not a port')
+    monkeypatch.setenv('S3_ENDPOINT_URL', 'http://127.0.0.1:9000')
+    monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test')
+    monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test')
+    assert Config.from_env(need_pg=False).pg == {}
+
+
+def test_report_main_without_postgres(system, monkeypatch):
+    config, store = system
+    monkeypatch.setattr(report.Config, 'from_env', lambda **kwargs: config if kwargs == {'need_pg': False} else pytest.fail('PG requested'))
+    monkeypatch.setattr(report, 'ObjectStore', lambda config: store)
+    # Importing this reporting module in isolation must not even load psycopg.
+    root = Path(__file__).resolve().parents[2]
+    result = subprocess.run([sys.executable, '-c',
+                             'import sys; import btcdata_archive.report; assert "psycopg" not in sys.modules; assert "btcdata_archive.postgres" not in sys.modules'],
+                            env={**os.environ, 'PYTHONPATH': str(root / 'src')}, capture_output=True)
+    assert result.returncode == 0
+    assert report.main([]) == 0
+
+
+def test_report_corrupt_download_cleans_up(system):
+    config, store = system
+    store.objects['btcprod/weeks/first.sqlite'] = b'not a database'
+    with pytest.raises(Exception):
+        list(report.files(store, config))
+    assert not list(config.work_dir.iterdir())
+
+
+def test_report_checksum_mismatch(system, monkeypatch):
+    config, store = system
+    head = store.head
+    def wrong_checksum(key):
+        result = head(key)
+        result['Metadata']['sha256'] = '0' * 64
+        return result
+    monkeypatch.setattr(store, 'head', wrong_checksum)
+    with pytest.raises(SafetyError, match='SHA-256'):
+        list(report.files(store, config))
+    assert not list(config.work_dir.iterdir())
+
+
+def test_report_disk_space(system, monkeypatch):
+    config, store = system
+    monkeypatch.setattr(shutil, 'disk_usage', lambda path: SimpleNamespace(free=0))
+    with pytest.raises(SafetyError, match='storage'):
+        list(report.files(store, config))
+    assert not store.downloads
+
+
+def test_report_missing_table(tmp_path):
+    path = tmp_path / 'bad.sqlite'
+    sqlite_archive.create(path)
+    db = sqlite_archive.connect(path)
+    db.execute('DROP TABLE strike_price')
+    db.commit()
+    db.close()
+    with pytest.raises(SafetyError, match='required table'):
+        report.sqlite_counts(path)
+
+
+def test_report_empty_listing(capsys, tmp_path):
+    report.show(Store({}), SimpleNamespace(bucket='test',prefix='btcprod/weeks/',work_dir=tmp_path))
+    assert '0 files' in capsys.readouterr().out