commit 885f618e6631493add16dcbf2fb1be69ece024d2 Author: Per Magnus Petersen Date: Sat Oct 3 19:42:42 2026 +0200 Implement verified weekly archives and SQLite inventory reporting diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5d643bd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +.venv +.env +.env.* +**/__pycache__ +**/*.egg-info +.pytest_cache +tests +*.sqlite +*.sqlite-* diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..65f8700 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,50 @@ +name: test-and-build +on: + push: + branches: [main, master] + pull_request: + workflow_dispatch: + +env: + IMAGE: harbor.pmagnus.cloud/btcdata-archive/btcdata-archive + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12.12' + - run: | + python -m pip install --require-hashes -r requirements-test.lock + export PYTHONPATH="$PWD/src" + echo "PYTHONPATH=$PYTHONPATH" >> "$GITHUB_ENV" + python -m pytest -q tests/unit + - run: bash scripts/test-integration.sh + build: + needs: test + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Immutable commit tag + id: tag + run: echo "image=$IMAGE:sha-$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host + - uses: docker/login-action@v3 + with: + registry: harbor.pmagnus.cloud + username: ${{ secrets.HARBOR_ROBOT_USER }} + password: ${{ secrets.HARBOR_ROBOT_TOKEN }} + - uses: docker/build-push-action@v6 + id: image + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.tag.outputs.image }} + - name: Deployment reference + run: echo '${{ env.IMAGE }}@${{ steps.image.outputs.digest }}' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..356888f --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.venv/ +.work/ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.env +.env.* +*.sqlite +*.sqlite-* +build/ +dist/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f1da402 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.12.12-slim-bookworm +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.lock ./ +RUN pip install --no-cache-dir --require-hashes -r requirements.lock \ + && mkdir /work && chown 10001:10001 /work +COPY src/ ./src/ +COPY scripts/ ./scripts/ +USER 10001:10001 +ENV ARCHIVE_WORK_DIR=/work +ENTRYPOINT ["python", "-m", "btcdata_archive"] +CMD ["--help"] diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..41fa833 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,277 @@ +# btcdata-archive implementation handoff + +Status: **IMPLEMENTED LOCALLY — restricted RustFS IAM identity, versioned archive bucket, PostgreSQL ledger and restricted runtime role provisioned on explicit requests; source index preflight, deployment and deletion have not been performed.** +Implementation artifacts and validation instructions are in `README.md`. The disposable +PostgreSQL/RustFS suite is prepared but requires a working Docker environment before rollout. +Prepared: 2026-10-03. +Target repository: `/Users/pmagnus/pi/btcdata-archive`. + +## 1. Objective and agreed decisions + +Build a standalone archiver for PostgreSQL database `btcprod`. Every Monday at 00:00 UTC, archive expired data from three fixed tables into one SQLite database per UTC week in RustFS. Delete source rows only after the uploaded archive has been downloaded and verified. Immediately after successful cleanup, PostgreSQL retains the last four completed UTC weeks plus any rows arriving in the current week. + +User decisions: + +- Tables: `public.strike_price`, `public.binance_price`, `public.kraken_minute`. +- Choose these tables once; do not dynamically select the largest tables each run. +- Archive only when a week expires, not immediately after that week finishes. +- One SQLite file per week containing all three tables. +- Dedicated RustFS bucket: `btcdata-archive`; prefix: `btcprod/weeks/`. +- Keep archives indefinitely. One verified RustFS copy is sufficient before PostgreSQL deletion; independent backups are outside this project. +- Initial catch-up must process **all old weeks, oldest first, strictly one week at a time**. Provide a dedicated script for this operation. +- Run the recurring archiver in Kubernetes namespace `sejs` as a CronJob pinned to Citadel and managed through the existing GitOps repository. +- Preserve PostgreSQL data when uploads or verification fail; retry and catch up later. +- Archived data is downloaded and queried manually. No historical-query API or application UI is required. +- At planning time this document was the only authorized deliverable. The subsequent request to implement this plan authorizes local code, tests, dependencies and setup/deployment artifacts, not production provisioning, jobs or source deletion. + +## 2. Evidence and environment + +Read-only inspection on 2026-10-03 found these PostgreSQL sizes, including indexes and auxiliary storage: + +| Table | Estimated live rows | Total storage | +| --- | ---: | ---: | +| strike_price | 1,727,101 | 1,148 MB | +| binance_price | 3,308,108 | 258 MB | +| kraken_minute | 294,591 | 35 MB | + +Estimates are planning evidence, not export counts. Recheck schemas, constraints, privileges, source volume, and free space before implementation rollout. + +Relevant existing repositories: + +- `/Users/pmagnus/pi/btcdata`: Clojure application, migrations in `resources/migrations`, SQL in `resources/queries.sql`. +- `/Users/pmagnus/pi/deploy`: GitOps manifests; btcdata currently has namespace `btcdata` and runs on Citadel or `srv1893094`. +- `/Users/pmagnus/pi/system/source/rustfs-xtdb-implementation.md`: infrastructure record for RustFS. + +Recorded infrastructure: + +- PostgreSQL: Citadel, documented LAN address `192.168.50.195:5432`, database `btcprod`. +- RustFS: documented S3 endpoint `http://192.168.50.45:9000`, with path-style access; credentials are held outside Git. +- RustFS record describes a 200 GiB data disk and existing bucket `xtdb`. Use a separate `btcdata-archive` bucket. +- Endpoint reachability, RustFS versioning behavior, available storage, and current Kubernetes configuration have not been validated for this archiver. +- The existing application holds advisory lock `728194106` for its own writer fencing. The archiver must use a different lock and must not start or contend with the application's writer components. + +Read applicable `AGENTS.md` and skills before touching any repository or host. Never print credential-bearing JDBC URLs, `.env` contents, or tokens. Git initialization and any remote repository creation are future work; no remote hosting destination has been selected. + +## 3. Retention and week calculation + +Use UTC exclusively, independent of host timezone and Europe/Copenhagen daylight saving. + +At invocation start: + +1. Capture the current UTC instant once. +2. Find Monday 00:00 UTC at the beginning of that instant's week, called `anchor`. +3. Set `cutoff = anchor - 28 days`. +4. Keep this cutoff fixed for the invocation, including a long catch-up run. +5. Eligible rows have their retention timestamp strictly less than `cutoff`. + +Weeks are half-open intervals `[Monday 00:00, next Monday 00:00)`. Archive only complete weeks ending at or before the cutoff. + +Use `recorded_at` for Binance and Strike. Use Unix-second `ts` for Kraken minute candles, not their insertion-time `created_at`. + +Example: invocation on Monday 2026-10-05 has cutoff 2026-09-07 00:00 UTC. In steady state, it archives 2026-08-31 through just before 2026-09-07. It retains 2026-09-07 onward. A row exactly at the cutoff stays in PostgreSQL. + +Weekly cleanup means the retained span grows from approximately 28 days to approximately 35 days between successful runs. Failed runs can leave more data. Do not add daily pruning or strict rolling-28-day enforcement. + +## 4. Standalone project layout and runtime + +Use Python with pinned dependencies: `psycopg` 3 for PostgreSQL, `boto3` for S3, and Python's standard `sqlite3`. Use `pytest` for tests. Pin a supported Python base image and dependency versions when implementing; do not choose floating production image tags. + +Proposed layout: + +```text +btcdata-archive/ + PLAN.md + README.md + pyproject.toml + + src/btcdata_archive/ + cli.py + config.py + weeks.py + postgres.py + sqlite_archive.py + object_store.py + archive.py + scripts/ + archive-old-weeks.sh + sql/ + setup.sql + indexes.sql + tests/ + unit/ + integration/ + Dockerfile + .dockerignore + .gitignore +``` + +Implementation stays in this new repository. Deployment manifests belong in the existing deploy repository, following its instructions. Do not add archiving logic or dependencies to the btcdata application's startup path. + +Runtime configuration: + +- `PGHOST`, `PGPORT`, `PGDATABASE=btcprod`, `PGUSER`, `PGPASSWORD`; support `PGSSLMODE` as appropriate to the existing database deployment. +- `S3_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION=us-east-1`. +- `ARCHIVE_BUCKET=btcdata-archive`, `ARCHIVE_PREFIX=btcprod/weeks/`. +- `ARCHIVE_WORK_DIR=/work`, `ARCHIVE_BATCH_ROWS=5000`. + +Fail clearly if required configuration is missing. Source credentials from Kubernetes Secrets, never tracked files or CLI arguments containing passwords. Use boto3 path-style S3 addressing. Do not silently fall back to a public AWS endpoint. + +## 5. CLI and initial catch-up script + +Provide these interfaces: + +- `python -m btcdata_archive inventory`: read-only report of anchor, cutoff, oldest eligible week, candidate weeks, and source counts. No bucket writes or source deletion. +- `python -m btcdata_archive run --mode verify-only`: export and upload all eligible weeks sequentially, verify them, but do not delete source rows. +- `python -m btcdata_archive run --mode archive-and-prune`: the production workflow over all eligible weeks. +- Both run modes accept optional `--max-weeks N` for a bounded rollout/test. Omission means all eligible weeks. Reject zero or negative values. +- `python -m btcdata_archive status`: report last completed week, incomplete runs, and remaining expired source data. + +`python -m btcdata_archive` requires a package `__main__.py` routing to the CLI. + +### Required script: `scripts/archive-old-weeks.sh` + +Implement a thin executable wrapper around the same tested CLI used by the CronJob. It must not contain a second implementation of archiving or deletion. + +- Require an explicit `--mode verify-only` or `--mode archive-and-prune`; no implicit destructive default. +- Forward optional `--max-weeks N`. +- Read all connection settings from the environment. +- Use strict shell error handling and `exec` the Python CLI so exit codes and termination signals propagate correctly. +- By default process **every eligible old week**, starting with the oldest, one at a time. +- Complete export, upload verification, and (in prune mode) source deletion for a week before advancing to the next week. +- Stop immediately on failure or unresolved changed rows. Do not skip a failed week and continue with later weeks. +- Print the fixed cutoff and a progress line for each week: started, verified, pruned, complete, or failed, with table counts. +- Rerunning the script resumes safely from archived objects and PostgreSQL progress records; it must not require a local checkpoint file. +- No parallel week processing, background subprocess exports, or loading the entire backlog into memory/disk. + +Discover the earliest eligible timestamp across the three tables, combine this with incomplete ledger entries, and enumerate weeks in ascending order through the cutoff. Create an empty weekly database for an internal gap between eligible weeks so the archive sequence is explicit. Do not create years of empty archives preceding the first actual source week. Recheck for late rows in old weeks on every new invocation. + +Suggested future operator usage: + +```sh +./scripts/archive-old-weeks.sh --mode verify-only --max-weeks 1 +./scripts/archive-old-weeks.sh --mode archive-and-prune +``` + +These are implementation targets, not commands to run while preparing this document. + +## 6. SQLite schema and object contract + +Object key: `btcprod/weeks/YYYY-MM-DD.sqlite`, where the date is the starting Monday. Example: `btcprod/weeks/2026-08-31.sqlite`. + +Every file includes: + +- `binance_price`: `id INTEGER PRIMARY KEY`, `price TEXT NOT NULL`, `recorded_at TEXT NOT NULL`. +- `strike_price`: `id INTEGER PRIMARY KEY`, `rates TEXT NOT NULL`, `recorded_at TEXT NOT NULL`, `quote_sats INTEGER NULL`. +- `kraken_minute`: `ts INTEGER PRIMARY KEY`; `open`, `high`, `low`, `close`, `vwap`, and `volume` as non-null TEXT; `trade_count INTEGER NOT NULL`; `created_at TEXT NOT NULL`. +- Indexes on `(recorded_at, id)` for the two price tables. +- Metadata tables containing archive format version, source database/schema, source column types, week start/end, creation time, per-table row counts, and canonical content checksums. + +Preserve PostgreSQL numeric values as exact decimal strings; never convert through Python float or SQLite REAL. Preserve all timestamp microseconds using a fixed UTC ISO-8601 representation. Preserve SQL NULL distinctly from JSON null. JSONB's original whitespace/key order is not meaningful; export deterministic JSON preserving exact numeric values. Obtain JSONB as text and canonicalize it without float conversion. + +Canonical row checksums use primary-key order and a documented deterministic encoding with explicit nulls and field boundaries (canonical JSON arrays of typed string/integer/null values are suitable). Use SHA-256. Compare the source stream's count/digest to a fresh SQLite read before upload. + +Check the live source schema against the supported schema before exporting. An added, removed, or incompatible column must fail closed with a clear migration-required error; never silently omit columns. Future schema changes require an explicit archive format migration and tests. + +Use a self-contained SQLite file: commit and close all connections before hashing/uploading. Do not rely on a separate WAL or journal file. Run `PRAGMA integrity_check`, requiring `ok`. + +## 7. Safe export, verification, deletion, and recovery + +Hold PostgreSQL advisory lock `728194107` on the same live session coordinating the run. If unavailable, exit without starting work. If that session is lost, stop; never silently reconnect and continue deleting. This lock only serializes archivers, not the application's writers. + +For each week: + +1. Check available working storage. Only keep files needed for the current week, including any downloaded prior archive and verification copy. Abort safely on inadequate space or disk errors. +2. If the object already exists, download its exact version and validate its schema, metadata, and integrity. Preserve all previously archived rows, including rows already deleted from PostgreSQL. +3. Read the source week's rows in a read-only repeatable-read transaction, using server-side cursors and bounded batches. Merge source rows into the local weekly archive by primary key. Current PostgreSQL values supersede that key's earlier archived values; prior object versions retain earlier revisions. Close the source snapshot before network upload. +4. Verify exported source rows against SQLite, including counts, values, and canonical hashes. Validate the full merged archive and recompute metadata. +5. Upload the finalized file. Require bucket versioning to be enabled and capture the returned object version ID. Use SHA-256 in object metadata, but do not treat ETag or metadata alone as proof of content. +6. Download that exact uploaded version into a separate local file. Verify its full SHA-256, integrity, schema, week boundaries, and table checksums. Persist the verified version ID and checksum in the ledger. +7. In verify-only mode, stop processing this week here and record it as verified, not pruned. +8. In prune mode, read bounded batches of rows from the verified downloaded file. Stage each batch in a PostgreSQL temporary table using the source column types. Delete using primary-key matches and null-safe equality of **every source column**, plus the week's timestamp bounds and fixed cutoff. Use `IS NOT DISTINCT FROM` comparisons, including typed JSONB and numeric comparisons. Commit each batch independently. +9. Recount eligible source rows for this week. If any remain, record an incomplete/reconciliation-required state, stop the run, and retain all remaining rows. A new invocation merges these rows into a new verified object version before attempting deletion again. +10. Mark the week pruned only after the final source check finds no rows. Remove current-week temporary files and proceed to the next oldest week. + +Do not hold table locks or a long write transaction during export/upload. Do not delete by timestamp alone. A row changed after export must fail the content match and remain in PostgreSQL. A late insertion after the final check is discovered on the next invocation. + +Durable progress: create schema `btcdata_archive` and a `weeks` ledger keyed by `week_start`, with `week_end`, format version, status, object key, object version ID, file SHA-256, JSONB row/deletion counts, attempt time, completion time, and last error. Statuses: `exporting`, `verified`, `pruning`, `complete`, `failed`. Do not store credentials or entire row payloads in errors. + +Recovery rules: + +- Crash before verified upload: source stays intact; rebuild or merge on retry. +- Crash after upload but before ledger update: validate the existing object and merge current source rows; never assume ledger absence means the object is empty. +- Crash during deletion: reuse/merge the full archived file, retaining already-pruned records, and continue idempotently. +- Completed ledger entry with new old source rows: reopen that week and reconcile. +- Corrupt or incompatible existing archive: stop and require operator recovery; never overwrite it with a partial source remainder. +- Failed verification or unavailable RustFS: no source deletion for that week. + +## 8. Database preparation and deployment + +Future setup SQL must be explicit and operator-run; the runtime job must not create database roles or migrate schemas automatically. + +- Add `(recorded_at, id)` indexes for the two price tables with `CREATE INDEX CONCURRENTLY`, outside transaction-wrapped migrations. Inspect existing equivalent indexes first. +- Reuse the `kraken_minute.ts` primary-key index. +- Give the runtime role SELECT and DELETE only on the three source tables, TEMP privilege for staging, and the necessary privileges on its own ledger. No application writer lock or INSERT/UPDATE privilege on source tables is needed. +- Use a bucket-scoped identity for list/get/version-get/put and necessary multipart operations. Enable versioning during provisioning with an administrative identity. Do not grant the runtime identity bucket administration or object-version deletion. +- Preserve bucket objects and all versions indefinitely; do not configure expiry or cleanup of completed archive versions. + +Kubernetes defaults: + +- Namespace `sejs`, CronJob name `btcdata-archive`. Provision the namespace through GitOps if it does not already exist. Create the archiver's Secrets, ServiceAccount, and any namespaced configuration in `sejs`; Secrets cannot be referenced across namespaces. The existing btcdata application remains in its current namespace. +- `schedule: "0 0 * * 1"`, `timeZone: Etc/UTC`, `concurrencyPolicy: Forbid`. +- Pin node affinity to `citadel`, where both database and RustFS LAN access should be validated. +- Execute the CLI in `archive-and-prune` mode; same sequential workflow as the catch-up script. +- Job `backoffLimit: 3`; missed schedules can start within six days (`startingDeadlineSeconds: 518400`). Each execution computes its own fixed current-week cutoff and catches up all overdue weeks. +- Retain three successful and three failed Job records; `restartPolicy: Never`; termination grace 60 seconds. +- Initial requests: 100m CPU, 256 MiB memory, 2 GiB ephemeral storage. Limits: 1 CPU, 1 GiB memory, 8 GiB ephemeral storage. Disk-backed `/work` emptyDir with 8 GiB size limit. Revalidate these against the oldest/largest actual week during preflight; increase before enabling if needed. +- Use a non-root container and an immutable image published by the new repository's CI. Follow the existing deploy repository's approved registry and GitOps conventions. +- Start with `suspend: true`; enabling is a separate rollout step after validation. + +Use structured logs for week boundaries, counts, verification state, deletion progress, durations, and failures. Return nonzero on errors. The ledger and `status` CLI provide durable status. External email/Slack alerts are outside scope; do not create notification integrations without a separate request. + +## 9. Tests and acceptance criteria + +Unit tests: + +- Monday boundary calculations, Sunday runs, year/ISO-week crossings, leap dates, and DST independence. +- Exactly-at-cutoff rows remain; exactly-at-week-start rows belong to that week. +- Timestamp column selection differs correctly for candles versus price snapshots. +- Exact decimal round trips, large IDs, null quote_sats, JSON numeric precision, UTC microseconds, empty tables, and deterministic checksums. +- Strict schema mismatch detection and validated CLI arguments. + +Integration tests with disposable PostgreSQL and the target RustFS image: + +- Multiple weeks export in ascending order into one SQLite file per week, with all three tables. +- Script processes all old weeks, holds a fixed cutoff, and never starts the next week until the current one completes. +- Internal empty week, partial first week, no eligible data, and bounded `--max-weeks` runs. +- Verify-only mode creates verified archives without deleting source data. +- Upload/download failure, checksum mismatch, SQLite corruption, full disk, and missing privileges preserve unverified source rows. +- Kill at each workflow boundary and restart; previously archived/deleted rows must not disappear from the final archive. +- Concurrent archiver exclusion and lock-session loss. +- Source changes and late arrivals during export/deletion remain unpruned until a later verified reconciliation. +- Restore a downloaded archive into an isolated PostgreSQL schema using its documented type mapping and compare every value/count/hash. +- Existing application latest-price and latest-candle reads remain usable while archival runs. + +Success means every deleted row exists in a verified RustFS object version with equivalent values; after a successful stable run, the selected source tables contain no rows older than the fixed cutoff. All other tables and application APIs are unchanged. + +## 10. Rollout sequence and operator documentation + +1. Implement and test entirely against disposable infrastructure. Initialize version control only when implementation is authorized; remote creation is separate from this plan. +2. Inspect live schemas/indexes, oldest timestamps, per-week sizes, disk capacity, versioning support, and cluster connectivity without exposing secrets. +3. Provision the bucket/versioning, restricted identities, ledger, and indexes through explicit setup steps. +4. Deploy the CronJob suspended. Run one oldest week in verify-only mode using the same production image. +5. Download that object, verify it independently, and restore into an isolated test database/schema. Confirm exact values and types. +6. Invoke the catch-up script in archive-and-prune mode with no week limit. It must process the entire backlog, oldest first, one week at a time. Monitor source activity and stop on the first failure. +7. Confirm the retention cutoff, archive inventory, checksums, and ledger agree; enable the Monday CronJob through GitOps. +8. Document download/query examples, exact decimal handling, manual invocation, status checks, retry recovery, and how to suspend the CronJob. + +Rollback: suspend the CronJob through GitOps and stop any active archiver gracefully. Committed deletions are not undone by rollback. Recovery requires restoring the verified SQLite archives; preserve objects and ledger records. Use normal PostgreSQL vacuum/autovacuum for reusable space. `VACUUM FULL`, immediate filesystem shrinkage, table partitioning, and redesigning application storage are outside scope. + +## 11. References + +- Kubernetes CronJobs: https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/ +- SQLite type behavior: https://sqlite.org/datatype3.html +- Existing application migrations and queries in the btcdata repository are the starting schema reference; live PostgreSQL schema is authoritative during preflight. + +## 12. Next-agent instruction + +Read this file fully and inspect applicable repository instructions. If asked only to revise the plan, change only this document. If explicitly authorized to implement, create the standalone project and catch-up script described here, complete automated tests, and prepare deployment/setup artifacts. Do not infer production deployment or source-data deletion authorization from a request to write code. Report any live-schema or infrastructure incompatibility before changing the agreed retention or archive contract. diff --git a/README.md b/README.md new file mode 100644 index 0000000..12c862e --- /dev/null +++ b/README.md @@ -0,0 +1,285 @@ +# btcdata-archive + +Standalone Python archiver for the three fixed `btcprod` tables: `public.binance_price`, +`public.strike_price`, and `public.kraken_minute`. **A restricted RustFS IAM identity +and versioned archive bucket, progress ledger and restricted PostgreSQL role have +been provisioned; source index preflight and deployment are still pending.** +An operator run verified the first weekly archive, then hit a ledger transition bug +before deletion. That bug is now fixed; no deployment has been performed. Remote +repository hosting and CI/Harbor provisioning remain operator decisions. + +## Install and test + +Python 3.12+; production image pins Python 3.12.12 (Debian Bookworm). + +```sh +python3.12 -m venv .venv +source .venv/bin/activate +python -m pip install --require-hashes -r requirements-test.lock +export PYTHONPATH="$PWD/src" +python -m pytest -q +# Requires Docker, only starts disposable PostgreSQL 18.0 and RustFS 1.0.0: +./scripts/test-integration.sh +``` + +The integration fixture **always creates a new random database, login and bucket**, +then removes only those test resources. To use already-running *disposable* services, +set `TEST_PGHOST`, `TEST_PGPORT`, `TEST_PGUSER`, `TEST_PGPASSWORD`, +`TEST_S3_ENDPOINT_URL`, `TEST_S3_ACCESS_KEY`, `TEST_S3_SECRET_KEY`. Never point tests +at production infrastructure. Full S3 integration tests skip without both required +TEST endpoints. PostgreSQL-only regressions require only the disposable PostgreSQL +TEST variables and can run with `python -m pytest -q tests/integration/test_postgres.py`. +Local validation of those regressions plus unit tests passed 43 tests on disposable +PostgreSQL 18.6; the complete RustFS integration suite remains a rollout gate. +Boundary/crash tests inject interruptions at workflow boundaries; actual process +kill/network/disk fault testing and production connectivity remain rollout gates. + +`requirements.lock` pins runtime packages and hashes; `requirements-test.lock` +adds pytest. Regenerate with Python 3.12, pip 25.2 and pip-tools 7.5.1: +`pip-compile --generate-hashes -o requirements.lock pyproject.toml` and +`pip-compile --extra test --generate-hashes -o requirements-test.lock pyproject.toml`. +CI tests before publishing a full-commit tag to the existing approved Harbor registry. +Deploy using the published digest, not a mutable tag. No remote has been configured. + +## Configuration + +Supply credentials through environment/Secret injection, never arguments or tracked +files. For local use, the catch-up wrapper automatically reads the repository's +untracked `.env` (keep mode `0600`). Set `ARCHIVE_ENV_FILE` to choose another file, +including for direct CLI use. Existing environment variables take precedence. +Entries use literal `KEY=value` syntax with optional shell-style quoting/comments; +no shell commands or variable interpolation are executed. Fill blank S3 credential +fields with the restricted archive identity before running. Required PostgreSQL variables: `PGHOST`, `PGUSER`, `PGPASSWORD`. +Defaults: `PGPORT=5432`, `PGDATABASE=btcprod`, `PGSSLMODE=prefer`. +Runs also require `S3_ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`. +S3 uses path-style addressing, with **no fallback to an AWS public endpoint**. + +Optional defaults: `AWS_DEFAULT_REGION=us-east-1`, `ARCHIVE_BUCKET=btcdata-archive`, +`ARCHIVE_PREFIX=btcprod/weeks/`, `ARCHIVE_WORK_DIR=/work`, +`ARCHIVE_BATCH_ROWS=5000`. Inventory/status do not require S3 credentials. The +SQLite file report requires only S3 settings, not PostgreSQL credentials. +Use a private work directory, accessible only to the runtime identity. The CLI +sanitizes unexpected exception messages; it does not dump SQL, row data, SDK responses, +connection strings or secrets. Database/SDK error classes are logged rather than raw +messages; investigate those failures via privileged tooling without publishing secrets. + +```sh +python -m btcdata_archive inventory +python -m btcdata_archive status +./scripts/archive-old-weeks.sh --mode verify-only --max-weeks 1 +# Destructive; run ONLY after explicit rollout approval: +./scripts/archive-old-weeks.sh --mode archive-and-prune +``` + +The catch-up wrapper uses `PYTHON` when explicitly set, otherwise the active virtual +environment, the repository's `.venv/bin/python`, or `python3`. It adds the repository's +`src` to `PYTHONPATH`; dependencies must still be installed in the selected interpreter. + +Both run modes require explicit `--mode`; optional `--max-weeks` must be positive. +Omission processes **all** eligible weeks, oldest first, one at a time. A failure or +unresolved changed row stops the invocation immediately. No local checkpoint is needed. +Inventory/status require the operator-created ledger and supported source schema. + +## List file sizes and SQLite table counts + +```sh +./scripts/list-archives.sh +``` + +This read-only report loads `.env` using the same interpreter selection as the +catch-up wrapper. It lists every current object under `ARCHIVE_PREFIX`, showing MiB +sizes and `binance_price`, `strike_price`, and `kraken_minute` row counts, plus totals. +Counts come from actual `SELECT COUNT(*)` queries on the downloaded SQLite tables, +**not archive metadata or PostgreSQL**. It neither imports the PostgreSQL backend +nor requires PostgreSQL connection settings. + +Each SQLite object's exact current version is downloaded one at a time into a private +temporary directory under `ARCHIVE_WORK_DIR`. The report checks file size, advertised +SHA-256 when present, and SQLite integrity, opens SQLite read-only, counts the tables, +and deletes its temporary copy before the next file. It stops on corrupt/missing +archives or insufficient disk space. Non-SQLite objects are listed with `-` counts. +Historical object versions are excluded; files published after listing begins may +appear only on the next invocation. Large inventories require downloading every +SQLite file again; this does not run archival, update the ledger, or delete any data. + +## Retention and safety + +One UTC instant is captured at CLI start. Its Monday 00:00 UTC is the anchor; cutoff +is anchor minus 28 days. Weekly intervals are half-open. Rows exactly at cutoff stay. +Price tables use `recorded_at`; candles use Unix-second `ts`, **not `created_at`**. +A Monday 2026-10-05 run has cutoff 2026-09-07: it retains four completed weeks plus +the current week's arrivals. Weekly scheduling permits about 28–35 days between runs; +failed runs retain more. There is no daily pruning. + +The same PostgreSQL session holds advisory lock **728194107** throughout the run. +It never takes the application's writer lock 728194106, starts application writers, +or reconnects a lost lock session. Export uses bounded server-side cursors in a +read-only repeatable-read transaction, which closes before network operations. + +For each week the archiver: + +1. Checks conservative available-disk estimates (existing object, text expansion, + SQLite journal, download copy and 64 MiB reserve). Disk errors fail safely. +2. Downloads an existing exact version, validates integrity/schema/metadata/hashes, + and preserves previously archived rows. Missing previously verified objects fail + closed; recover with an operator rather than rebuilding from a partial remainder. +3. Overlays current source rows by primary key. Source stream hashes/counts must match + fresh SQLite reads of those keys. Internal gaps produce explicit empty weekly files; + there are no empty files before the first source/incomplete week. +4. Closes/commits SQLite, validates the full archive, hashes it and uploads it with + SHA-256 metadata. **Bucket versioning must already be enabled.** The multipart + completion's version ID—not a subsequent HEAD—is the upload identity. +5. Downloads that exact uploaded version separately and verifies file SHA-256, + SQLite integrity, schema, week bounds, table counts and canonical content digests. + ETags and metadata alone are never content proof. +6. Records `verified` in `btcdata_archive.weeks`. Verify-only stops this week here, + leaves source rows untouched, then advances to the next week. +7. In prune mode stages bounded batches from the verified download into typed TEMP + tables. Deletes require primary-key matches, `IS NOT DISTINCT FROM` equality of + **every column**, week bounds and fixed cutoff. Each batch commits independently. +8. Recounts source rows; only zero remaining rows produces `complete`. Changed/late + rows produce `failed` with reconciliation required, and stop later-week processing. + +A late insertion after the final count is discovered next invocation. Repeated runs +reopen completed weeks with new old rows. Retry merges the full existing object, +including already-pruned records, and writes another indefinitely retained version. +Ledger statuses: `exporting`, `verified`, `pruning`, `complete`, `failed`. +Deletion counts describe committed batches of the current attempt, not lifetime totals. +A crash between batch commit and progress update can leave underreported progress; +source equality and final recount—not progress counts—govern recovery and completion. + +## SQLite format 1 + +Keys: `btcprod/weeks/YYYY-MM-DD.sqlite`, Monday's date. All three source tables are +always present. PKs/trade counts/quote_sats are SQLite INTEGER; decimals/JSON/timestamps +are TEXT. Price tables have `(recorded_at,id)` indexes. `metadata(key,value)` holds +JSON-encoded format version, source database/schema/types, week bounds, creation time, +per-table counts and checksums. Source columns/types/nullability/PKs must match the +supported migrations exactly; additions/removals/incompatibilities require an explicit +format migration. RLS, nonordinary tables and custom DELETE triggers are rejected. + +Decimals retain PostgreSQL's exact decimal string (including scale), never float/REAL. +Timestamps are UTC `YYYY-MM-DDTHH:MM:SS.ffffffZ`. JSONB text is sorted by Unicode key, +compact, with decimal parsing/serialization and no binary floating-point conversion. +JSON null remains part of JSON text, distinct from SQL NULL. JSONB whitespace and +original key order are not preserved. + +Canonical checksum: rows in ascending PK order, encoded as compact UTF-8 JSON arrays +of integer/string/null values, `ensure_ascii=False`, followed by one LF per row; +SHA-256 over that byte stream (empty stream for an empty table). Field types/boundaries +and nulls are unambiguous. Each file is closed, journal-free and self-contained. + +## Download, verify and query manually + +Get the exact `object_key`, `object_version_id`, and `file_sha256` from the ledger. +With restricted credentials already supplied via environment: + +```sh +aws --endpoint-url "$S3_ENDPOINT_URL" s3api get-object \ + --bucket btcdata-archive --key btcprod/weeks/2026-08-31.sqlite \ + --version-id "$VERSION_ID" /work/2026-08-31.sqlite +sha256sum /work/2026-08-31.sqlite # compare with the verified ledger SHA-256 +sqlite3 /work/2026-08-31.sqlite 'PRAGMA integrity_check;' +sqlite3 /work/2026-08-31.sqlite 'SELECT key,value FROM metadata;' +sqlite3 /work/2026-08-31.sqlite 'SELECT id,price,recorded_at FROM binance_price LIMIT 10;' +``` + +Full independent format/content validation using the installed package: + +```python +from datetime import datetime, timezone +from pathlib import Path +from btcdata_archive.sqlite_archive import validate, file_hash +p = Path('/work/2026-08-31.sqlite') +print(file_hash(p)) # compare with ledger, not only object metadata +print(validate(p, datetime(2026, 8, 31, tzinfo=timezone.utc), 'btcprod', 5000)) +``` + +For exact arithmetic use Python `Decimal(price_text)`, not SQLite casts to REAL or +Python float. Restore first into an **isolated database/schema** with original types: +BIGINT PKs, NUMERIC(18,8) price/OHLC/VWAP, NUMERIC(24,8) volume, JSONB rates, +TIMESTAMPTZ timestamps, INTEGER trade_count, nullable BIGINT quote_sats. Iterate SQLite +in batches; `schema.decode_row(table,row)` converts decimal/timestamp values exactly. +Insert JSON text with an explicit `%s::jsonb` cast; insert the other decoded values +through psycopg parameters. Read restored JSONB as `rates::text`, canonicalize using +`schema.encode_row`, and compare every value/count/digest with the SQLite archive. +`tests/integration/test_archive.py` demonstrates this independent round trip. +Never restore directly over active source tables without a separate recovery plan. + +## Current provisioning state + +On RustFS (`192.168.50.45`), IAM user `btcdata-archive` is attached only to policy +`btcdata-archive-runtime-v1`, matching `sql/s3-runtime-policy.json`. Its credentials +are saved in the ignored local `.env` (mode `0600`), not in tracked files. Negative +live checks confirmed denial of access to the existing `xtdb` bucket, administrative +APIs (including self-admin access), and object deletion. On subsequent explicit +approval, bucket `btcdata-archive` was created with versioning `Enabled` and no +lifecycle/expiry rules. Live checks using the restricted identity confirmed versioning +inspection, archive-prefix object/version listing, and denial of outside-prefix listing. +The operator subsequently verified the 2026-02-16 weekly archive (74,123 Binance +rows) with an exact-version download. Pruning then failed before deletion: PostgreSQL +checks an UPSERT's proposed INSERT before its ON CONFLICT UPDATE, and progress-only +ledger transitions omitted the stored verification fields. The fix preserves those +fields before INSERT checks; read-only diagnosis confirmed all 74,123 source rows +remained. Retry uses the existing verified archive, with no need to reset the ledger. + +On separate explicit approval, `btcdata_archive.weeks` and PostgreSQL login +`btcdata_archiver` were created in `btcprod`. Verified effective privileges are source +SELECT/DELETE, ledger SELECT/INSERT/UPDATE, schema USAGE, and database CONNECT/TEMP. +Source INSERT/UPDATE/TRUNCATE/REFERENCES/TRIGGER and ledger DELETE are denied; the +role has no permanent object creation privilege, elevated attributes or memberships. +A real login/read-only ledger/source-access check succeeded. `.env` now uses this +restricted role instead of the application's administrative credentials. No source +rows or indexes were changed. Index/capacity preflight and deployment remain pending. + +`scripts/rustfs-create-identity.py` is an explicit operator-only provisioning utility, +run on RustFS with access to its administrative credential file. It refuses to overwrite +existing users/policies and disables a newly created identity on incomplete setup. +It is not executed by the runtime job. RustFS 1.0.0 rejects bare `Resource: "*"` and +mixed action families in a single statement; the policy uses separate admin/STS deny +statements and S3 wildcard ARNs compatible with the deployed build. + +## Explicit operator setup and rollout (remaining steps not executed) + +1. Run disposable integration tests. Inspect live schemas, constraints/DELETE triggers, + RLS, existing indexes, source volumes/per-week sizes, source role privileges, + RustFS disk capacity, versioning support and pod LAN connectivity. Live schema + remains authoritative; migration references alone are not a production preflight. +2. Create a restricted PostgreSQL LOGIN through secure administrative tooling. + Run `psql -v ON_ERROR_STOP=1 -v runtime_role=btcdata_archiver -f sql/setup.sql` + against the chosen database as its administrator. This creates only the ledger + schema/table and grants SELECT/DELETE on the fixed source tables, TEMP, and ledger + SELECT/INSERT/UPDATE. Runtime does not create roles/schemas or run migrations. + Verify inherited/public privileges do not confer extra source writes or ownership. +3. Inspect equivalent indexes before running `sql/indexes.sql`, **outside a transaction**. + Inspect interrupted/invalid concurrent builds before retrying. Reuse Kraken's TS PK. +4. With an administrative RustFS identity, create bucket `btcdata-archive` and enable + versioning. Verify version PUT/GET behavior against the deployed RustFS build. + Attach `sql/s3-runtime-policy.json` to a dedicated bucket-scoped identity; validate + RustFS IAM compatibility. Runtime can inspect versioning but cannot enable it, + administer the bucket, or delete objects/versions. No expiry/lifecycle rules. + Provisioning and CI image pushes require separate credentials, never runtime keys. +5. GitOps artifacts are prepared in `../deploy/apps/btcdata-archive/`, **not registered + in the parent apps kustomization**. They remain suspended and have a deliberately + unpullable placeholder digest. Choose a remote, configure CI/Harbor, publish the + tested image and replace that digest before registration. Provision Secrets in + `sejs` through out-of-Git tooling; namespace references cannot cross namespaces. +6. Confirm requests/limits/work capacity against the largest old week. Initial work + limit is 8 GiB; disk estimates are conservative, not a capacity guarantee. Limit + whole-week files require an increase for large weeks; streaming is not parallelism. +7. After explicit approval, deploy suspended; verify-only one oldest week, download and + independently restore/compare it. Then authorize/run the unlimited oldest-first + prune catch-up script. Stop on the first failure. Check source/ledger/object agreement + and fixed cutoff before enabling the Monday 00:00 UTC CronJob through GitOps. + +Suspend to stop future schedules; terminate active work gracefully (SIGTERM exits +nonzero and closes the lock session). SIGKILL can leave local temporary files; disposable +pod storage is cleared with the pod. Standalone operators may remove orphan +`btcdata-week-*` directories **only when no archiver is running**; they are not checkpoints. +Retries use object versions and ledger, not these files. Failed multipart uploads may +need administrative inspection; never expire completed versions. + +Rollback cannot undo committed deletions. Preserve archives and ledger, suspend the +CronJob and restore verified versions through an explicit recovery procedure if needed. +Use normal autovacuum/vacuum for reusable space. VACUUM FULL, filesystem shrinking, +partitioning, notification integrations and historical-query APIs are out of scope. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..30bd235 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools==80.9.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "btcdata-archive" +version = "1.0.0" +requires-python = ">=3.12" +dependencies = ["psycopg[binary]==3.2.10", "boto3==1.40.45"] + +[project.optional-dependencies] +test = ["pytest==8.4.2"] + +[project.scripts] +btcdata-archive = "btcdata_archive.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = ["integration: requires disposable PostgreSQL and RustFS"] diff --git a/requirements-test.lock b/requirements-test.lock new file mode 100644 index 0000000..7ff54a2 --- /dev/null +++ b/requirements-test.lock @@ -0,0 +1,130 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --extra=test --generate-hashes --no-index --output-file=requirements.lock pyproject.toml +# +boto3==1.40.45 \ + --hash=sha256:5b145752d20f29908e3cb8c823bee31c77e6bcf18787e570f36bbc545cc779ed \ + --hash=sha256:e8d794dc1f01729d93dc188c90cf63cd0d32df8818a82ac46e641f6ffcea615e + # via btcdata-archive (pyproject.toml) +botocore==1.40.76 \ + --hash=sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc \ + --hash=sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4 + # via + # boto3 + # s3transfer +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +psycopg[binary]==3.2.10 \ + --hash=sha256:0bce99269d16ed18401683a8569b2c5abd94f72f8364856d56c0389bcd50972a \ + --hash=sha256:ab5caf09a9ec42e314a21f5216dbcceac528e0e05142e42eea83a3b28b320ac3 + # via btcdata-archive (pyproject.toml) +psycopg-binary==3.2.10 \ + --hash=sha256:037dc92fc7d3f2adae7680e17216934c15b919d6528b908ac2eb52aecc0addcf \ + --hash=sha256:0738320a8d405f98743227ff70ed8fac9670870289435f4861dc640cef4a61d3 \ + --hash=sha256:0c23e88e048bbc33f32f5a35981707c9418723d469552dd5ac4e956366e58492 \ + --hash=sha256:0c2b95e83fda70ed2b0b4fadd8538572e4a4d987b721823981862d1ab56cc760 \ + --hash=sha256:14bcbcac0cab465d88b2581e43ec01af4b01c9833e663f1352e05cb41be19e44 \ + --hash=sha256:183a59cbdcd7e156669577fd73a9e917b1ee664e620f1e31ae138d24c7714693 \ + --hash=sha256:1b29285474e3339d0840e1b5079fdb0481914108f92ec62de0c87ae333c60b24 \ + --hash=sha256:1dee2f4d2adc9adacbfecf8254bd82f6ac95cff707e1b9b99aa721cd1ef16b47 \ + --hash=sha256:1f6982609b8ff8fcd67299b67cd5787da1876f3bb28fedd547262cfa8ddedf94 \ + --hash=sha256:2028073fc12cd70ba003309d1439c0c4afab4a7eee7653b8c91213064fffe12b \ + --hash=sha256:20384985fbc650c09a547a13c6d7f91bb42020d38ceafd2b68b7fc4a48a1f160 \ + --hash=sha256:299834cce3eec0c48aae5a5207fc8f0c558fd65f2ceab1a36693329847da956b \ + --hash=sha256:29b6bb87959515bc8b6abef10d8d23a9a681f03e48e9f0c8adb4b9fb7fa73f11 \ + --hash=sha256:3234605839e7d7584bd0a20716395eba34d368a5099dafe7896c943facac98fc \ + --hash=sha256:37b42b2f5f58df1f07a5df1b0c2bcc9bd3b9c105e2e988923bfa47aa4ae967da \ + --hash=sha256:3bb4046973264ebc8cb7e20a83882d68577c1f26a6f8ad4fe52e4468cd9a8eee \ + --hash=sha256:3e115930af2f38f4bbb5f1b61b598ceb802f091c1592c0fe0571c796b714b89a \ + --hash=sha256:42ee399c2613b470a87084ed79b06d9d277f19b0457c10e03a4aef7059097abc \ + --hash=sha256:43d803fb4e108a67c78ba58f3e6855437ca25d56504cae7ebbfbd8fce9b59247 \ + --hash=sha256:447afc326cbc95ed67c0cd27606c0f81fa933b830061e096dbd37e08501cb3de \ + --hash=sha256:470594d303928ab72a1ffd179c9c7bde9d00f76711d6b0c28f8a46ddf56d9807 \ + --hash=sha256:484d2b1659afe0f8f1cef5ea960bb640e96fa864faf917086f9f833f5c7a8034 \ + --hash=sha256:50130c0d1a2a01ec3d41631df86b6c1646c76718be000600a399dc1aad80b813 \ + --hash=sha256:5334a61a00ccb722f0b28789e265c7a273cfd10d5a1ed6bf062686fbb71e7032 \ + --hash=sha256:5369202e0e764193eac311b5a337d8cd58b1e23b822ddb7a559ed9f683d97623 \ + --hash=sha256:55b14f2402be027fe1568bc6c4d75ac34628ff5442a70f74137dadf99f738e3b \ + --hash=sha256:6220d6efd6e2df7b67d70ed60d653106cd3b70c5cb8cbe4e9f0a142a5db14015 \ + --hash=sha256:62590dd113d10cd9c08251cb80b32e2e8aaf01ece04a700322e776b1d216959f \ + --hash=sha256:646048f46192c8d23786cc6ef19f35b7488d4110396391e407eca695fdfe9dcd \ + --hash=sha256:6fe450a98a0788b721b1b8302f0ba9be6eca82faf74bf7a86d794cd6484c7e27 \ + --hash=sha256:70bb7f665587dfd79e69f48b34efe226149454d7aab138ed22d5431d703de2f6 \ + --hash=sha256:725843fd444075cc6c9989f5b25ca83ac68d8d70b58e1f476fbb4096975e43cc \ + --hash=sha256:764a5b9b40ad371c55dfdf95374d89e44a82fd62272d4fceebea0adb8930e2fb \ + --hash=sha256:7950ff79df7a453ac8a7d7a74694055b6c15905b0a2b6e3c99eb59c51a3f9bf7 \ + --hash=sha256:7fa1626225a162924d2da0ff4ef77869f7a8501d320355d2732be5bf2dda6138 \ + --hash=sha256:810f65b9ef1fe9dddb5c05937884ea9563aaf4e1a2c3d138205231ed5f439511 \ + --hash=sha256:8390db6d2010ffcaf7f2b42339a2da620a7125d37029c1f9b72dfb04a8e7be6f \ + --hash=sha256:84f7e8c5e5031db342ae697c2e8fb48cd708ba56990573b33e53ce626445371d \ + --hash=sha256:8923487c3898c65e1450847e15d734bb2e6adbd2e79d2d1dd5ad829a1306bdc0 \ + --hash=sha256:89440355d1b163b11dc661ae64a5667578aab1b80bbf71ced90693d88e9863e1 \ + --hash=sha256:8b45e65383da9c4a42a56f817973e521e893f4faae897fe9f1a971f9fe799742 \ + --hash=sha256:8f4ae059c6c9e491cdc3f39f9fc4f09373ef281c6cc381499269dcff21abafc9 \ + --hash=sha256:8fa2efaf5e2f8c289a185c91c80a624a8f97aa17fbedcbc68f373d089b332afd \ + --hash=sha256:901729188b3fd5625970650ca1167786847dee0b92930c2858724d1a5e25dee1 \ + --hash=sha256:9c9f2728488ac5848acdbf14bb4fde50f8ba783cbf3c19e9abd506741389fa7f \ + --hash=sha256:a024b3ee539a475cbc59df877c8ecdd6f8552a1b522b69196935bc26dc6152fb \ + --hash=sha256:a1d4e4d309049e3cb61269652a3ca56cb598da30ecd7eb8cea561e0d18bc1a43 \ + --hash=sha256:a28f24a7b68456bd31209b027a5b04304d37eb1d622ef847bf8c47933218a738 \ + --hash=sha256:a5a81104d88780018005fe17c37fa55b4afbb6dd3c205963cc56c025d5f1cc32 \ + --hash=sha256:a92ff1c2cd79b3966d6a87e26ceb222ecd5581b5ae4b58961f126af806a861ed \ + --hash=sha256:ab1c6d761c4ee581016823dcc02f29b16ad69177fcbba88a9074c924fc31813e \ + --hash=sha256:ac0365398947879c9827b319217096be727da16c94422e0eb3cf98c930643162 \ + --hash=sha256:b34c278a58aa79562afe7f45e0455b1f4cad5974fc3d5674cc5f1f9f57e97fc5 \ + --hash=sha256:bd3676a04970cf825d2c771b0c147f91182c5a3653e0dbe958e12383668d0f79 \ + --hash=sha256:bf30dcf6aaaa8d4779a20d2158bdf81cc8e84ce8eee595d748a7671c70c7b890 \ + --hash=sha256:d2fe9eaa367f6171ab1a21a7dcb335eb2398be7f8bb7e04a20e2260aedc6f782 \ + --hash=sha256:d557a94cd6d2e775b3af6cc0bd0ff0d9d641820b5cc3060ccf1f5ca2bf971217 \ + --hash=sha256:d5c6a66a76022af41970bf19f51bc6bf87bd10165783dd1d40484bfd87d6b382 \ + --hash=sha256:d7d05174276bb403b8a57e01b857d96b0ac2a6879c5ce06a5cac2d1115763081 \ + --hash=sha256:d922fdd49ed17c558b6b2f9ae2054c3d0cced2a34e079ce5a41c86904d0203f7 \ + --hash=sha256:db0eb06a19e4c64a08db0db80875ede44939af6a2afc281762c338fad5d6e547 \ + --hash=sha256:e037aac8dc894d147ef33056fc826ee5072977107a3fdf06122224353a057598 + # via psycopg +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via pytest +pytest==8.4.2 \ + --hash=sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01 \ + --hash=sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79 + # via btcdata-archive (pyproject.toml) +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore +s3transfer==0.14.0 \ + --hash=sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456 \ + --hash=sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125 + # via boto3 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via psycopg +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via botocore diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..5878e8d --- /dev/null +++ b/requirements.lock @@ -0,0 +1,110 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes --no-index --output-file=requirements.lock pyproject.toml +# +boto3==1.40.45 \ + --hash=sha256:5b145752d20f29908e3cb8c823bee31c77e6bcf18787e570f36bbc545cc779ed \ + --hash=sha256:e8d794dc1f01729d93dc188c90cf63cd0d32df8818a82ac46e641f6ffcea615e + # via btcdata-archive (pyproject.toml) +botocore==1.40.76 \ + --hash=sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc \ + --hash=sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4 + # via + # boto3 + # s3transfer +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # boto3 + # botocore +psycopg[binary]==3.2.10 \ + --hash=sha256:0bce99269d16ed18401683a8569b2c5abd94f72f8364856d56c0389bcd50972a \ + --hash=sha256:ab5caf09a9ec42e314a21f5216dbcceac528e0e05142e42eea83a3b28b320ac3 + # via btcdata-archive (pyproject.toml) +psycopg-binary==3.2.10 \ + --hash=sha256:037dc92fc7d3f2adae7680e17216934c15b919d6528b908ac2eb52aecc0addcf \ + --hash=sha256:0738320a8d405f98743227ff70ed8fac9670870289435f4861dc640cef4a61d3 \ + --hash=sha256:0c23e88e048bbc33f32f5a35981707c9418723d469552dd5ac4e956366e58492 \ + --hash=sha256:0c2b95e83fda70ed2b0b4fadd8538572e4a4d987b721823981862d1ab56cc760 \ + --hash=sha256:14bcbcac0cab465d88b2581e43ec01af4b01c9833e663f1352e05cb41be19e44 \ + --hash=sha256:183a59cbdcd7e156669577fd73a9e917b1ee664e620f1e31ae138d24c7714693 \ + --hash=sha256:1b29285474e3339d0840e1b5079fdb0481914108f92ec62de0c87ae333c60b24 \ + --hash=sha256:1dee2f4d2adc9adacbfecf8254bd82f6ac95cff707e1b9b99aa721cd1ef16b47 \ + --hash=sha256:1f6982609b8ff8fcd67299b67cd5787da1876f3bb28fedd547262cfa8ddedf94 \ + --hash=sha256:2028073fc12cd70ba003309d1439c0c4afab4a7eee7653b8c91213064fffe12b \ + --hash=sha256:20384985fbc650c09a547a13c6d7f91bb42020d38ceafd2b68b7fc4a48a1f160 \ + --hash=sha256:299834cce3eec0c48aae5a5207fc8f0c558fd65f2ceab1a36693329847da956b \ + --hash=sha256:29b6bb87959515bc8b6abef10d8d23a9a681f03e48e9f0c8adb4b9fb7fa73f11 \ + --hash=sha256:3234605839e7d7584bd0a20716395eba34d368a5099dafe7896c943facac98fc \ + --hash=sha256:37b42b2f5f58df1f07a5df1b0c2bcc9bd3b9c105e2e988923bfa47aa4ae967da \ + --hash=sha256:3bb4046973264ebc8cb7e20a83882d68577c1f26a6f8ad4fe52e4468cd9a8eee \ + --hash=sha256:3e115930af2f38f4bbb5f1b61b598ceb802f091c1592c0fe0571c796b714b89a \ + --hash=sha256:42ee399c2613b470a87084ed79b06d9d277f19b0457c10e03a4aef7059097abc \ + --hash=sha256:43d803fb4e108a67c78ba58f3e6855437ca25d56504cae7ebbfbd8fce9b59247 \ + --hash=sha256:447afc326cbc95ed67c0cd27606c0f81fa933b830061e096dbd37e08501cb3de \ + --hash=sha256:470594d303928ab72a1ffd179c9c7bde9d00f76711d6b0c28f8a46ddf56d9807 \ + --hash=sha256:484d2b1659afe0f8f1cef5ea960bb640e96fa864faf917086f9f833f5c7a8034 \ + --hash=sha256:50130c0d1a2a01ec3d41631df86b6c1646c76718be000600a399dc1aad80b813 \ + --hash=sha256:5334a61a00ccb722f0b28789e265c7a273cfd10d5a1ed6bf062686fbb71e7032 \ + --hash=sha256:5369202e0e764193eac311b5a337d8cd58b1e23b822ddb7a559ed9f683d97623 \ + --hash=sha256:55b14f2402be027fe1568bc6c4d75ac34628ff5442a70f74137dadf99f738e3b \ + --hash=sha256:6220d6efd6e2df7b67d70ed60d653106cd3b70c5cb8cbe4e9f0a142a5db14015 \ + --hash=sha256:62590dd113d10cd9c08251cb80b32e2e8aaf01ece04a700322e776b1d216959f \ + --hash=sha256:646048f46192c8d23786cc6ef19f35b7488d4110396391e407eca695fdfe9dcd \ + --hash=sha256:6fe450a98a0788b721b1b8302f0ba9be6eca82faf74bf7a86d794cd6484c7e27 \ + --hash=sha256:70bb7f665587dfd79e69f48b34efe226149454d7aab138ed22d5431d703de2f6 \ + --hash=sha256:725843fd444075cc6c9989f5b25ca83ac68d8d70b58e1f476fbb4096975e43cc \ + --hash=sha256:764a5b9b40ad371c55dfdf95374d89e44a82fd62272d4fceebea0adb8930e2fb \ + --hash=sha256:7950ff79df7a453ac8a7d7a74694055b6c15905b0a2b6e3c99eb59c51a3f9bf7 \ + --hash=sha256:7fa1626225a162924d2da0ff4ef77869f7a8501d320355d2732be5bf2dda6138 \ + --hash=sha256:810f65b9ef1fe9dddb5c05937884ea9563aaf4e1a2c3d138205231ed5f439511 \ + --hash=sha256:8390db6d2010ffcaf7f2b42339a2da620a7125d37029c1f9b72dfb04a8e7be6f \ + --hash=sha256:84f7e8c5e5031db342ae697c2e8fb48cd708ba56990573b33e53ce626445371d \ + --hash=sha256:8923487c3898c65e1450847e15d734bb2e6adbd2e79d2d1dd5ad829a1306bdc0 \ + --hash=sha256:89440355d1b163b11dc661ae64a5667578aab1b80bbf71ced90693d88e9863e1 \ + --hash=sha256:8b45e65383da9c4a42a56f817973e521e893f4faae897fe9f1a971f9fe799742 \ + --hash=sha256:8f4ae059c6c9e491cdc3f39f9fc4f09373ef281c6cc381499269dcff21abafc9 \ + --hash=sha256:8fa2efaf5e2f8c289a185c91c80a624a8f97aa17fbedcbc68f373d089b332afd \ + --hash=sha256:901729188b3fd5625970650ca1167786847dee0b92930c2858724d1a5e25dee1 \ + --hash=sha256:9c9f2728488ac5848acdbf14bb4fde50f8ba783cbf3c19e9abd506741389fa7f \ + --hash=sha256:a024b3ee539a475cbc59df877c8ecdd6f8552a1b522b69196935bc26dc6152fb \ + --hash=sha256:a1d4e4d309049e3cb61269652a3ca56cb598da30ecd7eb8cea561e0d18bc1a43 \ + --hash=sha256:a28f24a7b68456bd31209b027a5b04304d37eb1d622ef847bf8c47933218a738 \ + --hash=sha256:a5a81104d88780018005fe17c37fa55b4afbb6dd3c205963cc56c025d5f1cc32 \ + --hash=sha256:a92ff1c2cd79b3966d6a87e26ceb222ecd5581b5ae4b58961f126af806a861ed \ + --hash=sha256:ab1c6d761c4ee581016823dcc02f29b16ad69177fcbba88a9074c924fc31813e \ + --hash=sha256:ac0365398947879c9827b319217096be727da16c94422e0eb3cf98c930643162 \ + --hash=sha256:b34c278a58aa79562afe7f45e0455b1f4cad5974fc3d5674cc5f1f9f57e97fc5 \ + --hash=sha256:bd3676a04970cf825d2c771b0c147f91182c5a3653e0dbe958e12383668d0f79 \ + --hash=sha256:bf30dcf6aaaa8d4779a20d2158bdf81cc8e84ce8eee595d748a7671c70c7b890 \ + --hash=sha256:d2fe9eaa367f6171ab1a21a7dcb335eb2398be7f8bb7e04a20e2260aedc6f782 \ + --hash=sha256:d557a94cd6d2e775b3af6cc0bd0ff0d9d641820b5cc3060ccf1f5ca2bf971217 \ + --hash=sha256:d5c6a66a76022af41970bf19f51bc6bf87bd10165783dd1d40484bfd87d6b382 \ + --hash=sha256:d7d05174276bb403b8a57e01b857d96b0ac2a6879c5ce06a5cac2d1115763081 \ + --hash=sha256:d922fdd49ed17c558b6b2f9ae2054c3d0cced2a34e079ce5a41c86904d0203f7 \ + --hash=sha256:db0eb06a19e4c64a08db0db80875ede44939af6a2afc281762c338fad5d6e547 \ + --hash=sha256:e037aac8dc894d147ef33056fc826ee5072977107a3fdf06122224353a057598 + # via psycopg +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore +s3transfer==0.14.0 \ + --hash=sha256:ea3b790c7077558ed1f02a3072fb3cb992bbbd253392f4b6e9e8976941c7d456 \ + --hash=sha256:eff12264e7c8b4985074ccce27a3b38a485bb7f7422cc8046fee9be4983e4125 + # via boto3 +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via psycopg +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via botocore diff --git a/scripts/archive-old-weeks.sh b/scripts/archive-old-weeks.sh new file mode 100755 index 0000000..2daf8f8 --- /dev/null +++ b/scripts/archive-old-weeks.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -n "${PYTHON:-}" ]]; then + python_bin="$PYTHON" +elif [[ -n "${VIRTUAL_ENV:-}" && -x "$VIRTUAL_ENV/bin/python" ]]; then + python_bin="$VIRTUAL_ENV/bin/python" +elif [[ -x "$repo_dir/.venv/bin/python" ]]; then + python_bin="$repo_dir/.venv/bin/python" +else + python_bin=python3 +fi +if ! command -v "$python_bin" >/dev/null 2>&1; then + printf 'Python not found. Install Python 3.12+ and dependencies; see README.md.\n' >&2 + exit 127 +fi +# Allow invocation outside the repository without an editable package install. +export PYTHONPATH="$repo_dir/src${PYTHONPATH:+:$PYTHONPATH}" +# Parsed as literal settings by Python; never sourced/executed as shell code. +export ARCHIVE_ENV_FILE="${ARCHIVE_ENV_FILE:-$repo_dir/.env}" +# argparse requires --mode, validates --max-weeks, and rejects unknown options. +exec "$python_bin" -m btcdata_archive run "$@" diff --git a/scripts/list-archives.sh b/scripts/list-archives.sh new file mode 100755 index 0000000..034c68c --- /dev/null +++ b/scripts/list-archives.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail +repo_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -n "${PYTHON:-}" ]]; then + python_bin="$PYTHON" +elif [[ -n "${VIRTUAL_ENV:-}" && -x "$VIRTUAL_ENV/bin/python" ]]; then + python_bin="$VIRTUAL_ENV/bin/python" +elif [[ -x "$repo_dir/.venv/bin/python" ]]; then + python_bin="$repo_dir/.venv/bin/python" +else + python_bin=python3 +fi +if ! command -v "$python_bin" >/dev/null 2>&1; then + printf 'Python not found. Install Python 3.12+ and dependencies; see README.md.\n' >&2 + exit 127 +fi +export PYTHONPATH="$repo_dir/src${PYTHONPATH:+:$PYTHONPATH}" +export ARCHIVE_ENV_FILE="${ARCHIVE_ENV_FILE:-$repo_dir/.env}" +exec "$python_bin" -m btcdata_archive.report "$@" diff --git a/scripts/rustfs-create-identity.py b/scripts/rustfs-create-identity.py new file mode 100644 index 0000000..82ca721 --- /dev/null +++ b/scripts/rustfs-create-identity.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Operator-run on RustFS only. Never logs credentials; does not provision buckets. + +Requires root access to /opt/rustfs/.env. Refuses to overwrite existing identities, +policies, or credential files. Credential delivery file is owner-only for pmp. +""" +import argparse +import datetime +import hashlib +import hmac +import json +import os +from pathlib import Path +import pwd +import re +import secrets +import shlex +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET + + +class Admin: + def __init__(self): + values = {} + for line in Path('/opt/rustfs/.env').read_text().splitlines(): + match = re.fullmatch(r'\s*(?:export\s+)?([A-Za-z_][A-Za-z_0-9]*)\s*=\s*(.*)', line) + if match: + parts = shlex.split(match.group(2), comments=True) + if len(parts) <= 1: + values[match.group(1)] = parts[0] if parts else '' + self.key = values['RUSTFS_ACCESS_KEY'] + self.secret = values['RUSTFS_SECRET_KEY'] + self.endpoint = 'http://127.0.0.1:9000' + + def request(self, method, path, query=None, body=b'', key=None, secret=None, content_type='application/json'): + key = self.key if key is None else key + secret = self.secret if secret is None else secret + query_string = '&'.join(urllib.parse.quote(str(k), safe='-_.~') + '=' + urllib.parse.quote(str(v), safe='-_.~') + for k, v in sorted((query or {}).items())) + now = datetime.datetime.now(datetime.timezone.utc) + date, instant = now.strftime('%Y%m%d'), now.strftime('%Y%m%dT%H%M%SZ') + payload_hash = hashlib.sha256(body).hexdigest() + headers = {'host': urllib.parse.urlsplit(self.endpoint).netloc, + 'x-amz-content-sha256': payload_hash, 'x-amz-date': instant} + signed_headers = ';'.join(sorted(headers)) + canonical_headers = ''.join(k + ':' + headers[k] + '\n' for k in sorted(headers)) + canonical_request = '\n'.join((method, path, query_string, canonical_headers, signed_headers, payload_hash)) + scope = date + '/us-east-1/s3/aws4_request' + to_sign = '\n'.join(('AWS4-HMAC-SHA256', instant, scope, hashlib.sha256(canonical_request.encode()).hexdigest())) + signing_key = ('AWS4' + secret).encode() + for item in (date, 'us-east-1', 's3', 'aws4_request'): + signing_key = hmac.new(signing_key, item.encode(), hashlib.sha256).digest() + signature = hmac.new(signing_key, to_sign.encode(), hashlib.sha256).hexdigest() + headers['Authorization'] = ('AWS4-HMAC-SHA256 Credential=' + key + '/' + scope + + ', SignedHeaders=' + signed_headers + ', Signature=' + signature) + if body: + headers['Content-Type'] = content_type + url = self.endpoint + path + ('?' + query_string if query_string else '') + req = urllib.request.Request(url, data=body if method in ('PUT', 'POST') else None, + headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=30) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + + +def require(status, data, expected, operation): + if status not in expected: + code = 'unknown' + try: + code = ET.fromstring(data).findtext('Code') or 'unknown' + except Exception: + pass + raise SystemExit(operation + ' failed: HTTP ' + str(status) + ', code ' + code) + + +def create(policy_path): + username, policy_name, bucket = 'btcdata-archive', 'btcdata-archive-runtime-v1', 'btcdata-archive' + target = Path('/home/pmp/.config/btcdata-archive') + path = target / 's3.env' + if path.exists(): + raise SystemExit('Credential delivery file already exists; refusing to overwrite') + policy = json.loads(policy_path.read_text()) + admin = Admin() + status, data = admin.request('GET', '/rustfs/admin/v3/list-users') + require(status, data, {200}, 'Administrative authentication') + users = json.loads(data) + if not isinstance(users, dict): + raise SystemExit('Unexpected user listing format') + if username in users: + raise SystemExit('Identity already exists; refusing to overwrite or rotate it') + status, data = admin.request('GET', '/rustfs/admin/v3/list-canned-policies') + require(status, data, {200}, 'Policy listing') + policies = json.loads(data) + if not isinstance(policies, dict): + raise SystemExit('Unexpected policy listing format') + if policy_name in policies: + raise SystemExit('Named policy already exists; refusing to overwrite it') + status, data = admin.request('HEAD', '/' + bucket) + require(status, data, {200, 404}, 'Archive bucket inspection') + bucket_exists = status == 200 + print('Archive bucket present: ' + str(bucket_exists), flush=True) + status, data = admin.request('PUT', '/rustfs/admin/v3/add-canned-policy', {'name': policy_name}, + json.dumps(policy, separators=(',', ':')).encode()) + require(status, data, {200, 204}, 'Restricted policy creation') + secret = secrets.token_urlsafe(40) + created = False + try: + status, data = admin.request('PUT', '/rustfs/admin/v3/add-user', {'accessKey': username}, + json.dumps({'secretKey': secret, 'status': 'enabled'}).encode()) + require(status, data, {200, 204}, 'Identity creation') + created = True + status, data = admin.request('PUT', '/rustfs/admin/v3/set-user-or-group-policy', + {'policyName': policy_name, 'userOrGroup': username, 'isGroup': 'false'}) + require(status, data, {200, 204}, 'Policy attachment') + status, data = admin.request('GET', '/rustfs/admin/v3/user-info', {'accessKey': username}) + require(status, data, {200}, 'Identity verification') + info = json.loads(data) + if info.get('policyName') != policy_name: + raise SystemExit('Unexpected policy binding; refusing credential delivery') + checks = [ + ('other bucket access', 'GET', '/xtdb', {'list-type': '2', 'prefix': 'btcvault/', 'max-keys': '1'}), + ('administrative access', 'GET', '/rustfs/admin/v3/list-users', {}), + ('self-admin access', 'GET', '/rustfs/admin/v3/user-info', {'accessKey': username}), + ('object deletion', 'DELETE', '/xtdb/.identity-deny-probe-' + secrets.token_hex(16), {}), + ] + if bucket_exists: + checks.append(('outside-prefix access', 'GET', '/' + bucket, + {'list-type': '2', 'prefix': 'outside/', 'max-keys': '1'})) + for label, method, route, query in checks: + status, data = admin.request(method, route, query, key=username, secret=secret) + require(status, data, {403}, 'Deny check: ' + label) + print('Verified denied: ' + label, flush=True) + if bucket_exists: + status, data = admin.request('GET', '/' + bucket, + {'list-type': '2', 'prefix': 'btcprod/weeks/', 'max-keys': '1'}, + key=username, secret=secret) + require(status, data, {200}, 'Allowed-prefix access') + print('Verified allowed: archive prefix listing', flush=True) + else: + print('Positive archive access checks pending bucket provisioning', flush=True) + target.mkdir(mode=0o700, parents=True, exist_ok=True) + identity = pwd.getpwnam('pmp') + os.chmod(target, 0o700) + os.chown(target, identity.pw_uid, identity.pw_gid) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, 'w') as output: + output.write('AWS_ACCESS_KEY_ID=' + shlex.quote(username) + '\nAWS_SECRET_ACCESS_KEY=' + shlex.quote(secret) + '\n') + os.chown(path, identity.pw_uid, identity.pw_gid) + print('Identity created; credentials saved owner-only for secure transfer. No secret values displayed.', flush=True) + except BaseException: + if created: + status, data = admin.request('PUT', '/rustfs/admin/v3/set-user-status', + {'accessKey': username, 'status': 'disabled'}) + print('Identity disabled after incomplete provisioning: ' + str(status in (200, 204)), flush=True) + raise + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('policy_file', type=Path) + args = parser.parse_args() + create(args.policy_file) diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh new file mode 100755 index 0000000..79aa8f3 --- /dev/null +++ b/scripts/test-integration.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +# Only new, disposable containers; no production connection settings are consumed. +export COMPOSE_PROJECT_NAME="btcdata-archive-test-$$" +compose=(docker compose -f tests/integration/compose.yaml) +trap '"${compose[@]}" down -v' EXIT +"${compose[@]}" up -d +export TEST_PGHOST=127.0.0.1 TEST_PGPORT=15432 TEST_PGUSER=postgres TEST_PGPASSWORD=disposable-test +export TEST_S3_ENDPOINT_URL=http://127.0.0.1:19000 +export TEST_S3_ACCESS_KEY=disposable-test TEST_S3_SECRET_KEY=disposable-test-secret +python - <<'PY' +import os, time +import boto3, psycopg +from botocore.config import Config +for attempt in range(90): + try: + with psycopg.connect(host=os.environ['TEST_PGHOST'], port=os.environ['TEST_PGPORT'], user='postgres', password='disposable-test', dbname='postgres', connect_timeout=2): + pass + boto3.client('s3', endpoint_url=os.environ['TEST_S3_ENDPOINT_URL'], aws_access_key_id='disposable-test', aws_secret_access_key='disposable-test-secret', region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=2, retries={'max_attempts': 0}, s3={'addressing_style': 'path'})).list_buckets() + break + except Exception: + time.sleep(2) +else: + raise SystemExit('disposable PostgreSQL/RustFS did not become ready') +PY +python -m pytest -q tests/integration diff --git a/sql/indexes.sql b/sql/indexes.sql new file mode 100644 index 0000000..4155ce9 --- /dev/null +++ b/sql/indexes.sql @@ -0,0 +1,9 @@ +-- Inspect pg_indexes first; equivalent indexes with other names should be reused. +-- Run outside a transaction: psql -v ON_ERROR_STOP=1 -f sql/indexes.sql +-- IF NOT EXISTS only checks names, not index validity or equivalence. Inspect invalid +-- indexes after an interrupted build before retrying. +CREATE INDEX CONCURRENTLY IF NOT EXISTS binance_price_recorded_at_id + ON public.binance_price (recorded_at, id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS strike_price_recorded_at_id + ON public.strike_price (recorded_at, id); +-- kraken_minute already has a ts primary-key index. diff --git a/sql/s3-runtime-policy.json b/sql/s3-runtime-policy.json new file mode 100644 index 0000000..b3d8cdf --- /dev/null +++ b/sql/s3-runtime-policy.json @@ -0,0 +1,34 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketVersioning"], + "Resource": "arn:aws:s3:::btcdata-archive" + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"], + "Resource": "arn:aws:s3:::btcdata-archive", + "Condition": {"StringLike": {"s3:prefix": ["btcprod/weeks/", "btcprod/weeks/*"]}} + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:GetObjectVersion", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"], + "Resource": "arn:aws:s3:::btcdata-archive/btcprod/weeks/*" + }, + { + "Effect": "Deny", + "Action": ["admin:*"] + }, + { + "Effect": "Deny", + "Action": ["sts:*"] + }, + { + "Effect": "Deny", + "Action": ["s3:DeleteObject", "s3:DeleteObjectVersion", "s3:DeleteBucket", "s3:PutBucketVersioning"], + "Resource": ["arn:aws:s3:::*", "arn:aws:s3:::*/*"] + } + ] +} diff --git a/sql/setup.sql b/sql/setup.sql new file mode 100644 index 0000000..d8bc7a4 --- /dev/null +++ b/sql/setup.sql @@ -0,0 +1,30 @@ +-- Operator-run only: psql -v ON_ERROR_STOP=1 -v runtime_role=btcdata_archiver -f sql/setup.sql +-- Create the login separately with a password supplied through secure operator tooling. +BEGIN; +CREATE SCHEMA IF NOT EXISTS btcdata_archive; +CREATE TABLE IF NOT EXISTS btcdata_archive.weeks ( + week_start timestamptz PRIMARY KEY, + week_end timestamptz NOT NULL, + format_version integer NOT NULL CHECK (format_version = 1), + status text NOT NULL CHECK (status IN ('exporting','verified','pruning','complete','failed')), + object_key text NOT NULL, + object_version_id text, + file_sha256 text CHECK (file_sha256 ~ '^[0-9a-f]{64}$'), + row_counts jsonb, + deletion_counts jsonb, + attempt_time timestamptz NOT NULL, + completion_time timestamptz, + last_error text, + CHECK (week_end - week_start = interval '168 hours'), + CHECK (extract(isodow FROM week_start AT TIME ZONE 'UTC') = 1), + CHECK ((week_start AT TIME ZONE 'UTC')::time = time '00:00:00'), + CHECK (status NOT IN ('verified','pruning','complete') OR + (object_version_id IS NOT NULL AND object_version_id <> 'null' AND file_sha256 IS NOT NULL)) +); +REVOKE ALL ON SCHEMA btcdata_archive FROM PUBLIC; +REVOKE ALL ON btcdata_archive.weeks FROM PUBLIC; +GRANT CONNECT, TEMPORARY ON DATABASE :"DBNAME" TO :"runtime_role"; +GRANT USAGE ON SCHEMA public, btcdata_archive TO :"runtime_role"; +GRANT SELECT, DELETE ON public.binance_price, public.strike_price, public.kraken_minute TO :"runtime_role"; +GRANT SELECT, INSERT, UPDATE ON btcdata_archive.weeks TO :"runtime_role"; +COMMIT; diff --git a/src/btcdata_archive/__init__.py b/src/btcdata_archive/__init__.py new file mode 100644 index 0000000..ad5b3fe --- /dev/null +++ b/src/btcdata_archive/__init__.py @@ -0,0 +1 @@ +"""Standalone verified weekly PostgreSQL archiver.""" diff --git a/src/btcdata_archive/__main__.py b/src/btcdata_archive/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/src/btcdata_archive/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/src/btcdata_archive/archive.py b/src/btcdata_archive/archive.py new file mode 100644 index 0000000..3e5f7af --- /dev/null +++ b/src/btcdata_archive/archive.py @@ -0,0 +1,91 @@ +import json +import shutil +import tempfile +import time +from pathlib import Path +from datetime import datetime, timezone +from itertools import islice + +from . import sqlite_archive +from .errors import SafetyError +from .weeks import WEEK, weeks, timestamp + + +def log(event, **fields): + print(json.dumps({"event": event, **fields}, default=lambda value: timestamp(value) if isinstance(value, datetime) else str(value), sort_keys=True), flush=True) + + +def require_space(directory, needed): + if shutil.disk_usage(directory).free < needed + 64 * 1024 * 1024: + raise SafetyError("insufficient working storage") + + +def run(pg, store, config, cutoff, mode, max_weeks=None): + pg.lock() + pg.check_schema() + store.require_versioning() + config.work_dir.mkdir(parents=True, exist_ok=True) + candidates = weeks(pg.first_week(cutoff), cutoff) + if max_weeks is not None: + candidates = islice(candidates, max_weeks) + log("invocation", cutoff=cutoff, mode=mode) + for start in candidates: + key = config.prefix + start.date().isoformat() + ".sqlite" + begun = time.monotonic() + try: + pg.alive() + counts = pg.counts(start, start + WEEK) + log("started", week_start=start, week_end=start + WEEK, cutoff=cutoff, counts=counts) + pg.ledger(start, "exporting", key) + existing = store.head(key) + if existing is None and pg.archived_version(start): + raise SafetyError("previously verified archive missing: operator recovery required") + old_size = existing["ContentLength"] if existing else 0 + # Conservative estimate for text encoding, SQLite overhead, journal and verification copy. + require_space(config.work_dir, old_size * 3 + int(pg.estimated_bytes(start)) * 8) + with tempfile.TemporaryDirectory(prefix="btcdata-week-", dir=config.work_dir) as directory: + local = Path(directory) / "archive.sqlite" + downloaded = Path(directory) / "verified.sqlite" + if existing: + store.download(key, existing["VersionId"], local) + sqlite_archive.validate(local, start, config.pg["dbname"], config.batch_rows) + advertised = existing.get("Metadata", {}).get("sha256") + if not advertised or sqlite_archive.file_hash(local) != advertised: + raise SafetyError("existing object file checksum mismatch") + else: + sqlite_archive.create(local) + pg.export(local, start) + metadata = sqlite_archive.finalize(local, start, config.pg["dbname"], datetime.now(timezone.utc), config.batch_rows) + checksum = sqlite_archive.file_hash(local) + require_space(config.work_dir, local.stat().st_size) + pg.alive() + version = store.upload(key, local, checksum) + store.download(key, version, downloaded) + if sqlite_archive.file_hash(downloaded) != checksum: + raise SafetyError("downloaded file SHA-256 mismatch") + verified = sqlite_archive.validate(downloaded, start, config.pg["dbname"], config.batch_rows) + if verified != metadata: + raise SafetyError("downloaded archive metadata mismatch") + archive_counts = {t: s["count"] for t, s in verified["tables"].items()} + pg.ledger(start, "verified", key, version, checksum, archive_counts) + log("verified", week_start=start, counts=archive_counts, version_id=version, sha256=checksum) + if mode == "archive-and-prune": + pg.ledger(start, "pruning", key) + + def progress(deleted): + pg.ledger(start, "pruning", key, deletions=deleted) + log("deletion_progress", week_start=start, counts=archive_counts, deleted=deleted) + + deleted = pg.prune(downloaded, start, cutoff, progress) + log("pruned", week_start=start, counts=archive_counts, deleted=deleted) + pg.ledger(start, "complete", key, deletions=deleted) + log("complete", week_start=start, counts=archive_counts, duration_seconds=round(time.monotonic() - begun, 3)) + except BaseException as exc: + # Do not serialize database/SDK exception messages: they may contain rows or secrets. + reason = str(exc) if isinstance(exc, SafetyError) else type(exc).__name__ + try: + pg.ledger(start, "failed", key, error=reason) + except Exception: + pass + log("failed", week_start=start, error=reason) + raise diff --git a/src/btcdata_archive/cli.py b/src/btcdata_archive/cli.py new file mode 100644 index 0000000..68939ea --- /dev/null +++ b/src/btcdata_archive/cli.py @@ -0,0 +1,65 @@ +import argparse +import signal +from datetime import datetime, timezone + +from .archive import log, run +from .config import Config +from .errors import SafetyError +from .object_store import ObjectStore +from .postgres import Postgres +from .weeks import boundaries, weeks, WEEK + + +def positive(value): + number = int(value) + if number <= 0: + raise argparse.ArgumentTypeError("must be a positive integer") + return number + + +def parser(): + result = argparse.ArgumentParser(description="Verified weekly btcprod archives; no implicit pruning mode") + commands = result.add_subparsers(dest="command", required=True) + commands.add_parser("inventory") + commands.add_parser("status") + command = commands.add_parser("run") + command.add_argument("--mode", required=True, choices=("verify-only", "archive-and-prune")) + command.add_argument("--max-weeks", type=positive) + return result + + +def main(argv=None): + args = parser().parse_args(argv) + # Capture once, before connecting; a catch-up run never slides its cutoff. + anchor, cutoff = boundaries(datetime.now(timezone.utc)) + pg = None + + def terminate(signum, frame): + raise KeyboardInterrupt() + + signal.signal(signal.SIGTERM, terminate) + try: + config = Config.from_env(need_s3=args.command == "run") + pg = Postgres(config) + if args.command == "run": + run(pg, ObjectStore(config), config, cutoff, args.mode, args.max_weeks) + elif args.command == "inventory": + pg.check_schema() + first = pg.first_week(cutoff) + log("inventory", anchor=anchor, cutoff=cutoff, oldest_eligible_week=next(weeks(first, cutoff), None)) + for start in weeks(first, cutoff): + log("candidate", week_start=start, counts=pg.counts(start, start + WEEK)) + else: + pg.check_schema() + log("status", anchor=anchor, cutoff=cutoff, **pg.status(cutoff)) + return 0 + except Exception as exc: + # Connection errors, SQL errors, SDK responses can contain credentials or row data. + log("error", error=str(exc) if isinstance(exc, SafetyError) else type(exc).__name__) + return 1 + except KeyboardInterrupt: + log("interrupted") + return 130 + finally: + if pg is not None: + pg.close() diff --git a/src/btcdata_archive/config.py b/src/btcdata_archive/config.py new file mode 100644 index 0000000..eb50e71 --- /dev/null +++ b/src/btcdata_archive/config.py @@ -0,0 +1,75 @@ +import os +import re +import shlex +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + +from .errors import SafetyError + + +def load_env_file(path: Path): + """Load literal KEY=value entries without shell execution or interpolation. + + Existing environment variables take precedence. Never include values in errors. + """ + if not path.exists(): + return + for number, line in enumerate(path.read_text().splitlines(), 1): + line = line.strip() + if not line or line.startswith('#'): + continue + match = re.fullmatch(r'(?:export\s+)?([A-Za-z_][A-Za-z_0-9]*)\s*=\s*(.*)', line) + if not match: + raise SafetyError(f"invalid environment file entry at line {number}") + try: + parts = shlex.split(match.group(2), comments=True, posix=True) + except ValueError: + raise SafetyError(f"invalid environment file quoting at line {number}") from None + if len(parts) > 1: + raise SafetyError(f"quote environment file values containing spaces at line {number}") + os.environ.setdefault(match.group(1), parts[0] if parts else '') + + +@dataclass(frozen=True, repr=False) +class Config: + pg: dict + endpoint: str | None + access_key: str | None + secret_key: str | None + region: str + bucket: str + prefix: str + work_dir: Path + batch_rows: int + + @classmethod + def from_env(cls, need_s3: bool = True, need_pg: bool = True): + if env_file := os.environ.get('ARCHIVE_ENV_FILE'): + load_env_file(Path(env_file)) + required = ["PGHOST", "PGUSER", "PGPASSWORD"] if need_pg else [] + if need_s3: + required += ["S3_ENDPOINT_URL", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"] + missing = [name for name in required if not os.environ.get(name)] + if missing: + raise SafetyError("missing configuration: " + ", ".join(missing)) + endpoint = os.environ.get("S3_ENDPOINT_URL") + if endpoint: + url = urlsplit(endpoint) + if url.scheme not in ("http", "https") or not url.hostname or url.username or url.password or url.query or url.fragment: + raise SafetyError("S3_ENDPOINT_URL must be an explicit HTTP(S) endpoint without credentials") + batch = int(os.environ.get("ARCHIVE_BATCH_ROWS", "5000")) + if batch <= 0: + raise SafetyError("ARCHIVE_BATCH_ROWS must be positive") + prefix = os.environ.get("ARCHIVE_PREFIX", "btcprod/weeks/") + if not prefix.endswith("/") or prefix.startswith("/"): + raise SafetyError("ARCHIVE_PREFIX must be relative and end in /") + return cls( + pg={"host": os.environ["PGHOST"], "port": int(os.environ.get("PGPORT", "5432")), + "dbname": os.environ.get("PGDATABASE", "btcprod"), "user": os.environ["PGUSER"], + "password": os.environ["PGPASSWORD"], "sslmode": os.environ.get("PGSSLMODE", "prefer"), + "connect_timeout": 15, "application_name": "btcdata-archive"} if need_pg else {}, + endpoint=endpoint, access_key=os.environ.get("AWS_ACCESS_KEY_ID"), + secret_key=os.environ.get("AWS_SECRET_ACCESS_KEY"), region=os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + bucket=os.environ.get("ARCHIVE_BUCKET", "btcdata-archive"), prefix=prefix, + work_dir=Path(os.environ.get("ARCHIVE_WORK_DIR", "/work")), batch_rows=batch) diff --git a/src/btcdata_archive/errors.py b/src/btcdata_archive/errors.py new file mode 100644 index 0000000..2f65e14 --- /dev/null +++ b/src/btcdata_archive/errors.py @@ -0,0 +1,2 @@ +class SafetyError(ValueError, RuntimeError): + """Operator-safe diagnostic containing no credentials or row payloads.""" diff --git a/src/btcdata_archive/object_store.py b/src/btcdata_archive/object_store.py new file mode 100644 index 0000000..c302570 --- /dev/null +++ b/src/btcdata_archive/object_store.py @@ -0,0 +1,72 @@ +import boto3 +from botocore.config import Config as S3Config +from botocore.exceptions import ClientError + +from .errors import SafetyError + + +class ObjectStore: + def __init__(self, config): + if not config.endpoint: + raise SafetyError("explicit S3 endpoint required") + self.bucket = config.bucket + self.client = boto3.client("s3", endpoint_url=config.endpoint, + aws_access_key_id=config.access_key, aws_secret_access_key=config.secret_key, + region_name=config.region, + config=S3Config(s3={"addressing_style": "path"}, connect_timeout=15, + read_timeout=120, retries={"max_attempts": 3, "mode": "standard"})) + + def require_versioning(self): + if self.client.get_bucket_versioning(Bucket=self.bucket).get("Status") != "Enabled": + raise SafetyError("bucket versioning must be enabled by an administrator") + + def head(self, key): + try: + info = self.client.head_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + if exc.response["Error"]["Code"] in ("404", "NoSuchKey", "NotFound"): + return None + raise + if info.get("VersionId") in (None, "", "null"): + raise SafetyError("existing object lacks a durable version ID") + return info + + def download(self, key, version, path): + if version in (None, "", "null"): + raise SafetyError("exact object version required") + response = self.client.get_object(Bucket=self.bucket, Key=key, VersionId=version) + if response.get("VersionId") != version: + response["Body"].close() + raise SafetyError("object store returned wrong version") + try: + with path.open("wb") as out: + while chunk := response["Body"].read(1024 * 1024): + out.write(chunk) + finally: + response["Body"].close() + + def upload(self, key, path, checksum): + self.require_versioning() + args = {"Bucket": self.bucket, "Key": key} + response = self.client.create_multipart_upload(**args, ContentType="application/vnd.sqlite3", Metadata={"sha256": checksum}) + upload_id = response["UploadId"] + try: + parts = [] + # Stay below S3's 10,000-part limit, with bounded memory even for large weeks. + part_size = max(8 * 1024 * 1024, (path.stat().st_size + 9998) // 9999) + with path.open("rb") as source: + while chunk := source.read(part_size): + number = len(parts) + 1 + part = self.client.upload_part(**args, UploadId=upload_id, PartNumber=number, Body=chunk) + parts.append({"PartNumber": number, "ETag": part["ETag"]}) + response = self.client.complete_multipart_upload(**args, UploadId=upload_id, MultipartUpload={"Parts": parts}) + except BaseException: + try: + self.client.abort_multipart_upload(**args, UploadId=upload_id) + except Exception: + pass + raise + version = response.get("VersionId") + if version in (None, "", "null"): + raise SafetyError("upload did not return a durable object version ID") + return version diff --git a/src/btcdata_archive/postgres.py b/src/btcdata_archive/postgres.py new file mode 100644 index 0000000..596cded --- /dev/null +++ b/src/btcdata_archive/postgres.py @@ -0,0 +1,175 @@ +import uuid +from datetime import datetime, timezone + +import psycopg +from psycopg import sql +from psycopg.types.json import Jsonb + +from .schema import TABLES, FORMAT_VERSION, Digest, encode_row, decode_row +from . import sqlite_archive +from .errors import SafetyError +from .weeks import WEEK + +LOCK_ID = 728194107 + + +class Postgres: + def __init__(self, config): + self.config = config + self.conn = psycopg.connect(**config.pg, autocommit=True) + self.conn.execute("SET TIME ZONE 'UTC'") + self.locked = False + + def close(self): + self.conn.close() + + def lock(self): + if self.locked: + self.alive() + return + if not self.conn.execute("SELECT pg_try_advisory_lock(%s)", (LOCK_ID,)).fetchone()[0]: + raise SafetyError("another archiver holds the advisory lock") + self.locked = True + + def alive(self): + if not self.locked: + raise SafetyError("archiver lock required") + self.conn.execute("SELECT 1").fetchone() # never reconnect this session + + def check_schema(self): + for table in TABLES: + relation = self.conn.execute("SELECT relkind, relrowsecurity FROM pg_class WHERE oid=to_regclass(%s)", ("public." + table.name,)).fetchone() + if relation != ('r', False): + raise SafetyError(f"migration required: public.{table.name} must be an ordinary table without RLS") + if self.conn.execute("SELECT count(*) FROM pg_trigger WHERE tgrelid=to_regclass(%s) AND NOT tgisinternal AND tgenabled <> 'D' AND (tgtype & 8) <> 0", ("public." + table.name,)).fetchone()[0]: + raise SafetyError(f"migration required: public.{table.name} has unsupported DELETE triggers") + if self.conn.execute("SELECT count(*) FROM pg_constraint WHERE confrelid=to_regclass(%s) AND contype='f' AND confdeltype IN ('c','n','d')", ("public." + table.name,)).fetchone()[0]: + raise SafetyError(f"migration required: public.{table.name} has mutating inbound DELETE foreign keys") + actual = self.conn.execute(""" + SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), NOT a.attnotnull + FROM pg_catalog.pg_attribute a + WHERE a.attrelid = to_regclass(%s) AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + """, ("public." + table.name,)).fetchall() + if sorted(actual) != sorted(table.columns): + raise SafetyError(f"migration required: unsupported public.{table.name} columns/types/nullability") + pk = self.conn.execute(""" + SELECT array_agg(a.attname ORDER BY k.ordinality) + FROM pg_constraint c CROSS JOIN LATERAL unnest(c.conkey) WITH ORDINALITY k(attnum, ordinality) + JOIN pg_attribute a ON a.attrelid=c.conrelid AND a.attnum=k.attnum + WHERE c.conrelid=to_regclass(%s) AND c.contype='p' + GROUP BY c.oid + """, ("public." + table.name,)).fetchall() + if pk != [([table.pk],)]: + raise SafetyError(f"migration required: unsupported public.{table.name} primary key") + + def first_week(self, cutoff): + values = [] + for table in TABLES: + _, end = table.bounds(cutoff, cutoff) + value = self.conn.execute(f"SELECT min({table.retention}) FROM public.{table.name} WHERE {table.retention} < %s", (end,)).fetchone()[0] + if value is not None: + values.append(datetime.fromtimestamp(value, timezone.utc) if table.retention == "ts" else value) + value = self.conn.execute("SELECT min(week_start) FROM btcdata_archive.weeks WHERE status <> 'complete' AND week_end <= %s", (cutoff,)).fetchone()[0] + if value is not None: + values.append(value) + return min(values) if values else None + + def counts(self, start, end): + return {t.name: self.conn.execute(f"SELECT count(*) FROM public.{t.name} WHERE {t.retention} >= %s AND {t.retention} < %s", t.bounds(start, end)).fetchone()[0] for t in TABLES} + + def archived_version(self, start): + row = self.conn.execute("SELECT object_version_id FROM btcdata_archive.weeks WHERE week_start=%s", (start,)).fetchone() + return row[0] if row else None + + def estimated_bytes(self, start): + return sum(self.conn.execute(f"SELECT coalesce(sum(pg_column_size(t)),0) FROM public.{t.name} t WHERE {t.retention} >= %s AND {t.retention} < %s", t.bounds(start, start + WEEK)).fetchone()[0] for t in TABLES) + + def ledger(self, start, status, key, version=None, checksum=None, counts=None, deletions=None, error=None): + self.alive() + # PostgreSQL checks the proposed INSERT before ON CONFLICT runs. Resolve + # durable verification fields first so progress-only transitions satisfy + # the same constraints as the final updated row; never weaken the checks. + self.conn.execute(""" + WITH previous AS ( + SELECT object_version_id,file_sha256 FROM btcdata_archive.weeks WHERE week_start=%s + ) + INSERT INTO btcdata_archive.weeks + (week_start,week_end,format_version,status,object_key,object_version_id,file_sha256,row_counts,deletion_counts,attempt_time,completion_time,last_error) + VALUES (%s,%s,%s,%s,%s, + coalesce(%s,(SELECT object_version_id FROM previous)), + coalesce(%s,(SELECT file_sha256 FROM previous)), + %s,%s,now(),CASE WHEN %s='complete' THEN now() ELSE NULL END,%s) + ON CONFLICT (week_start) DO UPDATE SET status=excluded.status,object_key=excluded.object_key, + object_version_id=coalesce(excluded.object_version_id,btcdata_archive.weeks.object_version_id), + file_sha256=coalesce(excluded.file_sha256,btcdata_archive.weeks.file_sha256), + row_counts=coalesce(excluded.row_counts,btcdata_archive.weeks.row_counts), + deletion_counts=coalesce(excluded.deletion_counts,btcdata_archive.weeks.deletion_counts), + attempt_time=excluded.attempt_time,completion_time=excluded.completion_time,last_error=excluded.last_error + """, (start, start, start + WEEK, FORMAT_VERSION, status, key, version, checksum, + Jsonb(counts) if counts is not None else None, Jsonb(deletions) if deletions is not None else None, status, error)) + + def export(self, path, start): + db = sqlite_archive.connect(path) + try: + with self.conn.transaction(): + self.conn.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY") + for table in TABLES: + db.execute("CREATE TEMP TABLE source_keys (pk INTEGER PRIMARY KEY)") + source = Digest() + projection = ",".join(name + "::text" if typ == "jsonb" else name for name, typ, _ in table.columns) + with self.conn.cursor(name="archive_" + uuid.uuid4().hex) as cur: + cur.execute(f"SELECT {projection} FROM public.{table.name} WHERE {table.retention} >= %s AND {table.retention} < %s ORDER BY {table.pk}", table.bounds(start, start + WEEK)) + while batch := cur.fetchmany(self.config.batch_rows): + encoded = [encode_row(table, row) for row in batch] + sqlite_archive.upsert(db, table, encoded) + db.executemany("INSERT INTO source_keys VALUES (?)", [(r[0],) for r in encoded]) + for row in encoded: + source.add(row) + db.commit() + fresh = Digest() + cur = db.execute(f"SELECT {','.join('t.' + n for n in table.names)} FROM {table.name} t JOIN source_keys k ON t.{table.pk}=k.pk ORDER BY t.{table.pk}") + while batch := cur.fetchmany(self.config.batch_rows): + for row in batch: + fresh.add(row) + if source.result() != fresh.result(): + raise SafetyError("source export checksum mismatch") + db.execute("DROP TABLE source_keys") + db.commit() + finally: + db.close() + + def prune(self, path, start, cutoff, progress): + deleted = {t.name: 0 for t in TABLES} + db = sqlite_archive.connect(path, readonly=True) + try: + for table in TABLES: + for batch in sqlite_archive.rows(db, table, self.config.batch_rows): + self.alive() + with self.conn.transaction(): + definitions = sql.SQL(',').join(sql.SQL('{} {}').format(sql.Identifier(n), sql.SQL(typ)) for n, typ, _ in table.columns) + self.conn.execute(sql.SQL("CREATE TEMP TABLE archive_stage ({}) ON COMMIT DROP").format(definitions)) + with self.conn.cursor().copy("COPY archive_stage FROM STDIN") as copy: + for row in batch: + copy.write_row(decode_row(table, row)) + comparisons = " AND ".join(f"s.{n} IS NOT DISTINCT FROM t.{n}" for n in table.names) + begin, end = table.bounds(start, start + WEEK) + _, limit = table.bounds(cutoff, cutoff) + cur = self.conn.execute(f"DELETE FROM public.{table.name} t USING archive_stage s WHERE t.{table.pk}=s.{table.pk} AND {comparisons} AND t.{table.retention} >= %s AND t.{table.retention} < %s AND t.{table.retention} < %s", (begin, end, limit)) + deleted[table.name] += cur.rowcount + progress(deleted.copy()) + finally: + db.close() + remaining = self.counts(start, start + WEEK) + if any(remaining.values()): + raise SafetyError("reconciliation required: changed or late rows remain") + return deleted + + def status(self, cutoff): + complete = self.conn.execute("SELECT max(week_start) FROM btcdata_archive.weeks WHERE status='complete'").fetchone()[0] + incomplete = self.conn.execute("SELECT week_start,status,attempt_time,last_error FROM btcdata_archive.weeks WHERE status<>'complete' ORDER BY week_start").fetchall() + expired = {} + for t in TABLES: + _, end = t.bounds(cutoff, cutoff) + expired[t.name] = self.conn.execute(f"SELECT count(*) FROM public.{t.name} WHERE {t.retention}<%s", (end,)).fetchone()[0] + return {"last_completed_week": complete, "incomplete": incomplete, "expired_counts": expired} diff --git a/src/btcdata_archive/report.py b/src/btcdata_archive/report.py new file mode 100644 index 0000000..fbe0c09 --- /dev/null +++ b/src/btcdata_archive/report.py @@ -0,0 +1,94 @@ +"""Read-only S3/SQLite file inventory. No PostgreSQL imports or connections.""" +import argparse +from contextlib import closing +from pathlib import Path +import shutil +import sqlite3 +import sys +import tempfile + +from .config import Config +from .errors import SafetyError +from .object_store import ObjectStore +from .schema import TABLES +from .sqlite_archive import file_hash + +TABLE_NAMES = tuple(table.name for table in TABLES) + + +def sqlite_counts(path: Path): + # Do not trust metadata row counts: count the actual downloaded tables. + with closing(sqlite3.connect(path.resolve().as_uri() + '?mode=ro', uri=True)) as db: + db.execute('PRAGMA query_only=ON') + if db.execute('PRAGMA integrity_check').fetchall() != [('ok',)]: + raise SafetyError('SQLite integrity check failed') + tables = {row[0] for row in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + if not set(TABLE_NAMES) <= tables: + raise SafetyError('SQLite archive is missing a required table') + return {name: db.execute(f'SELECT COUNT(*) FROM {name}').fetchone()[0] for name in TABLE_NAMES} + + +def files(store, config): + """Yield current object sizes and real table counts using at most one local file.""" + pages = store.client.get_paginator('list_objects_v2').paginate(Bucket=config.bucket, Prefix=config.prefix) + for page in pages: + for item in page.get('Contents', []): + key = item['Key'] + if not key.lower().endswith('.sqlite'): + yield key, item['Size'], None + continue + # Pin the version before downloading: a concurrent archiver may replace it. + head = store.head(key) + if head is None: + raise SafetyError('Listed archive is no longer available: ' + key) + size = head['ContentLength'] + config.work_dir.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(config.work_dir).free < size + 64 * 1024 * 1024: + raise SafetyError('Insufficient working storage to inspect: ' + key) + with tempfile.TemporaryDirectory(prefix='btcdata-report-', dir=config.work_dir) as directory: + path = Path(directory) / 'archive.sqlite' + store.download(key, head['VersionId'], path) + if path.stat().st_size != size: + raise SafetyError('Downloaded archive size mismatch: ' + key) + checksum = head.get('Metadata', {}).get('sha256') + if checksum and file_hash(path) != checksum: + raise SafetyError('Downloaded archive SHA-256 mismatch: ' + key) + counts = sqlite_counts(path) + # Clean up before yielding, so the next week cannot accumulate local files. + yield key, size, counts + + +def show(store, config): + print(f'Current files in s3://{config.bucket}/{config.prefix} (latest versions only)', flush=True) + print(f'{"FILE":<44} {"MiB":>10} ' + ' '.join(f'{name:>15}' for name in TABLE_NAMES), flush=True) + total_bytes = file_count = 0 + totals = {name: 0 for name in TABLE_NAMES} + for key, size, counts in files(store, config): + values = ' '.join(f'{counts[name]:>15,}' if counts is not None else f'{"-":>15}' for name in TABLE_NAMES) + print(f'{key:<44} {size / 1048576:>10.2f} {values}', flush=True) + total_bytes += size + file_count += 1 + if counts is not None: + for name in TABLE_NAMES: + totals[name] += counts[name] + print(f'{"TOTAL":<44} {total_bytes / 1048576:>10.2f} ' + ' '.join(f'{totals[name]:>15,}' for name in TABLE_NAMES), flush=True) + print(f'{file_count} files, {total_bytes:,} bytes ({total_bytes / 1073741824:.2f} GiB). Non-SQLite files show "-" counts.', flush=True) + + +def main(argv=None): + argparse.ArgumentParser(description='List archive files, sizes and actual SQLite table counts; never query PostgreSQL.').parse_args(argv) + try: + config = Config.from_env(need_pg=False) + show(ObjectStore(config), config) + return 0 + except KeyboardInterrupt: + print('Report interrupted.', file=sys.stderr) + return 130 + except Exception as error: + message = str(error) if isinstance(error, SafetyError) else type(error).__name__ + print('Archive report failed: ' + message, file=sys.stderr) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/btcdata_archive/schema.py b/src/btcdata_archive/schema.py new file mode 100644 index 0000000..1afe881 --- /dev/null +++ b/src/btcdata_archive/schema.py @@ -0,0 +1,95 @@ +from dataclasses import dataclass +from datetime import datetime +from decimal import Decimal +import hashlib +import json + +from .weeks import timestamp + +FORMAT_VERSION = 1 + + +@dataclass(frozen=True) +class Table: + name: str + columns: tuple[tuple[str, str, bool], ...] # PostgreSQL type, nullable + retention: str + + @property + def pk(self): + return self.columns[0][0] + + @property + def names(self): + return tuple(c[0] for c in self.columns) + + def bounds(self, start, end): + if self.retention == "ts": + return int(start.timestamp()), int(end.timestamp()) + return start, end + + +TABLES = ( + Table("binance_price", (("id", "bigint", False), ("price", "numeric(18,8)", False), + ("recorded_at", "timestamp with time zone", False)), "recorded_at"), + Table("strike_price", (("id", "bigint", False), ("rates", "jsonb", False), + ("recorded_at", "timestamp with time zone", False), ("quote_sats", "bigint", True)), "recorded_at"), + Table("kraken_minute", (("ts", "bigint", False), *tuple((c, "numeric(18,8)", False) for c in ("open", "high", "low", "close", "vwap")), + ("volume", "numeric(24,8)", False), ("trade_count", "integer", False), + ("created_at", "timestamp with time zone", False)), "ts"), +) + + +def exact_json(value): + """Sorted JSON without any binary floating-point conversion.""" + if isinstance(value, dict): + return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + exact_json(value[k]) for k in sorted(value)) + "}" + if isinstance(value, list): + return "[" + ",".join(exact_json(v) for v in value) + "]" + if isinstance(value, Decimal): + if not value.is_finite(): + raise ValueError("nonfinite JSON number") + return format(value, "f") + return json.dumps(value, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + + +def canonical_json(text): + return exact_json(json.loads(text, parse_float=Decimal, parse_int=Decimal)) + + +def encode_row(table, row): + result = [] + for value, (_, typ, nullable) in zip(row, table.columns, strict=True): + if value is None: + if not nullable: + raise ValueError("unexpected SQL NULL") + result.append(None) + elif typ == "jsonb": + result.append(canonical_json(value)) + elif typ.startswith("numeric"): + result.append(str(value)) + elif typ == "timestamp with time zone": + result.append(timestamp(value)) + else: + result.append(int(value)) + return tuple(result) + + +def decode_row(table, row): + return tuple(datetime.fromisoformat(v) if typ == "timestamp with time zone" else + Decimal(v) if typ.startswith("numeric") else v + for v, (_, typ, _) in zip(row, table.columns, strict=True)) + + +class Digest: + """SHA-256 of UTF-8 compact JSON row arrays, each followed by LF, in PK order.""" + def __init__(self): + self.count = 0 + self.hash = hashlib.sha256() + + def add(self, row): + self.hash.update((json.dumps(row, ensure_ascii=False, allow_nan=False, separators=(",", ":")) + "\n").encode()) + self.count += 1 + + def result(self): + return {"count": self.count, "sha256": self.hash.hexdigest()} diff --git a/src/btcdata_archive/sqlite_archive.py b/src/btcdata_archive/sqlite_archive.py new file mode 100644 index 0000000..d694b21 --- /dev/null +++ b/src/btcdata_archive/sqlite_archive.py @@ -0,0 +1,138 @@ +import hashlib +import json +import sqlite3 +from datetime import datetime +from decimal import Decimal + +from .schema import TABLES, FORMAT_VERSION, Digest, canonical_json +from .weeks import WEEK, timestamp +from .errors import SafetyError + + +def connect(path, readonly=False): + if readonly: + return sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True) + db = sqlite3.connect(path) + db.execute("PRAGMA journal_mode=DELETE") + return db + + +def sqlite_columns(table): + return [(name, "INTEGER" if typ in ("bigint", "integer") else "TEXT", int(not nullable and name != table.pk), int(name == table.pk)) + for name, typ, nullable in table.columns] + + +def create(path): + with connect(path) as db: + for table in TABLES: + cols = [f'{name} {typ}' + (" PRIMARY KEY" if pk else " NOT NULL" if notnull else "") + for name, typ, notnull, pk in sqlite_columns(table)] + db.execute(f"CREATE TABLE {table.name} ({', '.join(cols)})") + if table.retention == "recorded_at": + db.execute(f"CREATE INDEX {table.name}_recorded_at_id ON {table.name}(recorded_at, id)") + db.execute("CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + db.close() + + +def upsert(db, table, rows): + placeholders = ",".join("?" for _ in table.columns) + db.executemany(f"INSERT OR REPLACE INTO {table.name} VALUES ({placeholders})", rows) + + +def rows(db, table, batch_rows): + cur = db.execute(f"SELECT {', '.join(table.names)} FROM {table.name} ORDER BY {table.pk}") + while batch := cur.fetchmany(batch_rows): + yield batch + + +def stats(db, start, batch_rows): + result = {} + for table in TABLES: + digest = Digest() + for batch in rows(db, table, batch_rows): + for row in batch: + for value, (_, typ, nullable) in zip(row, table.columns, strict=True): + if value is None: + if not nullable: + raise ValueError("invalid archive NULL") + elif typ in ("bigint", "integer"): + if type(value) is not int or not -(2**63) <= value < 2**63: + raise ValueError("invalid archive integer") + if typ == "integer" and not -(2**31) <= value < 2**31: + raise ValueError("invalid archive integer") + elif not isinstance(value, str): + raise ValueError("invalid archive text") + elif typ.startswith("numeric"): + number = Decimal(value) + precision, scale = map(int, typ[8:-1].split(',')) + if (not number.is_finite() or number.as_tuple().exponent != -scale + or abs(number) >= Decimal(10) ** (precision - scale)): + raise SafetyError("invalid archive decimal precision/scale") + elif typ == "jsonb": + if canonical_json(value) != value: + raise ValueError("noncanonical archive JSON") + elif typ == "timestamp with time zone": + if timestamp(datetime.fromisoformat(value)) != value: + raise ValueError("noncanonical archive timestamp") + value = row[table.names.index(table.retention)] + instant = datetime.fromtimestamp(value, start.tzinfo) if table.retention == "ts" else datetime.fromisoformat(value) + if not start <= instant < start + WEEK: + raise ValueError("archive row outside week") + digest.add(row) + result[table.name] = digest.result() + return result + + +def finalize(path, start, database, created_at, batch_rows): + with connect(path) as db: + metadata = {"format_version": FORMAT_VERSION, "source_database": database, "source_schema": "public", + "source_columns": {t.name: list(t.columns) for t in TABLES}, + "week_start": timestamp(start), "week_end": timestamp(start + WEEK), + "created_at": timestamp(created_at), "tables": stats(db, start, batch_rows)} + db.execute("DELETE FROM metadata") + db.executemany("INSERT INTO metadata VALUES (?, ?)", [(k, json.dumps(v, sort_keys=True)) for k, v in metadata.items()]) + db.close() + return validate(path, start, database, batch_rows) + + +def validate(path, start, database, batch_rows): + db = connect(path, readonly=True) + try: + if db.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + raise SafetyError("SQLite integrity check failed") + if db.execute("SELECT count(*) FROM sqlite_master WHERE type IN ('trigger','view')").fetchone()[0]: + raise SafetyError("incompatible archive triggers/views") + tables = {r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type='table'")} + if tables != {t.name for t in TABLES} | {"metadata"}: + raise SafetyError("incompatible archive tables") + for table in TABLES: + actual = [(r[1], r[2], r[3], r[5]) for r in db.execute(f"PRAGMA table_info({table.name})")] + if actual != sqlite_columns(table): + raise SafetyError("incompatible archive schema") + if table.retention == "recorded_at": + indexes = [r[1] for r in db.execute(f"PRAGMA index_list({table.name})")] + if not any([r[2] for r in db.execute(f'PRAGMA index_info("{idx}")')] == ["recorded_at", "id"] for idx in indexes): + raise SafetyError("missing archive index") + if [(r[1], r[2], r[3], r[5]) for r in db.execute("PRAGMA table_info(metadata)")] != [("key", "TEXT", 0, 1), ("value", "TEXT", 1, 0)]: + raise SafetyError("incompatible archive metadata schema") + meta = {k: json.loads(v) for k, v in db.execute("SELECT key, value FROM metadata")} + expected = {"format_version": FORMAT_VERSION, "source_database": database, "source_schema": "public", + "source_columns": {t.name: [list(c) for c in t.columns] for t in TABLES}, + "week_start": timestamp(start), "week_end": timestamp(start + WEEK)} + if set(meta) != set(expected) | {"created_at", "tables"} or any(json.dumps(meta.get(k), sort_keys=True) != json.dumps(v, sort_keys=True) for k, v in expected.items()): + raise SafetyError("incompatible archive metadata") + if timestamp(datetime.fromisoformat(meta["created_at"])) != meta["created_at"]: + raise ValueError("invalid archive creation time") + if json.dumps(stats(db, start, batch_rows), sort_keys=True) != json.dumps(meta["tables"], sort_keys=True): + raise SafetyError("archive content checksum mismatch") + return meta + finally: + db.close() + + +def file_hash(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() diff --git a/src/btcdata_archive/weeks.py b/src/btcdata_archive/weeks.py new file mode 100644 index 0000000..f2938ae --- /dev/null +++ b/src/btcdata_archive/weeks.py @@ -0,0 +1,30 @@ +from datetime import datetime, timedelta, timezone + +UTC = timezone.utc +WEEK = timedelta(days=7) + + +def week_start(instant: datetime) -> datetime: + if instant.tzinfo is None: + raise ValueError("timezone-aware timestamp required") + instant = instant.astimezone(UTC) + return instant.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=instant.weekday()) + + +def boundaries(now: datetime) -> tuple[datetime, datetime]: + anchor = week_start(now) + return anchor, anchor - timedelta(days=28) + + +def weeks(first: datetime | None, cutoff: datetime): + if first is not None: + current = week_start(first) + while current + WEEK <= cutoff: + yield current + current += WEEK + + +def timestamp(value: datetime) -> str: + if value.tzinfo is None: + raise ValueError("timezone-aware timestamp required") + return value.astimezone(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") diff --git a/tests/integration/compose.yaml b/tests/integration/compose.yaml new file mode 100644 index 0000000..e151e69 --- /dev/null +++ b/tests/integration/compose.yaml @@ -0,0 +1,25 @@ +# Disposable tests ONLY. Known test credentials are intentionally not production secrets. +services: + postgres: + image: postgres:18.0-bookworm + environment: + POSTGRES_PASSWORD: disposable-test + ports: ["127.0.0.1:15432:5432"] + tmpfs: ["/var/lib/postgresql"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 3s + retries: 30 + rustfs: + image: rustfs/rustfs:1.0.0 + environment: + RUSTFS_ACCESS_KEY: disposable-test + RUSTFS_SECRET_KEY: disposable-test-secret + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_OBS_LOG_DIRECTORY: /logs + command: ["/data"] + ports: ["127.0.0.1:19000:9000"] + tmpfs: + - /data:uid=10001,gid=10001 + - /logs:uid=10001,gid=10001 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..c946e96 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,76 @@ +"""Opt-in integration infrastructure. Always creates NEW disposable resources.""" +import os +from pathlib import Path +from types import SimpleNamespace +import uuid + +import psycopg +from psycopg import sql +import pytest + +from btcdata_archive.postgres import Postgres +from btcdata_archive.object_store import ObjectStore +from btcdata_archive.schema import TABLES + + +@pytest.fixture +def postgres_infra(tmp_path): + if not os.environ.get('TEST_PGHOST'): + pytest.skip('set TEST_PGHOST for disposable PostgreSQL infrastructure') + suffix = uuid.uuid4().hex + database, role, password = 'archive_test_' + suffix, 'archive_test_' + suffix, uuid.uuid4().hex + admin_args = {'host': os.environ['TEST_PGHOST'], 'port': int(os.environ.get('TEST_PGPORT', '5432')), + 'user': os.environ.get('TEST_PGUSER', 'postgres'), 'password': os.environ.get('TEST_PGPASSWORD', 'disposable-test'), + 'dbname': 'postgres'} + config = SimpleNamespace(pg={**admin_args, 'dbname': database, 'user': role, 'password': password}, + endpoint=os.environ.get('TEST_S3_ENDPOINT_URL'), access_key=os.environ.get('TEST_S3_ACCESS_KEY', 'disposable-test'), + secret_key=os.environ.get('TEST_S3_SECRET_KEY', 'disposable-test-secret'), + region='us-east-1', bucket='archive-test-' + suffix, prefix='btcprod/weeks/', work_dir=tmp_path, batch_rows=2) + root = psycopg.connect(**admin_args, autocommit=True) + admin = pg = None + try: + root.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(database))) + root.execute(sql.SQL('CREATE ROLE {} LOGIN PASSWORD {}').format(sql.Identifier(role), sql.Literal(password))) + admin = psycopg.connect(**{**admin_args, 'dbname': database}, autocommit=True) + for table in TABLES: + cols = sql.SQL(',').join(sql.SQL('{} {} {}').format(sql.Identifier(name), sql.SQL(typ), + sql.SQL('PRIMARY KEY' if name == table.pk else '' if nullable else 'NOT NULL')) + for name, typ, nullable in table.columns) + admin.execute(sql.SQL('CREATE TABLE public.{} ({})').format(sql.Identifier(table.name), cols)) + setup = (Path(__file__).resolve().parents[2] / 'sql/setup.sql').read_text() + setup = setup.replace(':"DBNAME"', sql.Identifier(database).as_string()).replace(':"runtime_role"', sql.Identifier(role).as_string()) + admin.execute(setup) + pg = Postgres(config) + yield SimpleNamespace(config=config, pg=pg, admin=admin) + finally: + if pg: + pg.close() + if admin: + admin.close() + root.execute(sql.SQL('DROP DATABASE IF EXISTS {} WITH (FORCE)').format(sql.Identifier(database))) + root.execute(sql.SQL('DROP ROLE IF EXISTS {}').format(sql.Identifier(role))) + root.close() + + +@pytest.fixture +def infra(postgres_infra): + if not os.environ.get('TEST_S3_ENDPOINT_URL'): + pytest.skip('set TEST_S3_ENDPOINT_URL for disposable RustFS infrastructure') + config = postgres_infra.config + store = ObjectStore(config) + bucket_created = False + instance = SimpleNamespace(config=config, pg=postgres_infra.pg, store=store, admin=postgres_infra.admin) + try: + store.client.create_bucket(Bucket=config.bucket) + bucket_created = True + store.client.put_bucket_versioning(Bucket=config.bucket, VersioningConfiguration={'Status': 'Enabled'}) + yield instance + finally: + instance.pg.close() + if bucket_created: + # Test-only bucket with random unique name. Never use runtime cleanup in production. + for page in store.client.get_paginator('list_object_versions').paginate(Bucket=config.bucket): + items = [{'Key': v['Key'], 'VersionId': v['VersionId']} for v in page.get('Versions', []) + page.get('DeleteMarkers', [])] + if items: + store.client.delete_objects(Bucket=config.bucket, Delete={'Objects': items}) + store.client.delete_bucket(Bucket=config.bucket) diff --git a/tests/integration/test_archive.py b/tests/integration/test_archive.py new file mode 100644 index 0000000..d3fa031 --- /dev/null +++ b/tests/integration/test_archive.py @@ -0,0 +1,194 @@ +from datetime import datetime, timezone, timedelta +from decimal import Decimal +from pathlib import Path +import os +import subprocess +import sys + +from psycopg import sql +import pytest + +from btcdata_archive.archive import run +from btcdata_archive.postgres import Postgres +from btcdata_archive.schema import TABLES, encode_row, decode_row +from btcdata_archive.sqlite_archive import connect, validate +from btcdata_archive.weeks import boundaries, WEEK + +pytestmark = pytest.mark.integration +CUTOFF = boundaries(datetime.now(timezone.utc))[1] +START = CUTOFF - 3 * WEEK + + +def seed(infra): + for offset, pk in [(0, 1), (14, 3), (21, 4)]: + instant = START + timedelta(days=offset, microseconds=123456) + infra.admin.execute('INSERT INTO binance_price VALUES (%s,%s,%s)', (pk, Decimal('1234567890.12345678'), instant)) + infra.admin.execute("INSERT INTO strike_price VALUES (%s,'{\"a\":12345678901234567890.12345678901234567890,\"null\":null}'::jsonb,%s,NULL)", (pk, instant)) + infra.admin.execute('INSERT INTO kraken_minute VALUES (%s,1.00000001,2,3,4,5,1234567890123456.12345678,7,%s)', (int(instant.timestamp()), instant)) + + +def invoke(infra, mode='archive-and-prune', max_weeks=None): + return run(infra.pg, infra.store, infra.config, CUTOFF, mode, max_weeks) + + +def test_export_gap_precision_cutoff_restore(infra, tmp_path): + seed(infra) + original = {} + for table in TABLES: + projection = ','.join(n + '::text' if typ == 'jsonb' else n for n, typ, _ in table.columns) + original[table.name] = [encode_row(table, row) for row in infra.admin.execute(f'SELECT {projection} FROM public.{table.name} ORDER BY {table.pk}')] + # Exactly at cutoff must stay, regardless of current week. + infra.admin.execute('INSERT INTO binance_price VALUES (5,5,%s)', (CUTOFF,)) + invoke(infra, 'verify-only') + assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 4 + assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='verified'").fetchone()[0] == 3 + invoke(infra) + assert infra.admin.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(4,), (5,)] + assert infra.admin.execute('SELECT count(*) FROM kraken_minute').fetchone()[0] == 1 + for offset in (0, 7, 14): + start = START + timedelta(days=offset) + key = infra.config.prefix + start.date().isoformat() + '.sqlite' + head = infra.store.head(key) + path = tmp_path / ('restored-' + str(offset) + '.sqlite') + infra.store.download(key, head['VersionId'], path) + meta = validate(path, start, infra.config.pg['dbname'], 2) + assert all(v['count'] == (0 if offset == 7 else 1) for v in meta['tables'].values()) + # Restore every column through the documented type mapping into isolated PostgreSQL tables. + db = connect(path, readonly=True) + for table in TABLES: + restored = 'restored_' + str(offset) + '_' + table.name + infra.admin.execute(sql.SQL('CREATE TABLE {} (LIKE public.{})').format(sql.Identifier(restored), sql.Identifier(table.name))) + rows = db.execute(f'SELECT * FROM {table.name} ORDER BY {table.pk}').fetchall() + for row in rows: + placeholders = ','.join('%s::jsonb' if typ == 'jsonb' else '%s' for _, typ, _ in table.columns) + infra.admin.execute(sql.SQL(f'INSERT INTO {{}} VALUES ({placeholders})').format(sql.Identifier(restored)), decode_row(table, row)) + projection = ','.join(n + '::text' if typ == 'jsonb' else n for n, typ, _ in table.columns) + roundtrip = [encode_row(table, r) for r in infra.admin.execute(sql.SQL(f'SELECT {projection} FROM {{}} ORDER BY {table.pk}').format(sql.Identifier(restored)))] + assert roundtrip == rows + assert all(r in original[table.name] for r in rows) + db.close() + assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='complete'").fetchone()[0] == 3 + # Application latest-value reads still work with the retained rows. + assert infra.admin.execute('SELECT id FROM binance_price ORDER BY recorded_at DESC LIMIT 1').fetchone()[0] == 4 + + +def test_schema_mismatch(infra): + seed(infra) + infra.admin.execute('ALTER TABLE strike_price ADD COLUMN unexpected text') + with pytest.raises(RuntimeError, match='migration required'): + invoke(infra) + assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3 + + +def test_concurrent_lock_and_loss(infra, monkeypatch): + seed(infra) + second = Postgres(infra.config) + infra.pg.lock() + try: + with pytest.raises(RuntimeError, match='advisory lock'): + second.lock() + finally: + second.close() + # Closing the coordinating session after upload must prevent ANY deletion. + upload = infra.store.upload + def lose_lock(*args): + result = upload(*args) + infra.pg.close() + return result + monkeypatch.setattr(infra.store, 'upload', lose_lock) + # Don't acquire a second lock on the same session in run. + monkeypatch.setattr(infra.pg, 'lock', lambda: None) + with pytest.raises(Exception): + invoke(infra) + assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3 + + +def test_changed_and_late_rows(infra, monkeypatch): + seed(infra) + upload = infra.store.upload + def change(*args): + version = upload(*args) + infra.admin.execute('UPDATE binance_price SET price=9 WHERE id=1') + infra.admin.execute('INSERT INTO binance_price VALUES (2,2,%s)', (START,)) + return version + monkeypatch.setattr(infra.store, 'upload', change) + with pytest.raises(RuntimeError, match='reconciliation required'): + invoke(infra) + assert infra.admin.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(1,), (2,), (3,), (4,)] + # Other unchanged archived rows were safely pruned; their data must survive the merge. + assert infra.admin.execute('SELECT count(*) FROM strike_price').fetchone()[0] == 2 + monkeypatch.setattr(infra.store, 'upload', upload) + invoke(infra) + key = infra.config.prefix + START.date().isoformat() + '.sqlite' + path = infra.config.work_dir / 'late.sqlite' + infra.store.download(key, infra.store.head(key)['VersionId'], path) + db = connect(path, readonly=True) + assert db.execute('SELECT id,price FROM binance_price ORDER BY id').fetchall() == [(1,'9.00000000'), (2,'2.00000000')] + assert db.execute('SELECT count(*) FROM strike_price').fetchone()[0] == 1 + db.close() + + +@pytest.mark.parametrize('boundary', ['uploaded', 'verified', 'pruning', 'deletion_batch', 'complete']) +def test_crash_restart_boundaries(infra, monkeypatch, boundary): + seed(infra) + ledger, upload = infra.pg.ledger, infra.store.upload + fired = False + def crash_ledger(start, status, *args, **kwargs): + nonlocal fired + result = ledger(start, status, *args, **kwargs) + target = 'pruning' if boundary == 'deletion_batch' else boundary + if not fired and status == target and (boundary != 'deletion_batch' or kwargs.get('deletions')): + fired = True + raise KeyboardInterrupt() + return result + def crash_upload(*args): + result = upload(*args) + raise KeyboardInterrupt() + monkeypatch.setattr(infra.pg, 'ledger', crash_ledger) + if boundary == 'uploaded': + monkeypatch.setattr(infra.store, 'upload', crash_upload) + with pytest.raises(KeyboardInterrupt): + invoke(infra) + infra.pg.close() + infra.pg = Postgres(infra.config) + monkeypatch.setattr(infra.store, 'upload', upload) + invoke(infra) + key = infra.config.prefix + START.date().isoformat() + '.sqlite' + path = infra.config.work_dir / 'resumed.sqlite' + infra.store.download(key, infra.store.head(key)['VersionId'], path) + assert all(s['count'] == 1 for s in validate(path, START, infra.config.pg['dbname'], 2)['tables'].values()) + assert infra.admin.execute('SELECT id FROM binance_price').fetchall() == [(4,)] + + +@pytest.mark.parametrize('operation', ['upload', 'download']) +def test_network_failure_preserves_data(infra, monkeypatch, operation): + seed(infra) + def fail(*args): + raise OSError('network unavailable') + monkeypatch.setattr(infra.store, operation, fail) + with pytest.raises(OSError): + invoke(infra) + assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3 + + +def test_missing_delete_privilege(infra): + seed(infra) + infra.admin.execute(sql.SQL('REVOKE DELETE ON binance_price FROM {}').format(sql.Identifier(infra.config.pg['user']))) + with pytest.raises(Exception): + invoke(infra) + assert infra.admin.execute('SELECT count(*) FROM binance_price').fetchone()[0] == 3 + + +def test_wrapper_all_weeks_and_bound(infra): + seed(infra) + c = infra.config + env = {**os.environ, 'PGHOST': c.pg['host'], 'PGPORT': str(c.pg['port']), 'PGDATABASE': c.pg['dbname'], + 'PGUSER': c.pg['user'], 'PGPASSWORD': c.pg['password'], 'S3_ENDPOINT_URL': c.endpoint, + 'AWS_ACCESS_KEY_ID': c.access_key, 'AWS_SECRET_ACCESS_KEY': c.secret_key, 'ARCHIVE_BUCKET': c.bucket, + 'ARCHIVE_WORK_DIR': str(c.work_dir), 'PATH': str(Path(sys.executable).parent) + ':' + os.environ['PATH']} + script = str(Path(__file__).resolve().parents[2] / 'scripts/archive-old-weeks.sh') + first = subprocess.run([script, '--mode', 'verify-only', '--max-weeks', '1'], env=env, capture_output=True, text=True) + assert first.returncode == 0, first.stdout + second = subprocess.run([script, '--mode', 'archive-and-prune'], env=env, capture_output=True, text=True) + assert second.returncode == 0, second.stdout + assert infra.admin.execute("SELECT count(*) FROM btcdata_archive.weeks WHERE status='complete'").fetchone()[0] == 3 diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py new file mode 100644 index 0000000..4274828 --- /dev/null +++ b/tests/integration/test_postgres.py @@ -0,0 +1,88 @@ +"""PostgreSQL-only regressions; never connects using the production environment.""" +from datetime import datetime, timezone + +import psycopg +import pytest + +from btcdata_archive import sqlite_archive +from btcdata_archive.schema import TABLES +from btcdata_archive.weeks import WEEK + +pytestmark = pytest.mark.integration +START = datetime(2026, 2, 16, tzinfo=timezone.utc) + + +def test_ledger_progress_preserves_verification_before_insert_check(postgres_infra): + pg, admin = postgres_infra.pg, postgres_infra.admin + pg.lock() + key, version, checksum = 'test/week.sqlite', 'verified-version', 'a' * 64 + counts = {'binance_price': 3, 'strike_price': 0, 'kraken_minute': 0} + pg.ledger(START, 'exporting', key) + pg.ledger(START, 'verified', key, version, checksum, counts) + # These progress-only calls used to fail INSERT's CHECK before ON CONFLICT. + pg.ledger(START, 'pruning', key) + pg.ledger(START, 'pruning', key, deletions={'binance_price': 2}) + pg.ledger(START, 'complete', key, deletions={'binance_price': 3}) + row = admin.execute('SELECT status,object_version_id,file_sha256,row_counts,deletion_counts,completion_time IS NOT NULL FROM btcdata_archive.weeks WHERE week_start=%s', (START,)).fetchone() + assert row == ('complete', version, checksum, counts, {'binance_price': 3}, True) + pg.ledger(START, 'failed', key, error='test interruption') + pg.ledger(START, 'exporting', key) + pg.ledger(START, 'verified', key, 'new-version', 'b' * 64, counts) + pg.ledger(START, 'pruning', key) + assert admin.execute('SELECT object_version_id,file_sha256,completion_time FROM btcdata_archive.weeks').fetchone() == ('new-version', 'b' * 64, None) + + +def test_ledger_still_rejects_unverified_pruning(postgres_infra): + pg = postgres_infra.pg + pg.lock() + with pytest.raises(psycopg.errors.CheckViolation): + pg.ledger(START, 'pruning', 'test/week.sqlite') + pg.ledger(START, 'exporting', 'test/week.sqlite') + with pytest.raises(psycopg.errors.CheckViolation): + pg.ledger(START, 'complete', 'test/week.sqlite') + assert postgres_infra.admin.execute('SELECT status FROM btcdata_archive.weeks').fetchone() == ('exporting',) + + +def test_typed_pruning_changed_rows_and_recovery(postgres_infra, tmp_path): + pg, admin, config = postgres_infra.pg, postgres_infra.admin, postgres_infra.config + pg.lock() + pg.check_schema() + admin.execute('INSERT INTO binance_price VALUES (1,1.00000001,%s)', (START,)) + admin.execute('INSERT INTO strike_price VALUES (1,%s::jsonb,%s,NULL)', ('{"a":12345678901234567890.12345678901234567890}', START)) + admin.execute('INSERT INTO kraken_minute VALUES (%s,1.00000001,2,3,4,5,1234567890123456.12345678,7,%s)', (int(START.timestamp()), START.replace(microsecond=123456))) + path = tmp_path / 'verified.sqlite' + sqlite_archive.create(path) + key = 'test/week.sqlite' + + def export_and_record(version): + pg.ledger(START, 'exporting', key) + pg.export(path, START) + meta = sqlite_archive.finalize(path, START, config.pg['dbname'], START, 2) + pg.ledger(START, 'verified', key, version, sqlite_archive.file_hash(path), + {name: stats['count'] for name, stats in meta['tables'].items()}) + pg.ledger(START, 'pruning', key) + + def progress(deleted): + pg.ledger(START, 'pruning', key, deletions=deleted) + + export_and_record('test-v1') + # Source mutation and late insertion must remain; unchanged rows can be deleted. + admin.execute('UPDATE binance_price SET price=9 WHERE id=1') + admin.execute("INSERT INTO strike_price VALUES (2,'null'::jsonb,%s,NULL)", (START,)) + with pytest.raises(RuntimeError, match='reconciliation required'): + pg.prune(path, START, START + WEEK, progress) + assert admin.execute('SELECT id FROM binance_price').fetchall() == [(1,)] + assert admin.execute('SELECT id FROM strike_price').fetchall() == [(2,)] + assert admin.execute('SELECT count(*) FROM kraken_minute').fetchone() == (0,) + pg.ledger(START, 'failed', key, error='test reconciliation') + export_and_record('test-v2') + deleted = pg.prune(path, START, START + WEEK, progress) + pg.ledger(START, 'complete', key, deletions=deleted) + assert all(pg.counts(START, START + WEEK)[t.name] == 0 for t in TABLES) + db = sqlite_archive.connect(path, readonly=True) + try: + assert db.execute('SELECT id,price FROM binance_price').fetchall() == [(1, '9.00000000')] + assert db.execute('SELECT id,rates,quote_sats FROM strike_price ORDER BY id').fetchall() == [(1, '{"a":12345678901234567890.12345678901234567890}', None), (2, 'null', None)] + assert db.execute('SELECT count(*) FROM kraken_minute').fetchone() == (1,) + finally: + db.close() diff --git a/tests/unit/test_backends.py b/tests/unit/test_backends.py new file mode 100644 index 0000000..2c09770 --- /dev/null +++ b/tests/unit/test_backends.py @@ -0,0 +1,99 @@ +import io +from types import SimpleNamespace + +import pytest +from botocore.response import StreamingBody +from botocore.stub import Stubber + +from btcdata_archive.errors import SafetyError +from btcdata_archive.object_store import ObjectStore +from btcdata_archive.postgres import Postgres +from btcdata_archive.schema import TABLES + + +def store(): + return ObjectStore(SimpleNamespace(endpoint='http://127.0.0.1:19000', access_key='test', secret_key='test', region='us-east-1', bucket='test')) + + +def test_versioning_enforced(): + backend = store() + with Stubber(backend.client) as stub: + stub.add_response('get_bucket_versioning', {'Status':'Suspended'}, {'Bucket':'test'}) + with pytest.raises(SafetyError, match='versioning'): + backend.require_versioning() + + +def test_head_missing_not_forbidden(): + backend = store() + with Stubber(backend.client) as stub: + stub.add_client_error('head_object', 'NoSuchKey', http_status_code=404, expected_params={'Bucket':'test','Key':'week.sqlite'}) + assert backend.head('week.sqlite') is None + stub.add_client_error('head_object', 'AccessDenied', http_status_code=403, expected_params={'Bucket':'test','Key':'week.sqlite'}) + with pytest.raises(Exception): + backend.head('week.sqlite') + + +def test_download_exact_version(tmp_path): + backend = store() + body = StreamingBody(io.BytesIO(b'archive'), 7) + with Stubber(backend.client) as stub: + stub.add_response('get_object', {'VersionId':'v1','Body':body}, {'Bucket':'test','Key':'week.sqlite','VersionId':'v1'}) + path = tmp_path / 'archive' + backend.download('week.sqlite','v1',path) + assert path.read_bytes() == b'archive' + stub.add_response('get_object', {'VersionId':'v2','Body':StreamingBody(io.BytesIO(b'bad'), 3)}, {'Bucket':'test','Key':'week.sqlite','VersionId':'v1'}) + with pytest.raises(SafetyError, match='wrong version'): + backend.download('week.sqlite','v1',path) + + +def test_upload_captures_completion_version(tmp_path): + backend = store() + path = tmp_path / 'archive' + path.write_bytes(b'archive') + with Stubber(backend.client) as stub: + stub.add_response('get_bucket_versioning', {'Status':'Enabled'}, {'Bucket':'test'}) + stub.add_response('create_multipart_upload', {'UploadId':'upload'}, {'Bucket':'test','Key':'week.sqlite','ContentType':'application/vnd.sqlite3','Metadata':{'sha256':'digest'}}) + stub.add_response('upload_part', {'ETag':'not-a-checksum'}, {'Bucket':'test','Key':'week.sqlite','UploadId':'upload','PartNumber':1,'Body':b'archive'}) + stub.add_response('complete_multipart_upload', {'VersionId':'verified-version'}, {'Bucket':'test','Key':'week.sqlite','UploadId':'upload','MultipartUpload':{'Parts':[{'PartNumber':1,'ETag':'not-a-checksum'}]}}) + assert backend.upload('week.sqlite',path,'digest') == 'verified-version' + + +class Result: + def __init__(self, rows): + self.rows = rows + def fetchone(self): + return self.rows[0] + def fetchall(self): + return self.rows + + +class SchemaConnection: + def __init__(self, change=None): + self.change = change + def execute(self, query, params): + table = next(t for t in TABLES if params[0] == 'public.' + t.name) + if 'relkind' in query: + return Result([('r',False)]) + if 'pg_trigger' in query or 'confdeltype' in query: + return Result([(0,)]) + if 'format_type' in query: + columns = list(table.columns) + if self.change == 'added': + columns.append(('extra','text',True)) + elif self.change == 'removed': + columns.pop() + elif self.change == 'type': + columns[0] = ('id','integer',False) + return Result(columns) + return Result([([table.pk if self.change != 'pk' else 'wrong'],)]) + + +@pytest.mark.parametrize('change', [None,'added','removed','type','pk']) +def test_source_schema_fail_closed(change): + pg = Postgres.__new__(Postgres) + pg.conn = SchemaConnection(change) + if change is None: + pg.check_schema() + else: + with pytest.raises(SafetyError, match='migration required'): + pg.check_schema() diff --git a/tests/unit/test_contract.py b/tests/unit/test_contract.py new file mode 100644 index 0000000..d205b5f --- /dev/null +++ b/tests/unit/test_contract.py @@ -0,0 +1,169 @@ +from datetime import datetime, timezone, timedelta +from decimal import Decimal +import json +import sqlite3 + +import pytest + +from btcdata_archive import sqlite_archive as archive +from btcdata_archive.cli import parser, main +from btcdata_archive.config import Config, load_env_file +from btcdata_archive.errors import SafetyError +from btcdata_archive.schema import TABLES, Digest, encode_row, decode_row, canonical_json +from btcdata_archive.weeks import boundaries, week_start, weeks, timestamp + +UTC = timezone.utc +START = datetime(2026, 8, 31, tzinfo=UTC) + + +@pytest.mark.parametrize('instant,anchor,cutoff', [ + ('2026-10-05T00:00:00+00:00', '2026-10-05', '2026-09-07'), + ('2026-10-04T23:59:59+00:00', '2026-09-28', '2026-08-31'), + ('2027-01-01T12:00:00+00:00', '2026-12-28', '2026-11-30'), + ('2024-02-29T12:00:00+00:00', '2024-02-26', '2024-01-29'), + ('2026-03-29T03:30:00+02:00', '2026-03-23', '2026-02-23'), +]) +def test_boundaries(instant, anchor, cutoff): + result = boundaries(datetime.fromisoformat(instant)) + assert [d.date().isoformat() for d in result] == [anchor, cutoff] + + +def test_week_intervals(): + assert list(weeks(START + timedelta(days=2), START + timedelta(days=21))) == [START, START + timedelta(days=7), START + timedelta(days=14)] + assert list(weeks(None, START)) == [] + assert list(weeks(START, START)) == [] + with pytest.raises(ValueError): + week_start(datetime(2026, 1, 1)) + assert TABLES[0].retention == TABLES[1].retention == 'recorded_at' + assert TABLES[2].retention == 'ts' + + +def test_exact_roundtrips(tmp_path): + path = tmp_path / 'week.sqlite' + archive.create(path) + values = [ + (2**63 - 1, Decimal('1234567890.12345678'), START.replace(microsecond=123456)), + (2**63 - 2, '{"z":null,"b":12345678901234567890.12345678901234567890,"a":[1,true]}', START, None), + (int(START.timestamp()), *[Decimal('0.00000001')] * 6, 0, START.replace(microsecond=999999)), + ] + db = archive.connect(path) + for table, row in zip(TABLES, values): + encoded = encode_row(table, row) + archive.upsert(db, table, [encoded]) + assert encode_row(table, decode_row(table, encoded)) == encoded + db.commit() + db.close() + meta = archive.finalize(path, START, 'btcprod', START, 1) + assert all(s['count'] == 1 for s in meta['tables'].values()) + db = archive.connect(path, readonly=True) + rates = db.execute('SELECT rates, quote_sats FROM strike_price').fetchone() + assert rates[1] is None + assert '12345678901234567890.12345678901234567890' in rates[0] + assert json.loads(rates[0])['z'] is None + db.close() + assert not list(tmp_path.glob('*-wal')) + assert timestamp(START.replace(microsecond=1)).endswith('.000001Z') + + +@pytest.mark.parametrize('price', ['1.000000001', '10000000000.00000000']) +def test_decimal_type_limits(tmp_path, price): + path = tmp_path / 'week.sqlite' + archive.create(path) + db = archive.connect(path) + archive.upsert(db, TABLES[0], [(1, price, timestamp(START))]) + db.commit() + db.close() + with pytest.raises(SafetyError, match='precision/scale'): + archive.finalize(path, START, 'btcprod', START, 2) + + +def test_json_and_digest(): + assert canonical_json('{"b": 1.234567890123456789, "a": null}') == '{"a":null,"b":1.234567890123456789}' + a, b = Digest(), Digest() + a.add((1, 'a,b', None)) + b.add((1, 'a', 'b')) + assert a.result() != b.result() + + +def test_empty_and_corruption(tmp_path): + path = tmp_path / 'week.sqlite' + archive.create(path) + assert all(v['count'] == 0 for v in archive.finalize(path, START, 'btcprod', START, 2)['tables'].values()) + with pytest.raises(ValueError, match='metadata'): + archive.validate(path, START, 'other', 2) + db = archive.connect(path) + db.execute('ALTER TABLE strike_price ADD COLUMN extra TEXT') + db.commit() + db.close() + with pytest.raises(ValueError, match='schema'): + archive.validate(path, START, 'btcprod', 2) + path.write_bytes(b'corrupt') + with pytest.raises(sqlite3.DatabaseError): + archive.validate(path, START, 'btcprod', 2) + + +def test_outside_week_and_checksum(tmp_path): + path = tmp_path / 'week.sqlite' + archive.create(path) + db = archive.connect(path) + archive.upsert(db, TABLES[0], [(1, '1.00000000', timestamp(START))]) + db.commit() + db.close() + archive.finalize(path, START, 'btcprod', START, 2) + db = archive.connect(path) + db.execute("UPDATE binance_price SET price='2.00000000'") + db.commit() + db.close() + with pytest.raises(ValueError, match='checksum'): + archive.validate(path, START, 'btcprod', 2) + db = archive.connect(path) + db.execute('UPDATE binance_price SET recorded_at=?', (timestamp(START + timedelta(days=7)),)) + db.commit() + db.close() + with pytest.raises(ValueError, match='outside week'): + archive.finalize(path, START, 'btcprod', START, 2) + + +def test_literal_env_file(tmp_path, monkeypatch): + path = tmp_path / '.env' + path.write_text("# comment\nTEST_ARCHIVE_VALUE='space # $value'\nTEST_ARCHIVE_KEEP=file\nTEST_ARCHIVE_EMPTY=\nTEST_ARCHIVE_LITERAL='$(touch should-not-exist)'\n") + for key in ('TEST_ARCHIVE_VALUE', 'TEST_ARCHIVE_KEEP', 'TEST_ARCHIVE_EMPTY', 'TEST_ARCHIVE_LITERAL'): + monkeypatch.delenv(key, raising=False) + # Register environment restoration even for variables created by the loader. + monkeypatch.setenv(key, '') + monkeypatch.delenv(key) + monkeypatch.setenv('TEST_ARCHIVE_KEEP', 'environment') + load_env_file(path) + import os + assert os.environ['TEST_ARCHIVE_VALUE'] == 'space # $value' + assert os.environ['TEST_ARCHIVE_KEEP'] == 'environment' + assert os.environ['TEST_ARCHIVE_EMPTY'] == '' + assert os.environ['TEST_ARCHIVE_LITERAL'] == '$(touch should-not-exist)' + path.write_text("TEST_ARCHIVE_VALUE='private-unclosed-value\n") + with pytest.raises(SafetyError) as error: + load_env_file(path) + assert 'private-unclosed-value' not in str(error.value) + + +@pytest.mark.parametrize('args', [['run'], ['run', '--mode', 'other'], ['run', '--mode', 'verify-only', '--max-weeks', '0'], ['run', '--mode', 'verify-only', '--max-weeks', '-1']]) +def test_cli_rejects(args): + with pytest.raises(SystemExit): + parser().parse_args(args) + + +def test_config_and_safe_error(monkeypatch, capsys): + monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False) + for key in ('PGHOST','PGUSER','PGPASSWORD','S3_ENDPOINT_URL','AWS_ACCESS_KEY_ID','AWS_SECRET_ACCESS_KEY'): + monkeypatch.delenv(key, raising=False) + assert main(['run', '--mode', 'verify-only']) == 1 + assert 'missing configuration' in capsys.readouterr().out + for key in ('PGHOST','PGUSER','PGPASSWORD'): + monkeypatch.setenv(key, 'test') + assert Config.from_env(need_s3=False).endpoint is None + with pytest.raises(SafetyError): + Config.from_env() + monkeypatch.setenv('S3_ENDPOINT_URL', 'http://user:password@example.com') + monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test') + monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test') + with pytest.raises(SafetyError): + Config.from_env() diff --git a/tests/unit/test_report.py b/tests/unit/test_report.py new file mode 100644 index 0000000..e689d2d --- /dev/null +++ b/tests/unit/test_report.py @@ -0,0 +1,157 @@ +from pathlib import Path +from types import SimpleNamespace +import hashlib +import os +import shutil +import subprocess +import sys + +import pytest + +from btcdata_archive.config import Config +from btcdata_archive.errors import SafetyError +from btcdata_archive import report, sqlite_archive +from btcdata_archive.schema import TABLES + + +class Store: + def __init__(self, objects): + self.objects = objects + self.client = self + self.downloads = [] + + def get_paginator(self, operation): + assert operation == 'list_objects_v2' + return self + + def paginate(self, **kwargs): + # Multiple pages; counts must include every object, not only the first page. + for key, data in self.objects.items(): + yield {'Contents': [{'Key': key, 'Size': len(data)}]} + + def head(self, key): + data = self.objects[key] + return {'VersionId': 'pinned-version', 'ContentLength': len(data), + 'Metadata': {'sha256': hashlib.sha256(data).hexdigest()}} + + def download(self, key, version, path): + assert version == 'pinned-version' + # Previously downloaded report files must already have been removed. + assert not list(path.parent.parent.glob('btcdata-report-*/archive.sqlite')) + self.downloads.append(key) + path.write_bytes(self.objects[key]) + + +def archived_file(path, populated=True): + sqlite_archive.create(path) + db = sqlite_archive.connect(path) + if populated: + sqlite_archive.upsert(db, TABLES[0], [(1, '1.00000000', '2026-02-16T00:00:00.000000Z'), + (2, '2.00000000', '2026-02-16T00:00:00.000000Z')]) + sqlite_archive.upsert(db, TABLES[1], [(1, 'null', '2026-02-16T00:00:00.000000Z', None)]) + # Deliberately incorrect metadata; reporting must count the actual SQLite rows. + db.execute('INSERT INTO metadata VALUES (?,?)', ('tables', '{"binance_price":{"count":99999}}')) + db.commit() + db.close() + return path.read_bytes() + + +@pytest.fixture +def system(tmp_path): + first = archived_file(tmp_path / 'first.sqlite') + second = archived_file(tmp_path / 'second.sqlite', populated=False) + config = SimpleNamespace(bucket='test', prefix='btcprod/weeks/', work_dir=tmp_path / 'work') + store = Store({'btcprod/weeks/first.sqlite': first, 'btcprod/weeks/second.sqlite': second, + 'btcprod/weeks/readme.txt': b'not sqlite'}) + return config, store + + +def test_report_actual_sqlite_counts_pagination_cleanup(system): + config, store = system + result = list(report.files(store, config)) + assert result[0][2] == {'binance_price': 2, 'strike_price': 1, 'kraken_minute': 0} + assert result[1][2] == dict.fromkeys(report.TABLE_NAMES, 0) + assert result[2] == ('btcprod/weeks/readme.txt', 10, None) + assert store.downloads == ['btcprod/weeks/first.sqlite', 'btcprod/weeks/second.sqlite'] + assert not list(config.work_dir.iterdir()) + + +def test_report_output_and_totals(system, capsys): + config, store = system + report.show(store, config) + output = capsys.readouterr().out + assert all(name in output for name in report.TABLE_NAMES) + assert 'first.sqlite' in output and 'second.sqlite' in output and 'readme.txt' in output + assert 'TOTAL' in output and '3 files' in output + assert '99999' not in output + total_line = next(line for line in output.splitlines() if line.startswith('TOTAL')) + assert total_line.split()[-3:] == ['2', '1', '0'] + + +def test_report_config_requires_no_postgres(monkeypatch): + monkeypatch.delenv('ARCHIVE_ENV_FILE', raising=False) + for key in ('PGHOST','PGUSER','PGPASSWORD'): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv('PGPORT', 'not a port') + monkeypatch.setenv('S3_ENDPOINT_URL', 'http://127.0.0.1:9000') + monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'test') + monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'test') + assert Config.from_env(need_pg=False).pg == {} + + +def test_report_main_without_postgres(system, monkeypatch): + config, store = system + monkeypatch.setattr(report.Config, 'from_env', lambda **kwargs: config if kwargs == {'need_pg': False} else pytest.fail('PG requested')) + monkeypatch.setattr(report, 'ObjectStore', lambda config: store) + # Importing this reporting module in isolation must not even load psycopg. + root = Path(__file__).resolve().parents[2] + result = subprocess.run([sys.executable, '-c', + 'import sys; import btcdata_archive.report; assert "psycopg" not in sys.modules; assert "btcdata_archive.postgres" not in sys.modules'], + env={**os.environ, 'PYTHONPATH': str(root / 'src')}, capture_output=True) + assert result.returncode == 0 + assert report.main([]) == 0 + + +def test_report_corrupt_download_cleans_up(system): + config, store = system + store.objects['btcprod/weeks/first.sqlite'] = b'not a database' + with pytest.raises(Exception): + list(report.files(store, config)) + assert not list(config.work_dir.iterdir()) + + +def test_report_checksum_mismatch(system, monkeypatch): + config, store = system + head = store.head + def wrong_checksum(key): + result = head(key) + result['Metadata']['sha256'] = '0' * 64 + return result + monkeypatch.setattr(store, 'head', wrong_checksum) + with pytest.raises(SafetyError, match='SHA-256'): + list(report.files(store, config)) + assert not list(config.work_dir.iterdir()) + + +def test_report_disk_space(system, monkeypatch): + config, store = system + monkeypatch.setattr(shutil, 'disk_usage', lambda path: SimpleNamespace(free=0)) + with pytest.raises(SafetyError, match='storage'): + list(report.files(store, config)) + assert not store.downloads + + +def test_report_missing_table(tmp_path): + path = tmp_path / 'bad.sqlite' + sqlite_archive.create(path) + db = sqlite_archive.connect(path) + db.execute('DROP TABLE strike_price') + db.commit() + db.close() + with pytest.raises(SafetyError, match='required table'): + report.sqlite_counts(path) + + +def test_report_empty_listing(capsys, tmp_path): + report.show(Store({}), SimpleNamespace(bucket='test',prefix='btcprod/weeks/',work_dir=tmp_path)) + assert '0 files' in capsys.readouterr().out diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py new file mode 100644 index 0000000..ba0ccf6 --- /dev/null +++ b/tests/unit/test_workflow.py @@ -0,0 +1,213 @@ +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace +import shutil +import subprocess +import os + +import pytest + +from btcdata_archive.archive import run +from btcdata_archive import sqlite_archive +from btcdata_archive.schema import TABLES +from btcdata_archive.weeks import timestamp +from test_contract import START + + +class PG: + def __init__(self, config): + self.config = config + self.data = {START: [(1, '1.00000000', timestamp(START))], START + timedelta(days=14): [(3, '3.00000000', timestamp(START + timedelta(days=14)))]} + self.states = {} + self.events = [] + self.locked = False + self.change = False + + def lock(self): + self.locked = True + + def alive(self): + if not self.locked: + raise RuntimeError('session lost') + + def check_schema(self): + pass + + def first_week(self, cutoff): + return min([s for s, data in self.data.items() if data and s < cutoff] + [s for s, state in self.states.items() if state != 'complete'], default=None) + + def counts(self, start, end): + return {t.name: len(self.data.get(start, [])) if t == TABLES[0] else 0 for t in TABLES} + + def archived_version(self, start): + return None + + def estimated_bytes(self, start): + return 1024 + + def ledger(self, start, status, *args, **kwargs): + self.alive() + self.events.append((start, status)) + self.states[start] = status + + def export(self, path, start): + self.alive() + db = sqlite_archive.connect(path) + sqlite_archive.upsert(db, TABLES[0], self.data.get(start, [])) + db.commit() + db.close() + + def prune(self, path, start, cutoff, progress): + self.alive() + if self.change: + self.data[start][0] = (1, '9.00000000', timestamp(start)) + self.change = False + db = sqlite_archive.connect(path, readonly=True) + archived = set(db.execute('SELECT * FROM binance_price')) + db.close() + old = self.data.get(start, []) + self.data[start] = [r for r in old if r not in archived] + progress({'binance_price': len(old) - len(self.data[start])}) + if self.data[start]: + raise RuntimeError('reconciliation required') + return {'binance_price': len(old)} + + +class Store: + def __init__(self): + self.objects = {} + self.fail = None + self.pg = None + + def require_versioning(self): + if self.fail == 'versioning': + raise RuntimeError('disabled') + + def head(self, key): + if key not in self.objects: + return None + version, data, checksum = self.objects[key][-1] + return {'VersionId': version, 'ContentLength': len(data), 'Metadata': {'sha256': checksum}} + + def upload(self, key, path, checksum): + if self.fail == 'upload': + raise OSError('sensitive password') + version = str(len(self.objects.get(key, [])) + 1) + self.objects.setdefault(key, []).append((version, path.read_bytes(), checksum)) + if self.fail == 'lock_loss': + self.pg.locked = False + return version + + def download(self, key, version, path): + if self.fail == 'download': + raise OSError('sensitive payload') + data = next(data for v, data, sha in self.objects[key] if v == version) + path.write_bytes(data if self.fail != 'checksum' else data + b'bad') + + +@pytest.fixture +def system(tmp_path): + config = SimpleNamespace(prefix='btcprod/weeks/', work_dir=tmp_path, pg={'dbname': 'btcprod'}, batch_rows=2) + pg, store = PG(config), Store() + store.pg = pg + return config, pg, store + + +def test_sequential_gap_and_fixed_cutoff(system): + config, pg, store = system + cutoff = START + timedelta(days=21) + run(pg, store, config, cutoff, 'archive-and-prune') + assert [s for s, state in pg.events if state == 'exporting'] == [START + timedelta(days=d) for d in (0,7,14)] + assert len(store.objects) == 3 + assert all(not values for values in pg.data.values()) + for i, (s, state) in enumerate(pg.events): + if state == 'exporting' and s != START: + assert pg.events[i-1][1] == 'complete' + assert not list(config.work_dir.iterdir()) + + +def test_verify_only_bounded_resume_preserves_old_rows(system, tmp_path): + config, pg, store = system + cutoff = START + timedelta(days=21) + run(pg, store, config, cutoff, 'verify-only', 1) + assert pg.states[START] == 'verified' + assert pg.data[START] + run(pg, store, config, cutoff, 'archive-and-prune', 1) + assert not pg.data[START] + pg.data[START] = [(2, '2.00000000', timestamp(START))] + run(pg, store, config, cutoff, 'archive-and-prune', 1) + key = config.prefix + START.date().isoformat() + '.sqlite' + path = tmp_path / 'restore.sqlite' + store.download(key, store.head(key)['VersionId'], path) + db = sqlite_archive.connect(path, readonly=True) + assert db.execute('SELECT id FROM binance_price ORDER BY id').fetchall() == [(1,), (2,)] + db.close() + assert len(store.objects[key]) == 3 + + +@pytest.mark.parametrize('failure', ['upload', 'download', 'checksum', 'versioning', 'lock_loss']) +def test_failure_never_prunes_unverified_or_advances(system, failure, capsys): + config, pg, store = system + store.fail = failure + with pytest.raises(Exception): + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune') + assert pg.data[START] + assert not any(s > START for s, state in pg.events) + assert 'sensitive' not in capsys.readouterr().out + assert not list(config.work_dir.iterdir()) + + +def test_changed_row_reconciliation(system): + config, pg, store = system + pg.change = True + with pytest.raises(RuntimeError): + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune') + assert pg.states[START] == 'failed' + assert pg.data[START][0][1] == '9.00000000' + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune', 1) + assert not pg.data[START] + + +def test_full_disk(system, monkeypatch): + config, pg, store = system + monkeypatch.setattr(shutil, 'disk_usage', lambda p: SimpleNamespace(free=0)) + with pytest.raises(RuntimeError, match='storage'): + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune') + assert pg.data[START] + assert not store.objects + + +def test_missing_previously_verified_object(system, monkeypatch): + config, pg, store = system + monkeypatch.setattr(pg, 'archived_version', lambda start: 'previous-version') + with pytest.raises(RuntimeError, match='operator recovery'): + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune') + assert pg.data[START] + assert not store.objects + + +def test_empty(system): + config, pg, store = system + pg.data.clear() + run(pg, store, config, START + timedelta(days=21), 'archive-and-prune') + assert not store.objects + + +def test_wrapper_interpreter_override_and_arguments(tmp_path): + interpreter = tmp_path / 'python-test' + interpreter.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\nexit 17\n') + interpreter.chmod(0o755) + script = Path('scripts/archive-old-weeks.sh').resolve() + result = subprocess.run([str(script), '--mode', 'verify-only', '--max-weeks', '1'], + cwd=tmp_path, env={**os.environ, 'PYTHON': str(interpreter)}, + capture_output=True, text=True) + assert result.returncode == 17 + assert result.stdout.splitlines() == ['-m', 'btcdata_archive', 'run', '--mode', 'verify-only', '--max-weeks', '1'] + + +def test_wrapper_requires_mode(): + import sys + env = {**os.environ, 'PATH': str(Path(sys.executable).parent) + ':' + os.environ['PATH']} + result = subprocess.run(['scripts/archive-old-weeks.sh'], env=env, capture_output=True, text=True) + assert result.returncode == 2 + assert '--mode' in result.stderr