diff --git a/README.md b/README.md index 12c862e..0633daa 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,9 @@ Standalone Python archiver for the three fixed `btcprod` tables: `public.binance and versioned archive bucket, progress ledger and restricted PostgreSQL role have been provisioned; source index preflight and deployment are still pending.** An operator run verified the first weekly archive, then hit a ledger transition bug -before deletion. That bug is now fixed; no deployment has been performed. Remote -repository hosting and CI/Harbor provisioning remain operator decisions. +before deletion. That bug is now fixed; deployment is being prepared suspended. +The repository is hosted at https://git.pmagnus.cloud/magnus/btcdata-archive; +Gitea Actions and project-scoped Harbor CI/pull credentials are configured. ## Install and test @@ -38,8 +39,12 @@ kill/network/disk fault testing and production connectivity remain rollout gates adds pytest. Regenerate with Python 3.12, pip 25.2 and pip-tools 7.5.1: `pip-compile --generate-hashes -o requirements.lock pyproject.toml` and `pip-compile --extra test --generate-hashes -o requirements-test.lock pyproject.toml`. -CI tests before publishing a full-commit tag to the existing approved Harbor registry. -Deploy using the published digest, not a mutable tag. No remote has been configured. +CI runs the full suite against disposable PostgreSQL/RustFS before publishing a +full-commit tag to the private `btcdata-archive` project in the approved Harbor registry. +CI has project-scoped push/pull access; Kubernetes has a separate pull-only identity. +Tests run inside a disposable container on an isolated Docker network, without +published ports or production credentials, including when CI uses a Docker job container. +Deploy using the published digest, not a mutable tag. Remote: `gitea:magnus/btcdata-archive.git`. ## Configuration diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh index 79aa8f3..be95c00 100755 --- a/scripts/test-integration.sh +++ b/scripts/test-integration.sh @@ -1,27 +1,17 @@ #!/usr/bin/env bash set -euo pipefail cd "$(dirname "$0")/.." -# Only new, disposable containers; no production connection settings are consumed. -export COMPOSE_PROJECT_NAME="btcdata-archive-test-$$" +# Only disposable containers; no production connection settings or published ports. +export COMPOSE_PROJECT_NAME="btcdata-archive-test-${GITHUB_RUN_ID:-$$}-${GITHUB_RUN_ATTEMPT:-1}" compose=(docker compose -f tests/integration/compose.yaml) trap '"${compose[@]}" down -v' EXIT -"${compose[@]}" up -d -export TEST_PGHOST=127.0.0.1 TEST_PGPORT=15432 TEST_PGUSER=postgres TEST_PGPASSWORD=disposable-test -export TEST_S3_ENDPOINT_URL=http://127.0.0.1:19000 -export TEST_S3_ACCESS_KEY=disposable-test TEST_S3_SECRET_KEY=disposable-test-secret -python - <<'PY' -import os, time -import boto3, psycopg -from botocore.config import Config -for attempt in range(90): - try: - with psycopg.connect(host=os.environ['TEST_PGHOST'], port=os.environ['TEST_PGPORT'], user='postgres', password='disposable-test', dbname='postgres', connect_timeout=2): - pass - boto3.client('s3', endpoint_url=os.environ['TEST_S3_ENDPOINT_URL'], aws_access_key_id='disposable-test', aws_secret_access_key='disposable-test-secret', region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=2, retries={'max_attempts': 0}, s3={'addressing_style': 'path'})).list_buckets() - break - except Exception: - time.sleep(2) -else: - raise SystemExit('disposable PostgreSQL/RustFS did not become ready') -PY -python -m pytest -q tests/integration +"${compose[@]}" up -d postgres rustfs +"${compose[@]}" create --no-deps tests +runner="$("${compose[@]}" ps -aq tests)" +# CI's workspace is inside its job container, not on the Docker host. Copy only +# test inputs; never bind-mount the workspace or copy .env, .git, or production secrets. +COPYFILE_DISABLE=1 tar -cf - requirements-test.lock src tests sql \ + scripts/archive-old-weeks.sh scripts/list-archives.sh scripts/wait-test-infrastructure.py \ + | docker cp - "$runner:/app" +docker start --attach "$runner" +exit "$(docker inspect --format '{{.State.ExitCode}}' "$runner")" diff --git a/scripts/wait-test-infrastructure.py b/scripts/wait-test-infrastructure.py new file mode 100644 index 0000000..7b027ec --- /dev/null +++ b/scripts/wait-test-infrastructure.py @@ -0,0 +1,26 @@ +"""Wait only for explicitly configured disposable integration services.""" +import os +import time + +import boto3 +from botocore.config import Config +import psycopg + + +for attempt in range(90): + try: + with psycopg.connect(host=os.environ['TEST_PGHOST'], port=os.environ['TEST_PGPORT'], + user=os.environ['TEST_PGUSER'], password=os.environ['TEST_PGPASSWORD'], + dbname='postgres', connect_timeout=2): + pass + boto3.client('s3', endpoint_url=os.environ['TEST_S3_ENDPOINT_URL'], + aws_access_key_id=os.environ['TEST_S3_ACCESS_KEY'], + aws_secret_access_key=os.environ['TEST_S3_SECRET_KEY'], region_name='us-east-1', + config=Config(connect_timeout=2, read_timeout=2, retries={'max_attempts': 0}, + s3={'addressing_style': 'path'})).list_buckets() + break + except Exception as error: + last_error = type(error).__name__ + time.sleep(2) +else: + raise SystemExit('Disposable PostgreSQL/RustFS did not become ready: ' + last_error) diff --git a/tests/integration/compose.yaml b/tests/integration/compose.yaml index e151e69..a96edbc 100644 --- a/tests/integration/compose.yaml +++ b/tests/integration/compose.yaml @@ -1,10 +1,10 @@ # Disposable tests ONLY. Known test credentials are intentionally not production secrets. +# No published ports: tests run on this isolated network, including in nested Gitea jobs. services: postgres: image: postgres:18.0-bookworm environment: POSTGRES_PASSWORD: disposable-test - ports: ["127.0.0.1:15432:5432"] tmpfs: ["/var/lib/postgresql"] healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] @@ -19,7 +19,25 @@ services: RUSTFS_CONSOLE_ENABLE: "false" RUSTFS_OBS_LOG_DIRECTORY: /logs command: ["/data"] - ports: ["127.0.0.1:19000:9000"] tmpfs: - /data:uid=10001,gid=10001 - /logs:uid=10001,gid=10001 + tests: + image: python:3.12.12-slim-bookworm + working_dir: /app + environment: + PYTHONPATH: /app/src + TEST_PGHOST: postgres + TEST_PGPORT: "5432" + TEST_PGUSER: postgres + TEST_PGPASSWORD: disposable-test + TEST_S3_ENDPOINT_URL: http://rustfs:9000 + TEST_S3_ACCESS_KEY: disposable-test + TEST_S3_SECRET_KEY: disposable-test-secret + command: + - /bin/sh + - -c + - >- + pip install --disable-pip-version-check --require-hashes -r requirements-test.lock && + python scripts/wait-test-infrastructure.py && + python -m pytest -q