Run isolated PostgreSQL and RustFS tests in Gitea CI

README.md

@@ -5,8 +5,9 @@ Standalone Python archiver for the three fixed `btcprod` tables: `public.binance
 and versioned archive bucket, progress ledger and restricted PostgreSQL role have
 been provisioned; source index preflight and deployment are still pending.**
 An operator run verified the first weekly archive, then hit a ledger transition bug
-before deletion. That bug is now fixed; no deployment has been performed. Remote
+before deletion. That bug is now fixed; deployment is being prepared suspended.
-repository hosting and CI/Harbor provisioning remain operator decisions.
+The repository is hosted at https://git.pmagnus.cloud/magnus/btcdata-archive;
+Gitea Actions and project-scoped Harbor CI/pull credentials are configured.
 
 ## Install and test
 
@@ -38,8 +39,12 @@ kill/network/disk fault testing and production connectivity remain rollout gates
 adds pytest. Regenerate with Python 3.12, pip 25.2 and pip-tools 7.5.1:
 `pip-compile --generate-hashes -o requirements.lock pyproject.toml` and
 `pip-compile --extra test --generate-hashes -o requirements-test.lock pyproject.toml`.
-CI tests before publishing a full-commit tag to the existing approved Harbor registry.
+CI runs the full suite against disposable PostgreSQL/RustFS before publishing a
-Deploy using the published digest, not a mutable tag. No remote has been configured.
+full-commit tag to the private `btcdata-archive` project in the approved Harbor registry.
+CI has project-scoped push/pull access; Kubernetes has a separate pull-only identity.
+Tests run inside a disposable container on an isolated Docker network, without
+published ports or production credentials, including when CI uses a Docker job container.
+Deploy using the published digest, not a mutable tag. Remote: `gitea:magnus/btcdata-archive.git`.
 
 ## Configuration
 

scripts/test-integration.sh

@@ -1,27 +1,17 @@
 #!/usr/bin/env bash
 set -euo pipefail
 cd "$(dirname "$0")/.."
-# Only new, disposable containers; no production connection settings are consumed.
+# Only disposable containers; no production connection settings or published ports.
-export COMPOSE_PROJECT_NAME="btcdata-archive-test-$$"
+export COMPOSE_PROJECT_NAME="btcdata-archive-test-${GITHUB_RUN_ID:-$$}-${GITHUB_RUN_ATTEMPT:-1}"
 compose=(docker compose -f tests/integration/compose.yaml)
 trap '"${compose[@]}" down -v' EXIT
-"${compose[@]}" up -d
+"${compose[@]}" up -d postgres rustfs
-export TEST_PGHOST=127.0.0.1 TEST_PGPORT=15432 TEST_PGUSER=postgres TEST_PGPASSWORD=disposable-test
+"${compose[@]}" create --no-deps tests
-export TEST_S3_ENDPOINT_URL=http://127.0.0.1:19000
+runner="$("${compose[@]}" ps -aq tests)"
-export TEST_S3_ACCESS_KEY=disposable-test TEST_S3_SECRET_KEY=disposable-test-secret
+# CI's workspace is inside its job container, not on the Docker host. Copy only
-python - <<'PY'
+# test inputs; never bind-mount the workspace or copy .env, .git, or production secrets.
-import os, time
+COPYFILE_DISABLE=1 tar -cf - requirements-test.lock src tests sql \
-import boto3, psycopg
+  scripts/archive-old-weeks.sh scripts/list-archives.sh scripts/wait-test-infrastructure.py \
-from botocore.config import Config
+  | docker cp - "$runner:/app"
-for attempt in range(90):
+docker start --attach "$runner"
-    try:
+exit "$(docker inspect --format '{{.State.ExitCode}}' "$runner")"
-        with psycopg.connect(host=os.environ['TEST_PGHOST'], port=os.environ['TEST_PGPORT'], user='postgres', password='disposable-test', dbname='postgres', connect_timeout=2):
-            pass
-        boto3.client('s3', endpoint_url=os.environ['TEST_S3_ENDPOINT_URL'], aws_access_key_id='disposable-test', aws_secret_access_key='disposable-test-secret', region_name='us-east-1', config=Config(connect_timeout=2, read_timeout=2, retries={'max_attempts': 0}, s3={'addressing_style': 'path'})).list_buckets()
-        break
-    except Exception:
-        time.sleep(2)
-else:
-    raise SystemExit('disposable PostgreSQL/RustFS did not become ready')
-PY
-python -m pytest -q tests/integration

scripts/wait-test-infrastructure.py

@@ -0,0 +1,26 @@
+"""Wait only for explicitly configured disposable integration services."""
+import os
+import time
+
+import boto3
+from botocore.config import Config
+import psycopg
+
+
+for attempt in range(90):
+    try:
+        with psycopg.connect(host=os.environ['TEST_PGHOST'], port=os.environ['TEST_PGPORT'],
+                             user=os.environ['TEST_PGUSER'], password=os.environ['TEST_PGPASSWORD'],
+                             dbname='postgres', connect_timeout=2):
+            pass
+        boto3.client('s3', endpoint_url=os.environ['TEST_S3_ENDPOINT_URL'],
+                     aws_access_key_id=os.environ['TEST_S3_ACCESS_KEY'],
+                     aws_secret_access_key=os.environ['TEST_S3_SECRET_KEY'], region_name='us-east-1',
+                     config=Config(connect_timeout=2, read_timeout=2, retries={'max_attempts': 0},
+                                   s3={'addressing_style': 'path'})).list_buckets()
+        break
+    except Exception as error:
+        last_error = type(error).__name__
+        time.sleep(2)
+else:
+    raise SystemExit('Disposable PostgreSQL/RustFS did not become ready: ' + last_error)

tests/integration/compose.yaml

@@ -1,10 +1,10 @@
 # Disposable tests ONLY. Known test credentials are intentionally not production secrets.
+# No published ports: tests run on this isolated network, including in nested Gitea jobs.
 services:
   postgres:
     image: postgres:18.0-bookworm
     environment:
       POSTGRES_PASSWORD: disposable-test
-    ports: ["127.0.0.1:15432:5432"]
     tmpfs: ["/var/lib/postgresql"]
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
@@ -19,7 +19,25 @@ services:
       RUSTFS_CONSOLE_ENABLE: "false"
       RUSTFS_OBS_LOG_DIRECTORY: /logs
     command: ["/data"]
-    ports: ["127.0.0.1:19000:9000"]
     tmpfs:
       - /data:uid=10001,gid=10001
       - /logs:uid=10001,gid=10001
+  tests:
+    image: python:3.12.12-slim-bookworm
+    working_dir: /app
+    environment:
+      PYTHONPATH: /app/src
+      TEST_PGHOST: postgres
+      TEST_PGPORT: "5432"
+      TEST_PGUSER: postgres
+      TEST_PGPASSWORD: disposable-test
+      TEST_S3_ENDPOINT_URL: http://rustfs:9000
+      TEST_S3_ACCESS_KEY: disposable-test
+      TEST_S3_SECRET_KEY: disposable-test-secret
+    command:
+      - /bin/sh
+      - -c
+      - >-
+        pip install --disable-pip-version-check --require-hashes -r requirements-test.lock &&
+        python scripts/wait-test-infrastructure.py &&
+        python -m pytest -q