From 382da4351a250c17df338e901c5b156f87fadb52 Mon Sep 17 00:00:00 2001 From: Per Magnus Petersen Date: Thu, 17 Sep 2026 11:19:04 +0200 Subject: [PATCH] Prepare btcdata for Kubernetes deployment --- .gitea/workflows/build.yml | 81 ++++++ Makefile | 2 +- btcdata-deploy.md | 267 ++++++++++++++++++ deps.edn | 4 +- resources/queries.sql | 4 + .../btcdata/web/controllers/health.clj | 11 + src/clj/pmagnus/btcdata/web/routes/api.clj | 2 + .../btcdata/web/controllers/health_test.clj | 26 ++ 8 files changed, 394 insertions(+), 3 deletions(-) create mode 100644 .gitea/workflows/build.yml create mode 100644 btcdata-deploy.md create mode 100644 test/clj/pmagnus/btcdata/web/controllers/health_test.clj diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml new file mode 100644 index 0000000..617f6a2 --- /dev/null +++ b/.gitea/workflows/build.yml @@ -0,0 +1,81 @@ +name: build + +on: + push: + branches: [master] + tags: ['v*'] + +env: + REGISTRY: harbor.pmagnus.cloud + IMAGE: harbor.pmagnus.cloud/btcdata/btcdata + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + + - uses: DeLaGuardo/setup-clojure@13.2 + with: + cli: latest + + - name: Test + run: make test + + - name: Generate immutable tags + id: tags + shell: bash + run: | + set -euo pipefail + SHA=$(git rev-parse --short HEAD) + case "$GITHUB_REF" in + refs/tags/v*) VERSION="${GITHUB_REF_NAME#v}" ;; + *) VERSION="main-$(date -u +%Y%m%d%H%M%S)" ;; + esac + echo "primary=$IMAGE:$VERSION" >> "$GITHUB_OUTPUT" + echo "qualified=$IMAGE:$VERSION-$SHA" >> "$GITHUB_OUTPUT" + + # anvil is not a NetBird peer. Keep BuildKit on the runner host network + # until anvil can resolve and reach Harbor directly from job networks. + - uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host + + - uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.HARBOR_ROBOT_USER }} + password: ${{ secrets.HARBOR_ROBOT_TOKEN }} + + - uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.tags.outputs.primary }} + cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache + cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max + + - name: Wait for Harbor policy, then add SHA tag + env: + PRIMARY: ${{ steps.tags.outputs.primary }} + QUALIFIED: ${{ steps.tags.outputs.qualified }} + run: | + for attempt in $(seq 1 60); do + if docker buildx imagetools inspect "$PRIMARY" >/dev/null 2>&1; then + docker buildx imagetools create --tag "$QUALIFIED" "$PRIMARY" + exit 0 + fi + echo "waiting for Harbor scan/pull policy ($attempt/60)" + sleep 10 + done + echo "Harbor did not approve $PRIMARY within 10 minutes" >&2 + exit 1 + + - name: Summary + run: echo "pushed ${{ steps.tags.outputs.primary }} and ${{ steps.tags.outputs.qualified }}" diff --git a/Makefile b/Makefile index 3ff33ac..7516c05 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -include .env +-include .env export .PHONY: clean run repl test uberjar diff --git a/btcdata-deploy.md b/btcdata-deploy.md new file mode 100644 index 0000000..a49eef7 --- /dev/null +++ b/btcdata-deploy.md @@ -0,0 +1,267 @@ +# btcdata Kubernetes deployment plan + +## Goal and interpretation + +Deploy `btcdata` through the existing Flux-managed two-node k3s cluster (`srv1893094` and `citadel`), with **exactly one btcdata pod running at a time**. The pod may run on either node and Kubernetes may recreate it on the other node after failure. + +The current Compose instance is assumed to be `/home/pmp/apps/btcdata` on **citadel**. It must be stopped and verified stopped before the Kubernetes replica is increased from zero to one, because btcdata runs Binance, Kraken, Strike, and currency-rate writers. + +Production traffic remains internal: `btcprice` proxies all browser HTTP/WebSocket traffic to btcdata. btcdata does not need an Ingress. + +> A one-replica Deployment is active/passive scheduling, not a warm standby. It normally prevents duplicate writers, but it cannot strictly fence a partitioned old process. If strict single-writer guarantees are required, add PostgreSQL advisory-lock leader election before deployment. + +## Target architecture + +```text +btcprice pods (namespace btcprice) + -> http://btcdata.btcdata.svc.cluster.local:4101 + -> ClusterIP Service btcdata + -> one btcdata pod on srv1893094 OR citadel + -> PostgreSQL btcprod on citadel + -> Frankfurter Service in namespace btcdata + -> Binance, Kraken, and Strike APIs +``` + +Kubernetes objects and image automation belong in `magnus/deploy`; application image building belongs in this repository. Do not deploy the workload with Compose or direct `kubectl apply`. + +## Important limitations + +- PostgreSQL remains on citadel and is still a single point of failure. Moving the btcdata pod to `srv1893094` does not help if citadel or its database is unavailable. +- The database must be reachable from both k3s nodes. Use a stable reachable address, preferably citadel's NetBird address, rather than the LAN-only `192.168.50.195`, after verifying PostgreSQL listens on and permits that path. +- One replica causes a short interruption during upgrades and node failover. `btcprice` should become unready and reconnect when btcdata returns. +- Kubernetes `Deployment` with one replica is not absolute split-brain protection during a network partition. PostgreSQL advisory locking is the recommended follow-up. + +## Phase 1 — preflight and decisions + +1. Synchronize the GitOps checkout before editing: + + ```bash + git -C ../deploy switch main + git -C ../deploy pull --ff-only + ``` + +2. Confirm both nodes are Ready and amd64: + - `srv1893094` + - `citadel` +3. From a disposable pod pinned to each node, verify: + - PostgreSQL TCP connectivity; + - authentication to `btcprod` with a temporary, non-logged command; + - outbound HTTPS/WebSocket access to Binance, Kraken, Strike, and Harbor. +4. Decide and record the production JDBC hostname. Do not put the password in Git. +5. Confirm the legacy workload is the Compose project at `/home/pmp/apps/btcdata` on citadel and record: + - current image ID; + - effective non-secret environment; + - container names and Compose project name; + - a tested restart command for rollback. +6. Back up `btcprod` and test that the backup can be read/restored. +7. Confirm whether strict fencing is required. If yes, implement a PostgreSQL advisory lock so only the lock holder starts all writer components before proceeding. + +**Gate:** both nodes can reach PostgreSQL and required external APIs, a database backup exists, and the exact legacy stop/rollback commands are known. + +## Phase 2 — application readiness + +1. Run the existing test suite with `make test`. +2. Add or confirm tests for startup, shutdown, migrations, health, and disabled pollers. +3. Add a dependency-aware `/api/ready` endpoint if practical. It should check the database and required in-process state without performing external API calls. Keep `/api/health` process-only for liveness. +4. Ensure SIGTERM shuts down Undertow and all polling futures within the Kubernetes termination grace period. +5. Review database migrations for backward compatibility so rollback to the Compose image remains possible. +6. Keep runtime configuration in environment variables: + - `BTCDATA_PORT=4101` + - `JDBC_URL` from a Secret + - `STRIKE_API_KEY` from a Secret + - `CORS_ORIGIN` restricted to the actual frontend origins if direct browser access is ever enabled + - `FRANKFURTER_URL=http://frankfurter:8080` + - `BINANCE_ENABLED=true` + - `KRAKEN_ENABLED=true` + - `STRIKE_ENABLED=true` + +**Gate:** tests pass, termination is clean, and the production image can start with no embedded credentials. + +## Phase 3 — CI image publication + +Create `.gitea/workflows/build.yml`, following the established btcprice pipeline: + +1. Trigger on `main` and `v*` tags. +2. Run `make test`. +3. Build `linux/amd64` from the existing multi-stage `Dockerfile`. +4. Publish to `harbor.pmagnus.cloud/btcdata/btcdata` using the Harbor CI robot. +5. Publish immutable tags: + - `main-YYYYMMDDhhmmss` + - `main-YYYYMMDDhhmmss-` +6. Wait for Harbor scanning/policy approval before creating the companion tag. +7. Configure Harbor scan-on-push, vulnerability blocking, retention, and garbage collection consistently with btcprice. +8. Verify both k3s nodes can pull the approved digest with the existing pull credentials. + +Do not give CI a kubeconfig or a GitOps repository write key. Flux remains responsible for deployment. + +**Gate:** an approved immutable image is available and pullable from both nodes. + +## Phase 4 — secrets + +The deploy repository currently has no encrypted-secret mechanism. Until SOPS or an external secret operator is introduced: + +1. Create a `btcdata` namespace. +2. Provision a `btcdata-runtime` Secret out of band, using stdin or a temporary mode-`0600` file, with: + - `JDBC_URL` + - `STRIKE_API_KEY` +3. Never put decoded values, command output, or generated Secret YAML in either repository. +4. Document how to recreate the Secret after cluster recovery in a protected operations document. +5. Reference the Secret with `env[].valueFrom.secretKeyRef` in the Deployment. + +Prefer adding SOPS/age to Flux as a follow-up so secret recovery is declarative and encrypted. + +## Phase 5 — GitOps manifests, initially scaled to zero + +Create `../deploy/apps/btcdata/`: + +```text +apps/btcdata/ + namespace.yaml + deployment.yaml + service.yaml + frankfurter-deployment.yaml + frankfurter-service.yaml + pdb.yaml + kustomization.yaml +``` + +Update: + +- `../deploy/apps/kustomization.yaml` to include `btcdata`; +- `../deploy/infrastructure/image-automation/kustomization.yaml`; +- add `../deploy/infrastructure/image-automation/btcdata-image.yaml`. + +### btcdata Deployment + +Use: + +- namespace `btcdata`; +- image `harbor.pmagnus.cloud/btcdata/btcdata:`; +- Flux image-policy marker for `flux-system:btcdata`; +- **`replicas: 0` for the initial reconciliation**; +- `strategy.type: Recreate` to avoid overlapping writer pods during normal upgrades; +- node affinity allowing only `srv1893094` and `citadel`; +- container port `4101`, named `http`; +- `terminationGracePeriodSeconds: 30` or longer if shutdown testing requires it; +- startup and liveness probes on `/api/health`; +- readiness on `/api/ready` if added, otherwise `/api/health` temporarily; +- conservative initial requests/limits, then tune from observed usage; +- no host ports, host networking, persistent volume, or Ingress. + +A one-replica PDB with `minAvailable: 1` protects against accidental voluntary eviction, but it also blocks a normal drain. Document the controlled PDB override/removal required for planned node maintenance. + +### Service + +Create a selector-based ClusterIP Service named `btcdata` in namespace `btcdata`, port `4101` to the named `http` container port. + +### Frankfurter + +Deploy `lineofflight/frankfurter` behind a ClusterIP Service named `frankfurter` on port `8080`. Pin an immutable image version or digest rather than a floating tag. One replica is sufficient because btcdata retains rates in PostgreSQL and can retry temporary failures. + +### Image automation + +Add an `ImageRepository` and `ImagePolicy` matching only plain main tags: + +```yaml +filterTags: + pattern: '^main-(?P[0-9]{14})$' + extract: '$ts' +policy: + numerical: + order: asc +``` + +Validate all kustomizations locally. Commit and push to `magnus/deploy`; do not manually apply the workload. Confirm Flux creates the namespace, Service, Frankfurter, and a zero-replica btcdata Deployment. + +**Gate:** Flux is Ready, Frankfurter is healthy, the btcdata Service exists, and no btcdata Kubernetes pod is running. + +## Phase 6 — controlled cutover + +This sequence deliberately includes a short outage to guarantee that the old and new writers do not intentionally overlap. + +1. Pause Flux btcdata image automation for the cutover window, or verify that no new image can be selected during the procedure. +2. On citadel, stop **only** the legacy btcdata Compose project in `/home/pmp/apps/btcdata`. + - Do not stop PostgreSQL. + - Do not remove `btc-network`, images, volumes, or deployment files. +3. Verify before Kubernetes startup: + - the btcdata Compose container is stopped; + - port `4102` is no longer listening for btcdata; + - no legacy Java btcdata process remains; + - database write timestamps stop advancing except for expected external writers. +4. In `magnus/deploy`, change the btcdata Deployment from `replicas: 0` to **`replicas: 1`**, commit, and push. +5. Let Flux reconcile. Do not use `kubectl scale`, because Git would revert it. +6. Confirm exactly one pod is Running and Ready on either allowed node. +7. Check startup logs for successful migrations, database connection, Binance WebSocket, Kraken pollers, Frankfurter rates, and Strike ticker. +8. From disposable pods pinned to each node, test the new Service FQDN: + - `/api/health`; + - `/api/price/latest`; + - `/api/kraken-minute` and `/api/kraken-hour`; + - `/api/price/ws` and `/api/strike/ws` WebSockets. +9. Confirm database rows advance once at the expected cadence and no duplicate-writer symptoms appear. + +**Gate:** one Kubernetes pod is healthy, data writes are correct, and both nodes can reach its ClusterIP Service. + +## Phase 7 — switch btcprice to Kubernetes btcdata + +Modify `../deploy/apps/btcprice/` in one GitOps change: + +1. Change the Deployment environment to: + + ```text + BTCDATA_URL=http://btcdata.btcdata.svc.cluster.local:4101 + ``` + +2. Remove the legacy selectorless `btcdata` Service from the `btcprice` namespace. +3. Remove `btcdata-endpointslice.yaml` and its kustomization entry. +4. Keep btcprice's existing two-replica scheduling, readiness behavior, and WebSocket ingress timeouts. +5. Validate both application kustomizations and push the change to `magnus/deploy`. +6. Let Flux roll out btcprice and verify: + - both btcprice pods become Ready; + - `/api/ready` succeeds; + - chart/API requests return current data; + - `/ws/price` and `/ws/strike` reconnect and continue updating; + - `https://btcprice.k8s.pmagnus.cloud` and `https://btc.pmagnus.cloud` work. + +After successful validation, re-enable btcdata image automation. + +## Phase 8 — failover and acceptance tests + +1. Delete the btcdata pod and verify Kubernetes creates one replacement, never a steady state of two Ready replicas. +2. Cordon the node currently hosting btcdata, perform a controlled pod deletion, and verify it starts on the other node. Uncordon afterward. +3. Confirm btcprice becomes temporarily unready during failover and recovers without manual changes. +4. Verify WebSocket clients reconnect and fresh price data resumes. +5. Perform an automated image update and confirm `Recreate` stops the old pod before starting the new pod. +6. Review logs and database data for duplicate Binance/Kraken/Strike writes. +7. Observe CPU, memory, restart count, startup time, and failover duration; tune resources and probes. +8. Keep the stopped Compose files and image for a defined rollback window, at least 24 hours. + +**Acceptance criteria:** + +- Flux reports all resources Ready. +- Exactly one btcdata pod runs across the two eligible nodes. +- btcprice uses only Kubernetes DNS for btcdata. +- HTTP and WebSocket functions work through both production hostnames. +- A source push can reach production through Gitea, Harbor, and Flux without SSH deployment or manual `kubectl apply`. +- The legacy Compose btcdata remains stopped. + +## Rollback + +Rollback must also preserve the single-writer rule: + +1. Change the Kubernetes btcdata Deployment to `replicas: 0` in `magnus/deploy`, commit, and wait for Flux. +2. Verify no btcdata pod or process remains in Kubernetes. +3. Restore btcprice's selectorless Service and EndpointSlice pointing to citadel, and restore `BTCDATA_URL=http://btcdata:4102` through GitOps. +4. Restart the recorded btcdata Compose service on citadel. +5. Verify health, HTTP/WebSocket data, and database write cadence. +6. Keep the failed Kubernetes image and logs for diagnosis; do not rewrite Git history or delete Harbor evidence. + +For an application-only rollback, pause image automation, commit the previous approved immutable image tag to `magnus/deploy`, and let Flux perform the `Recreate` rollout. + +## Follow-up improvements + +1. Add PostgreSQL advisory-lock leader election, then consider two btcdata pods with one writer leader and one standby/API replica. +2. Move PostgreSQL to a highly available service reachable independently of citadel. +3. Add SOPS/age or an external secret operator. +4. Add dependency-aware readiness and metrics/alerts for stale prices, failed pollers, database errors, and leader status. +5. Update this repository's `README.md` and `CLAUDE.md`, btcprice documentation, and system infrastructure documentation after the rollback window closes. +6. Retire `deploy-citadel.sh` and legacy Compose deployment files only after rollback is no longer required. diff --git a/deps.edn b/deps.edn index 2157611..b9e9722 100644 --- a/deps.edn +++ b/deps.edn @@ -46,10 +46,10 @@ com.lambdaisland/classpath {:mvn/version "0.4.44"} ring/ring-devel {:mvn/version "1.15.3"}}} - :test {:extra-paths ["env/test/resources"] + :test {:extra-paths ["test/clj" "env/test/resources"] :extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" :git/sha "dfb30dd"} peridot/peridot {:mvn/version "0.5.4"} - clj-commons/byte-streams {:mvn/version "0.3.4"}} + org.clj-commons/byte-streams {:mvn/version "0.3.4"}} :main-opts ["-m" "cognitect.test-runner"] :exec-fn cognitect.test-runner.api/test} diff --git a/resources/queries.sql b/resources/queries.sql index f9aaa50..364cf96 100644 --- a/resources/queries.sql +++ b/resources/queries.sql @@ -1,5 +1,9 @@ -- queries for btcdata +-- :name check-database :? :1 +-- :doc Verify that PostgreSQL accepts queries +SELECT 1 AS healthy + -- :name insert-binance-price! :! :n -- :doc Insert a Binance BTC price INSERT INTO binance_price (price) VALUES (:price) diff --git a/src/clj/pmagnus/btcdata/web/controllers/health.clj b/src/clj/pmagnus/btcdata/web/controllers/health.clj index fe59f94..a9df489 100644 --- a/src/clj/pmagnus/btcdata/web/controllers/health.clj +++ b/src/clj/pmagnus/btcdata/web/controllers/health.clj @@ -1,5 +1,6 @@ (ns pmagnus.btcdata.web.controllers.health (:require + [clojure.tools.logging :as log] [ring.util.http-response :as response]) (:import [java.lang.management ManagementFactory])) @@ -9,3 +10,13 @@ {:time (str (java.time.Instant/now)) :up-time (.. ManagementFactory getRuntimeMXBean getUptime) :app {:status "up"}})) + +(defn readiness-check! + "Report readiness only while the application can query PostgreSQL." + [query-fn _req] + (try + (query-fn :check-database {}) + (response/ok {:status "ready"}) + (catch Exception e + (log/warn e "Database readiness check failed") + (response/service-unavailable {:status "not ready"})))) diff --git a/src/clj/pmagnus/btcdata/web/routes/api.clj b/src/clj/pmagnus/btcdata/web/routes/api.clj index 85a4d71..c482ce0 100644 --- a/src/clj/pmagnus/btcdata/web/routes/api.clj +++ b/src/clj/pmagnus/btcdata/web/routes/api.clj @@ -180,6 +180,8 @@ :handler (swagger/create-swagger-handler)}}] ["/health" {:get health/healthcheck!}] + ["/ready" + {:get (fn [req] (health/readiness-check! (:query-fn opts) req))}] ["/price/ws" {:get (fn [req] (ws-price-handler opts req)) :no-doc true diff --git a/test/clj/pmagnus/btcdata/web/controllers/health_test.clj b/test/clj/pmagnus/btcdata/web/controllers/health_test.clj new file mode 100644 index 0000000..64a8ed0 --- /dev/null +++ b/test/clj/pmagnus/btcdata/web/controllers/health_test.clj @@ -0,0 +1,26 @@ +(ns pmagnus.btcdata.web.controllers.health-test + (:require + [clojure.test :refer [deftest is testing]] + [pmagnus.btcdata.web.controllers.health :as health])) + +(deftest healthcheck-is-process-only + (let [response (health/healthcheck! {})] + (is (= 200 (:status response))) + (is (= "up" (get-in response [:body :app :status]))))) + +(deftest readiness-check-tests-database + (testing "ready when the database query succeeds" + (let [called (atom nil) + query-fn (fn [query params] + (reset! called [query params]) + {:healthy 1}) + response (health/readiness-check! query-fn {})] + (is (= 200 (:status response))) + (is (= {:status "ready"} (:body response))) + (is (= [:check-database {}] @called)))) + + (testing "not ready when the database query fails" + (let [query-fn (fn [_ _] (throw (ex-info "database unavailable" {}))) + response (health/readiness-check! query-fn {})] + (is (= 503 (:status response))) + (is (= {:status "not ready"} (:body response))))))