Prepare btcdata for Kubernetes deployment

.gitea/workflows/build.yml

@@ -0,0 +1,81 @@
+name: build
+
+on:
+  push:
+    branches: [master]
+    tags: ['v*']
+
+env:
+  REGISTRY: harbor.pmagnus.cloud
+  IMAGE: harbor.pmagnus.cloud/btcdata/btcdata
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - uses: DeLaGuardo/setup-clojure@13.2
+        with:
+          cli: latest
+
+      - name: Test
+        run: make test
+
+      - name: Generate immutable tags
+        id: tags
+        shell: bash
+        run: |
+          set -euo pipefail
+          SHA=$(git rev-parse --short HEAD)
+          case "$GITHUB_REF" in
+            refs/tags/v*) VERSION="${GITHUB_REF_NAME#v}" ;;
+            *) VERSION="main-$(date -u +%Y%m%d%H%M%S)" ;;
+          esac
+          echo "primary=$IMAGE:$VERSION" >> "$GITHUB_OUTPUT"
+          echo "qualified=$IMAGE:$VERSION-$SHA" >> "$GITHUB_OUTPUT"
+
+      # anvil is not a NetBird peer. Keep BuildKit on the runner host network
+      # until anvil can resolve and reach Harbor directly from job networks.
+      - uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.HARBOR_ROBOT_USER }}
+          password: ${{ secrets.HARBOR_ROBOT_TOKEN }}
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.tags.outputs.primary }}
+          cache-from: type=registry,ref=${{ env.IMAGE }}:buildcache
+          cache-to: type=registry,ref=${{ env.IMAGE }}:buildcache,mode=max
+
+      - name: Wait for Harbor policy, then add SHA tag
+        env:
+          PRIMARY: ${{ steps.tags.outputs.primary }}
+          QUALIFIED: ${{ steps.tags.outputs.qualified }}
+        run: |
+          for attempt in $(seq 1 60); do
+            if docker buildx imagetools inspect "$PRIMARY" >/dev/null 2>&1; then
+              docker buildx imagetools create --tag "$QUALIFIED" "$PRIMARY"
+              exit 0
+            fi
+            echo "waiting for Harbor scan/pull policy ($attempt/60)"
+            sleep 10
+          done
+          echo "Harbor did not approve $PRIMARY within 10 minutes" >&2
+          exit 1
+
+      - name: Summary
+        run: echo "pushed ${{ steps.tags.outputs.primary }} and ${{ steps.tags.outputs.qualified }}"

Makefile

@@ -1,4 +1,4 @@
-include .env
+-include .env
 export
 
 .PHONY: clean run repl test uberjar

btcdata-deploy.md

@@ -0,0 +1,267 @@
+# btcdata Kubernetes deployment plan
+
+## Goal and interpretation
+
+Deploy `btcdata` through the existing Flux-managed two-node k3s cluster (`srv1893094` and `citadel`), with **exactly one btcdata pod running at a time**. The pod may run on either node and Kubernetes may recreate it on the other node after failure.
+
+The current Compose instance is assumed to be `/home/pmp/apps/btcdata` on **citadel**. It must be stopped and verified stopped before the Kubernetes replica is increased from zero to one, because btcdata runs Binance, Kraken, Strike, and currency-rate writers.
+
+Production traffic remains internal: `btcprice` proxies all browser HTTP/WebSocket traffic to btcdata. btcdata does not need an Ingress.
+
+> A one-replica Deployment is active/passive scheduling, not a warm standby. It normally prevents duplicate writers, but it cannot strictly fence a partitioned old process. If strict single-writer guarantees are required, add PostgreSQL advisory-lock leader election before deployment.
+
+## Target architecture
+
+```text
+btcprice pods (namespace btcprice)
+  -> http://btcdata.btcdata.svc.cluster.local:4101
+  -> ClusterIP Service btcdata
+  -> one btcdata pod on srv1893094 OR citadel
+       -> PostgreSQL btcprod on citadel
+       -> Frankfurter Service in namespace btcdata
+       -> Binance, Kraken, and Strike APIs
+```
+
+Kubernetes objects and image automation belong in `magnus/deploy`; application image building belongs in this repository. Do not deploy the workload with Compose or direct `kubectl apply`.
+
+## Important limitations
+
+- PostgreSQL remains on citadel and is still a single point of failure. Moving the btcdata pod to `srv1893094` does not help if citadel or its database is unavailable.
+- The database must be reachable from both k3s nodes. Use a stable reachable address, preferably citadel's NetBird address, rather than the LAN-only `192.168.50.195`, after verifying PostgreSQL listens on and permits that path.
+- One replica causes a short interruption during upgrades and node failover. `btcprice` should become unready and reconnect when btcdata returns.
+- Kubernetes `Deployment` with one replica is not absolute split-brain protection during a network partition. PostgreSQL advisory locking is the recommended follow-up.
+
+## Phase 1 — preflight and decisions
+
+1. Synchronize the GitOps checkout before editing:
+
+   ```bash
+   git -C ../deploy switch main
+   git -C ../deploy pull --ff-only
+   ```
+
+2. Confirm both nodes are Ready and amd64:
+   - `srv1893094`
+   - `citadel`
+3. From a disposable pod pinned to each node, verify:
+   - PostgreSQL TCP connectivity;
+   - authentication to `btcprod` with a temporary, non-logged command;
+   - outbound HTTPS/WebSocket access to Binance, Kraken, Strike, and Harbor.
+4. Decide and record the production JDBC hostname. Do not put the password in Git.
+5. Confirm the legacy workload is the Compose project at `/home/pmp/apps/btcdata` on citadel and record:
+   - current image ID;
+   - effective non-secret environment;
+   - container names and Compose project name;
+   - a tested restart command for rollback.
+6. Back up `btcprod` and test that the backup can be read/restored.
+7. Confirm whether strict fencing is required. If yes, implement a PostgreSQL advisory lock so only the lock holder starts all writer components before proceeding.
+
+**Gate:** both nodes can reach PostgreSQL and required external APIs, a database backup exists, and the exact legacy stop/rollback commands are known.
+
+## Phase 2 — application readiness
+
+1. Run the existing test suite with `make test`.
+2. Add or confirm tests for startup, shutdown, migrations, health, and disabled pollers.
+3. Add a dependency-aware `/api/ready` endpoint if practical. It should check the database and required in-process state without performing external API calls. Keep `/api/health` process-only for liveness.
+4. Ensure SIGTERM shuts down Undertow and all polling futures within the Kubernetes termination grace period.
+5. Review database migrations for backward compatibility so rollback to the Compose image remains possible.
+6. Keep runtime configuration in environment variables:
+   - `BTCDATA_PORT=4101`
+   - `JDBC_URL` from a Secret
+   - `STRIKE_API_KEY` from a Secret
+   - `CORS_ORIGIN` restricted to the actual frontend origins if direct browser access is ever enabled
+   - `FRANKFURTER_URL=http://frankfurter:8080`
+   - `BINANCE_ENABLED=true`
+   - `KRAKEN_ENABLED=true`
+   - `STRIKE_ENABLED=true`
+
+**Gate:** tests pass, termination is clean, and the production image can start with no embedded credentials.
+
+## Phase 3 — CI image publication
+
+Create `.gitea/workflows/build.yml`, following the established btcprice pipeline:
+
+1. Trigger on `main` and `v*` tags.
+2. Run `make test`.
+3. Build `linux/amd64` from the existing multi-stage `Dockerfile`.
+4. Publish to `harbor.pmagnus.cloud/btcdata/btcdata` using the Harbor CI robot.
+5. Publish immutable tags:
+   - `main-YYYYMMDDhhmmss`
+   - `main-YYYYMMDDhhmmss-<short-sha>`
+6. Wait for Harbor scanning/policy approval before creating the companion tag.
+7. Configure Harbor scan-on-push, vulnerability blocking, retention, and garbage collection consistently with btcprice.
+8. Verify both k3s nodes can pull the approved digest with the existing pull credentials.
+
+Do not give CI a kubeconfig or a GitOps repository write key. Flux remains responsible for deployment.
+
+**Gate:** an approved immutable image is available and pullable from both nodes.
+
+## Phase 4 — secrets
+
+The deploy repository currently has no encrypted-secret mechanism. Until SOPS or an external secret operator is introduced:
+
+1. Create a `btcdata` namespace.
+2. Provision a `btcdata-runtime` Secret out of band, using stdin or a temporary mode-`0600` file, with:
+   - `JDBC_URL`
+   - `STRIKE_API_KEY`
+3. Never put decoded values, command output, or generated Secret YAML in either repository.
+4. Document how to recreate the Secret after cluster recovery in a protected operations document.
+5. Reference the Secret with `env[].valueFrom.secretKeyRef` in the Deployment.
+
+Prefer adding SOPS/age to Flux as a follow-up so secret recovery is declarative and encrypted.
+
+## Phase 5 — GitOps manifests, initially scaled to zero
+
+Create `../deploy/apps/btcdata/`:
+
+```text
+apps/btcdata/
+  namespace.yaml
+  deployment.yaml
+  service.yaml
+  frankfurter-deployment.yaml
+  frankfurter-service.yaml
+  pdb.yaml
+  kustomization.yaml
+```
+
+Update:
+
+- `../deploy/apps/kustomization.yaml` to include `btcdata`;
+- `../deploy/infrastructure/image-automation/kustomization.yaml`;
+- add `../deploy/infrastructure/image-automation/btcdata-image.yaml`.
+
+### btcdata Deployment
+
+Use:
+
+- namespace `btcdata`;
+- image `harbor.pmagnus.cloud/btcdata/btcdata:<approved-main-tag>`;
+- Flux image-policy marker for `flux-system:btcdata`;
+- **`replicas: 0` for the initial reconciliation**;
+- `strategy.type: Recreate` to avoid overlapping writer pods during normal upgrades;
+- node affinity allowing only `srv1893094` and `citadel`;
+- container port `4101`, named `http`;
+- `terminationGracePeriodSeconds: 30` or longer if shutdown testing requires it;
+- startup and liveness probes on `/api/health`;
+- readiness on `/api/ready` if added, otherwise `/api/health` temporarily;
+- conservative initial requests/limits, then tune from observed usage;
+- no host ports, host networking, persistent volume, or Ingress.
+
+A one-replica PDB with `minAvailable: 1` protects against accidental voluntary eviction, but it also blocks a normal drain. Document the controlled PDB override/removal required for planned node maintenance.
+
+### Service
+
+Create a selector-based ClusterIP Service named `btcdata` in namespace `btcdata`, port `4101` to the named `http` container port.
+
+### Frankfurter
+
+Deploy `lineofflight/frankfurter` behind a ClusterIP Service named `frankfurter` on port `8080`. Pin an immutable image version or digest rather than a floating tag. One replica is sufficient because btcdata retains rates in PostgreSQL and can retry temporary failures.
+
+### Image automation
+
+Add an `ImageRepository` and `ImagePolicy` matching only plain main tags:
+
+```yaml
+filterTags:
+  pattern: '^main-(?P<ts>[0-9]{14})$'
+  extract: '$ts'
+policy:
+  numerical:
+    order: asc
+```
+
+Validate all kustomizations locally. Commit and push to `magnus/deploy`; do not manually apply the workload. Confirm Flux creates the namespace, Service, Frankfurter, and a zero-replica btcdata Deployment.
+
+**Gate:** Flux is Ready, Frankfurter is healthy, the btcdata Service exists, and no btcdata Kubernetes pod is running.
+
+## Phase 6 — controlled cutover
+
+This sequence deliberately includes a short outage to guarantee that the old and new writers do not intentionally overlap.
+
+1. Pause Flux btcdata image automation for the cutover window, or verify that no new image can be selected during the procedure.
+2. On citadel, stop **only** the legacy btcdata Compose project in `/home/pmp/apps/btcdata`.
+   - Do not stop PostgreSQL.
+   - Do not remove `btc-network`, images, volumes, or deployment files.
+3. Verify before Kubernetes startup:
+   - the btcdata Compose container is stopped;
+   - port `4102` is no longer listening for btcdata;
+   - no legacy Java btcdata process remains;
+   - database write timestamps stop advancing except for expected external writers.
+4. In `magnus/deploy`, change the btcdata Deployment from `replicas: 0` to **`replicas: 1`**, commit, and push.
+5. Let Flux reconcile. Do not use `kubectl scale`, because Git would revert it.
+6. Confirm exactly one pod is Running and Ready on either allowed node.
+7. Check startup logs for successful migrations, database connection, Binance WebSocket, Kraken pollers, Frankfurter rates, and Strike ticker.
+8. From disposable pods pinned to each node, test the new Service FQDN:
+   - `/api/health`;
+   - `/api/price/latest`;
+   - `/api/kraken-minute` and `/api/kraken-hour`;
+   - `/api/price/ws` and `/api/strike/ws` WebSockets.
+9. Confirm database rows advance once at the expected cadence and no duplicate-writer symptoms appear.
+
+**Gate:** one Kubernetes pod is healthy, data writes are correct, and both nodes can reach its ClusterIP Service.
+
+## Phase 7 — switch btcprice to Kubernetes btcdata
+
+Modify `../deploy/apps/btcprice/` in one GitOps change:
+
+1. Change the Deployment environment to:
+
+   ```text
+   BTCDATA_URL=http://btcdata.btcdata.svc.cluster.local:4101
+   ```
+
+2. Remove the legacy selectorless `btcdata` Service from the `btcprice` namespace.
+3. Remove `btcdata-endpointslice.yaml` and its kustomization entry.
+4. Keep btcprice's existing two-replica scheduling, readiness behavior, and WebSocket ingress timeouts.
+5. Validate both application kustomizations and push the change to `magnus/deploy`.
+6. Let Flux roll out btcprice and verify:
+   - both btcprice pods become Ready;
+   - `/api/ready` succeeds;
+   - chart/API requests return current data;
+   - `/ws/price` and `/ws/strike` reconnect and continue updating;
+   - `https://btcprice.k8s.pmagnus.cloud` and `https://btc.pmagnus.cloud` work.
+
+After successful validation, re-enable btcdata image automation.
+
+## Phase 8 — failover and acceptance tests
+
+1. Delete the btcdata pod and verify Kubernetes creates one replacement, never a steady state of two Ready replicas.
+2. Cordon the node currently hosting btcdata, perform a controlled pod deletion, and verify it starts on the other node. Uncordon afterward.
+3. Confirm btcprice becomes temporarily unready during failover and recovers without manual changes.
+4. Verify WebSocket clients reconnect and fresh price data resumes.
+5. Perform an automated image update and confirm `Recreate` stops the old pod before starting the new pod.
+6. Review logs and database data for duplicate Binance/Kraken/Strike writes.
+7. Observe CPU, memory, restart count, startup time, and failover duration; tune resources and probes.
+8. Keep the stopped Compose files and image for a defined rollback window, at least 24 hours.
+
+**Acceptance criteria:**
+
+- Flux reports all resources Ready.
+- Exactly one btcdata pod runs across the two eligible nodes.
+- btcprice uses only Kubernetes DNS for btcdata.
+- HTTP and WebSocket functions work through both production hostnames.
+- A source push can reach production through Gitea, Harbor, and Flux without SSH deployment or manual `kubectl apply`.
+- The legacy Compose btcdata remains stopped.
+
+## Rollback
+
+Rollback must also preserve the single-writer rule:
+
+1. Change the Kubernetes btcdata Deployment to `replicas: 0` in `magnus/deploy`, commit, and wait for Flux.
+2. Verify no btcdata pod or process remains in Kubernetes.
+3. Restore btcprice's selectorless Service and EndpointSlice pointing to citadel, and restore `BTCDATA_URL=http://btcdata:4102` through GitOps.
+4. Restart the recorded btcdata Compose service on citadel.
+5. Verify health, HTTP/WebSocket data, and database write cadence.
+6. Keep the failed Kubernetes image and logs for diagnosis; do not rewrite Git history or delete Harbor evidence.
+
+For an application-only rollback, pause image automation, commit the previous approved immutable image tag to `magnus/deploy`, and let Flux perform the `Recreate` rollout.
+
+## Follow-up improvements
+
+1. Add PostgreSQL advisory-lock leader election, then consider two btcdata pods with one writer leader and one standby/API replica.
+2. Move PostgreSQL to a highly available service reachable independently of citadel.
+3. Add SOPS/age or an external secret operator.
+4. Add dependency-aware readiness and metrics/alerts for stale prices, failed pollers, database errors, and leader status.
+5. Update this repository's `README.md` and `CLAUDE.md`, btcprice documentation, and system infrastructure documentation after the rollback window closes.
+6. Retire `deploy-citadel.sh` and legacy Compose deployment files only after rollback is no longer required.

deps.edn

@@ -46,10 +46,10 @@
                       com.lambdaisland/classpath        {:mvn/version "0.4.44"}
                       ring/ring-devel                   {:mvn/version "1.15.3"}}}
 
-  :test {:extra-paths ["env/test/resources"]
+  :test {:extra-paths ["test/clj" "env/test/resources"]
          :extra-deps  {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" :git/sha "dfb30dd"}
                        peridot/peridot                      {:mvn/version "0.5.4"}
-                       clj-commons/byte-streams              {:mvn/version "0.3.4"}}
+                       org.clj-commons/byte-streams          {:mvn/version "0.3.4"}}
          :main-opts   ["-m" "cognitect.test-runner"]
          :exec-fn     cognitect.test-runner.api/test}
 

resources/queries.sql

@@ -1,5 +1,9 @@
 -- queries for btcdata
 
+-- :name check-database :? :1
+-- :doc Verify that PostgreSQL accepts queries
+SELECT 1 AS healthy
+
 -- :name insert-binance-price! :! :n
 -- :doc Insert a Binance BTC price
 INSERT INTO binance_price (price) VALUES (:price)

src/clj/pmagnus/btcdata/web/controllers/health.clj

@@ -1,5 +1,6 @@
 (ns pmagnus.btcdata.web.controllers.health
   (:require
+   [clojure.tools.logging :as log]
    [ring.util.http-response :as response])
   (:import
    [java.lang.management ManagementFactory]))
@@ -9,3 +10,13 @@
    {:time    (str (java.time.Instant/now))
     :up-time (.. ManagementFactory getRuntimeMXBean getUptime)
     :app     {:status "up"}}))
+
+(defn readiness-check!
+  "Report readiness only while the application can query PostgreSQL."
+  [query-fn _req]
+  (try
+    (query-fn :check-database {})
+    (response/ok {:status "ready"})
+    (catch Exception e
+      (log/warn e "Database readiness check failed")
+      (response/service-unavailable {:status "not ready"}))))

src/clj/pmagnus/btcdata/web/routes/api.clj

@@ -180,6 +180,8 @@
            :handler (swagger/create-swagger-handler)}}]
    ["/health"
     {:get health/healthcheck!}]
+   ["/ready"
+    {:get (fn [req] (health/readiness-check! (:query-fn opts) req))}]
    ["/price/ws"
     {:get        (fn [req] (ws-price-handler opts req))
      :no-doc     true

test/clj/pmagnus/btcdata/web/controllers/health_test.clj

@@ -0,0 +1,26 @@
+(ns pmagnus.btcdata.web.controllers.health-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [pmagnus.btcdata.web.controllers.health :as health]))
+
+(deftest healthcheck-is-process-only
+  (let [response (health/healthcheck! {})]
+    (is (= 200 (:status response)))
+    (is (= "up" (get-in response [:body :app :status])))))
+
+(deftest readiness-check-tests-database
+  (testing "ready when the database query succeeds"
+    (let [called   (atom nil)
+          query-fn (fn [query params]
+                     (reset! called [query params])
+                     {:healthy 1})
+          response (health/readiness-check! query-fn {})]
+      (is (= 200 (:status response)))
+      (is (= {:status "ready"} (:body response)))
+      (is (= [:check-database {}] @called))))
+
+  (testing "not ready when the database query fails"
+    (let [query-fn (fn [_ _] (throw (ex-info "database unavailable" {})))
+          response (health/readiness-check! query-fn {})]
+      (is (= 503 (:status response)))
+      (is (= {:status "not ready"} (:body response))))))