From 91f0367337fdb9e853c6132e5d8a5f08e3d1d2c7 Mon Sep 17 00:00:00 2001
From: Per Magnus Petersen <magnus@pmagnus.dk>
Date: Fri, 9 Oct 2026 14:58:53 +0200
Subject: [PATCH] Restart btcdata when its fenced database session closes

---
 btcdata-deploy.md                             | 24 +++++++++++++++++++
 .../btcdata/web/controllers/health.clj        | 18 +++++++++-----
 src/clj/pmagnus/btcdata/web/routes/api.clj    |  2 +-
 src/clj/pmagnus/btcdata/writer.clj            |  9 +++++++
 .../btcdata/web/controllers/health_test.clj   | 14 +++++++++--
 test/clj/pmagnus/btcdata/writer_test.clj      |  9 +++++++
 6 files changed, 67 insertions(+), 9 deletions(-)

diff --git a/btcdata-deploy.md b/btcdata-deploy.md
index 66bd968..47f6cc5 100644
--- a/btcdata-deploy.md
+++ b/btcdata-deploy.md
@@ -1,5 +1,29 @@
 # btcdata Kubernetes deployment plan
 
+## 2026-10-09 incident: closed writer session never restarted
+
+The citadel pod was Running but not Ready, and both btcprice pods were unready.
+All writer queries failed with `Connection is closed`. Readiness correctly
+returned 503, but `/api/health` always returned 200, so Kubernetes did not restart
+the process. The initial connection-loss cause was not established from the
+available logs. Fenced connections deliberately never reconnect in place.
+
+Recovery deleted only the failed pod (normal graceful deletion); its Deployment
+recreated it, and btcdata and both btcprice pods became Ready. No lock was
+bypassed or database backend terminated for this incident.
+
+The source fix makes `/api/health` return 503 when the writer session is closed
+(or absent). This checks local JDBC state only: no SQL, external APIs, or Binance
+freshness are used in liveness. Existing probes will then restart an irrecoverable
+writer after three failed checks. Readiness retains database/freshness checks.
+Unit tests passed: 21 tests, 63 assertions. PostgreSQL integration tests were
+not run because the local Docker daemon was unavailable.
+
+The recovered pod still runs `main-20260930155338`, not the new source changes.
+Flux apps reconciliation is blocked on infrastructure health checks for image
+repositories; building/publishing and a successful Flux rollout remain necessary
+to activate the liveness and keepalive source fixes.
+
 ## 2026-10-07 incident: orphaned writer session
 
 The `sejs/btcdata` pod on citadel entered CrashLoopBackOff with
diff --git a/src/clj/pmagnus/btcdata/web/controllers/health.clj b/src/clj/pmagnus/btcdata/web/controllers/health.clj
index 2f21a98..bd29fb7 100644
--- a/src/clj/pmagnus/btcdata/web/controllers/health.clj
+++ b/src/clj/pmagnus/btcdata/web/controllers/health.clj
@@ -2,15 +2,21 @@
   (:require
    [clojure.tools.logging :as log]
    [ring.util.http-response :as response]
-   [pmagnus.btcdata.ws.binance :as binance])
+   [pmagnus.btcdata.ws.binance :as binance]
+   [pmagnus.btcdata.writer :as writer])
   (:import
    [java.lang.management ManagementFactory]))
 
-(defn healthcheck! [_req]
-  (response/ok
-   {:time    (str (java.time.Instant/now))
-    :up-time (.. ManagementFactory getRuntimeMXBean getUptime)
-    :app     {:status "up"}}))
+(defn healthcheck!
+  "No remote dependency checks. A permanently closed writer session needs restart."
+  [query-fn _req]
+  (if (writer/session-live? query-fn)
+    (response/ok
+     {:time    (str (java.time.Instant/now))
+      :up-time (.. ManagementFactory getRuntimeMXBean getUptime)
+      :app     {:status "up"}})
+    (response/service-unavailable {:status "restart required"
+                                   :reason "writer session closed"})))
 
 (defn readiness-check!
   "Require the lock-owning database session and a fresh upstream stream."
diff --git a/src/clj/pmagnus/btcdata/web/routes/api.clj b/src/clj/pmagnus/btcdata/web/routes/api.clj
index 4d6bda1..a1352af 100644
--- a/src/clj/pmagnus/btcdata/web/routes/api.clj
+++ b/src/clj/pmagnus/btcdata/web/routes/api.clj
@@ -180,7 +180,7 @@
            :swagger {:info {:title "btcdata API"}}
            :handler (swagger/create-swagger-handler)}}]
    ["/health"
-    {:get health/healthcheck!}]
+    {:get (fn [req] (health/healthcheck! (:query-fn opts) req))}]
    ["/ready"
     {:get (fn [req] (health/readiness-check! (:query-fn opts) (:binance opts) req))}]
    ["/price/ws"
diff --git a/src/clj/pmagnus/btcdata/writer.clj b/src/clj/pmagnus/btcdata/writer.clj
index 14fd3b3..fe0d930 100644
--- a/src/clj/pmagnus/btcdata/writer.clj
+++ b/src/clj/pmagnus/btcdata/writer.clj
@@ -16,6 +16,15 @@
                "SET tcp_user_timeout = 90000"]]
     (jdbc/execute-one! connection [sql])))
 
+(defn session-live?
+  "Local-only liveness check; a closed fenced session can never recover in place."
+  [query-fn]
+  (if-let [connection (:connection (meta query-fn))]
+    (try
+      (not (.isClosed ^java.sql.Connection connection))
+      (catch Exception _ false))
+    false))
+
 (defn guarded-query [connection query-fn]
   (fn [query params]
     (locking connection
diff --git a/test/clj/pmagnus/btcdata/web/controllers/health_test.clj b/test/clj/pmagnus/btcdata/web/controllers/health_test.clj
index c855ebc..82085b4 100644
--- a/test/clj/pmagnus/btcdata/web/controllers/health_test.clj
+++ b/test/clj/pmagnus/btcdata/web/controllers/health_test.clj
@@ -4,8 +4,18 @@
 
 (def disabled-stream {:state (atom {:enabled? false}) :latest-price (atom nil)})
 
-(deftest healthcheck-is-process-only
-  (is (= 200 (:status (health/healthcheck! {})))))
+(deftest liveness-check-is-local-and-detects-lost-writer
+  (let [closed? (atom false)
+        connection (proxy [java.sql.Connection] []
+                     (isClosed [] @closed?))
+        query-fn (with-meta (fn [& _] (throw (Exception. "Must not query database")))
+                   {:connection connection})]
+    (is (= 200 (:status (health/healthcheck! query-fn {}))))
+    (reset! closed? true)
+    (is (= 503 (:status (health/healthcheck! query-fn {}))))
+    (is (= "writer session closed"
+           (get-in (health/healthcheck! query-fn {}) [:body :reason]))))
+  (is (= 503 (:status (health/healthcheck! (fn [& _]) {})))))
 
 (deftest readiness-check-tests-database
   (let [called (atom nil)
diff --git a/test/clj/pmagnus/btcdata/writer_test.clj b/test/clj/pmagnus/btcdata/writer_test.clj
index 4353604..bdbecca 100644
--- a/test/clj/pmagnus/btcdata/writer_test.clj
+++ b/test/clj/pmagnus/btcdata/writer_test.clj
@@ -13,6 +13,13 @@
     (is (= :result (query :insert-price {:price 1})))
     (is (= [connection :insert-price {:price 1}] @seen))))
 
+(deftest session-liveness-fails-closed
+  (is (false? (writer/session-live? (fn [& _]))))
+  (let [connection (proxy [java.sql.Connection] []
+                     (isClosed [] (throw (java.sql.SQLException. "broken"))))]
+    (is (false? (writer/session-live? (with-meta (fn [& _])
+                                       {:connection connection}))))))
+
 (deftest server-session-has-bounded-dead-peer-detection
   (let [connection (Object.)
         calls (atom [])]
@@ -44,7 +51,9 @@
         (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Another btcdata writer"
               (ig/init-key :writer/query-fn opts)))
         ;; Closing the owning session cannot transparently reconnect for writes.
+        (is (writer/session-live? first-writer))
         (.close (:connection (meta first-writer)))
+        (is (false? (writer/session-live? first-writer)))
         (is (thrown? Exception (first-writer :check {})))
         (let [replacement (ig/init-key :writer/query-fn opts)]
           (try