diff --git a/btcdata-deploy.md b/btcdata-deploy.md index 508da8b..66bd968 100644 --- a/btcdata-deploy.md +++ b/btcdata-deploy.md @@ -1,5 +1,36 @@ # btcdata Kubernetes deployment plan +## 2026-10-07 incident: orphaned writer session + +The `sejs/btcdata` pod on citadel entered CrashLoopBackOff with +`Another btcdata writer holds the database lock`. The prior pod on srv1893094 +had been removed after readiness/network failures, but its PostgreSQL backend +(PID 3531439, client 100.75.252.111) still held advisory lock 728194106. No old +btcdata Java process or database TCP connection remained on that node. + +Recovery terminated only that verified orphaned backend. Kubernetes restarted +btcdata successfully; btcdata and both btcprice pods became Ready. Do not bypass +the advisory lock or terminate its owner without first verifying the old writer +is stopped. + +Production database `btcprod` now has connection defaults: +`tcp_keepalives_idle=60`, `tcp_keepalives_interval=10`, +`tcp_keepalives_count=3`, and `tcp_user_timeout=90000` (milliseconds). +These apply to **new connections**, not the already-running recovered session. +Server-side dead-peer detection prevents a disappeared node from retaining the +lock for the default multi-hour TCP timeout; it does not evict a live competing +writer. The application now also sets these values on its lock-owning session +before acquiring the fence (source change; not yet built/deployed). + +Validation: 20 unit tests / 58 assertions passed; disposable PostgreSQL integration +tests were not run because the local Docker daemon was unavailable. New +production connection settings were verified with `SHOW`. + +Rollback database defaults, if necessary, with `ALTER DATABASE btcprod RESET +` for each of the four settings above; new connections inherit the +server defaults again. Reverting the application change requires a normal image +build and Flux rollout. The orphan-session termination is not reversible. + ## 2026-09-19 status and proposed recovery Kubernetes deployment already exists: one `Recreate` replica on citadel, eligible diff --git a/src/clj/pmagnus/btcdata/writer.clj b/src/clj/pmagnus/btcdata/writer.clj index a2777b9..14fd3b3 100644 --- a/src/clj/pmagnus/btcdata/writer.clj +++ b/src/clj/pmagnus/btcdata/writer.clj @@ -7,6 +7,15 @@ (def lock-id 728194106) +(defn configure-session! [connection] + ;; Client-side network timeouts cannot reap a server session left behind by a + ;; dead node. Server TCP keepalives release its fence after ~90s of silence. + (doseq [sql ["SET tcp_keepalives_idle = 60" + "SET tcp_keepalives_interval = 10" + "SET tcp_keepalives_count = 3" + "SET tcp_user_timeout = 90000"]] + (jdbc/execute-one! connection [sql]))) + (defn guarded-query [connection query-fn] (fn [query params] (locking connection @@ -20,6 +29,7 @@ (.setNetworkTimeout connection (reify java.util.concurrent.Executor (execute [_ task] (.run ^Runnable task))) 10000) + (configure-session! connection) (when-not (:locked (jdbc/execute-one! connection ["select pg_try_advisory_lock(?) as locked" lock-id])) (throw (ex-info "Another btcdata writer holds the database lock" {}))) diff --git a/test/clj/pmagnus/btcdata/writer_test.clj b/test/clj/pmagnus/btcdata/writer_test.clj index a6a1124..4353604 100644 --- a/test/clj/pmagnus/btcdata/writer_test.clj +++ b/test/clj/pmagnus/btcdata/writer_test.clj @@ -13,6 +13,18 @@ (is (= :result (query :insert-price {:price 1}))) (is (= [connection :insert-price {:price 1}] @seen)))) +(deftest server-session-has-bounded-dead-peer-detection + (let [connection (Object.) + calls (atom [])] + (with-redefs [jdbc/execute-one! (fn [conn sql] (swap! calls conj [conn sql]))] + (writer/configure-session! connection)) + (is (= (mapv #(vector connection [%]) + ["SET tcp_keepalives_idle = 60" + "SET tcp_keepalives_interval = 10" + "SET tcp_keepalives_count = 3" + "SET tcp_user_timeout = 90000"]) + @calls)))) + (deftest postgres-session-fencing ;; Optional disposable PostgreSQL integration test; never use production here. (if-let [url (System/getenv "BTCDATA_TEST_JDBC_URL")] @@ -22,6 +34,13 @@ first-writer (ig/init-key :writer/query-fn opts)] (try (is (= 1 (:healthy (first-writer :check {})))) + (let [connection (:connection (meta first-writer))] + (doseq [[setting expected] [["tcp_keepalives_idle" "60"] + ["tcp_keepalives_interval" "10"] + ["tcp_keepalives_count" "3"] + ["tcp_user_timeout" "90000"]]] + (is (= expected (:value (jdbc/execute-one! connection + ["select current_setting(?) as value" setting])))))) (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Another btcdata writer" (ig/init-key :writer/query-fn opts))) ;; Closing the owning session cannot transparently reconnect for writes.