Bound PostgreSQL dead-peer detection for writer fencing

Reviewed-on: #2

btcdata-deploy.md

@@ -1,5 +1,36 @@
 # btcdata Kubernetes deployment plan
 
+## 2026-10-07 incident: orphaned writer session
+
+The `sejs/btcdata` pod on citadel entered CrashLoopBackOff with
+`Another btcdata writer holds the database lock`. The prior pod on srv1893094
+had been removed after readiness/network failures, but its PostgreSQL backend
+(PID 3531439, client 100.75.252.111) still held advisory lock 728194106. No old
+btcdata Java process or database TCP connection remained on that node.
+
+Recovery terminated only that verified orphaned backend. Kubernetes restarted
+btcdata successfully; btcdata and both btcprice pods became Ready. Do not bypass
+the advisory lock or terminate its owner without first verifying the old writer
+is stopped.
+
+Production database `btcprod` now has connection defaults:
+`tcp_keepalives_idle=60`, `tcp_keepalives_interval=10`,
+`tcp_keepalives_count=3`, and `tcp_user_timeout=90000` (milliseconds).
+These apply to **new connections**, not the already-running recovered session.
+Server-side dead-peer detection prevents a disappeared node from retaining the
+lock for the default multi-hour TCP timeout; it does not evict a live competing
+writer. The application now also sets these values on its lock-owning session
+before acquiring the fence (source change; not yet built/deployed).
+
+Validation: 20 unit tests / 58 assertions passed; disposable PostgreSQL integration
+tests were not run because the local Docker daemon was unavailable. New
+production connection settings were verified with `SHOW`.
+
+Rollback database defaults, if necessary, with `ALTER DATABASE btcprod RESET
+<setting>` for each of the four settings above; new connections inherit the
+server defaults again. Reverting the application change requires a normal image
+build and Flux rollout. The orphan-session termination is not reversible.
+
 ## 2026-09-19 status and proposed recovery
 
 Kubernetes deployment already exists: one `Recreate` replica on citadel, eligible

scripts/fetch-year.sh

@@ -1,14 +1,26 @@
 #!/usr/bin/env bash
-# Usage: ./scripts/fetch-year.sh YEAR
+# Usage: ./scripts/fetch-year.sh YEAR [--reverse]
 # Fetches sequentially; prints JSON responses to stdout and errors to stderr.
 set -euo pipefail
 
-if [[ $# -ne 1 ]]; then
-  echo "Usage: $0 YEAR" >&2
+reverse=false
+year=
+for arg in "$@"; do
+  case "$arg" in
+    --reverse) reverse=true ;;
+    *)
+      if [[ -n $year ]]; then
+        echo "Usage: $0 YEAR [--reverse]" >&2
+        exit 1
+      fi
+      year=$arg
+      ;;
+  esac
+done
+if [[ -z $year ]]; then
+  echo "Usage: $0 YEAR [--reverse]" >&2
   exit 1
 fi
-
-year=$1
 if [[ ! $year =~ ^[0-9]{4}$ || $year == 0000 ]]; then
   echo "YEAR must be a four-digit year from 0001 to 9999." >&2
   exit 1
@@ -31,47 +43,109 @@ cleanup() {
   if [[ -n $forward_pid ]]; then
     kill "$forward_pid" 2>/dev/null || true
     wait "$forward_pid" 2>/dev/null || true
+    forward_pid=
   fi
 }
 trap cleanup EXIT
 trap 'exit 130' INT
 trap 'exit 143' TERM
 
-if ! service_ready; then
+wait_for_pod() {
+  local selector
+  # Use the service's selector rather than assuming a particular pod label.
+  if ! selector=$(kubectl -n sejs get svc btcdata --request-timeout=10s \
+    -o go-template='{{range $key, $value := .spec.selector}}{{$key}}={{$value}},{{end}}'); then
+    return 1
+  fi
+  selector=${selector%,}
+  if [[ -z $selector ]]; then
+    echo "Service sejs/btcdata has no pod selector." >&2
+    return 1
+  fi
+  echo "Waiting up to 5 minutes for btcdata pods to become Ready..." >&2
+  kubectl -n sejs wait --for=condition=Ready pod --selector="$selector" \
+    --timeout=300s --request-timeout=310s >&2
+}
+
+start_forward() {
+  local attempt
+  cleanup
   if ! command -v kubectl >/dev/null 2>&1; then
     echo "Service unavailable on localhost:4101 and kubectl is not installed." >&2
-    exit 1
+    return 1
+  fi
+  if ! wait_for_pod; then
+    echo "btcdata pod readiness check failed; not opening a tunnel." >&2
+    return 1
   fi
   echo "Starting kubectl port-forward for sejs/btcdata..." >&2
   kubectl -n sejs port-forward svc/btcdata 4101:4101 >&2 &
   forward_pid=$!
 
-  ready=false
   for (( attempt = 0; attempt < 30; attempt++ )); do
     if ! kill -0 "$forward_pid" 2>/dev/null; then
       echo "Port-forward exited before the service became ready." >&2
-      exit 1
+      cleanup
+      return 1
     fi
     if service_ready; then
-      ready=true
-      break
+      return 0
     fi
     sleep 1
   done
-  if [[ $ready != true ]]; then
-    echo "Timed out waiting for the service on localhost:4101." >&2
-    exit 1
+  echo "Timed out waiting for the service on localhost:4101." >&2
+  cleanup
+  return 1
+}
+
+ensure_service() {
+  local retry
+  if service_ready; then
+    return 0
   fi
-fi
+  for (( retry = 1; retry <= 3; retry++ )); do
+    echo "Recovering port-forward (attempt $retry/3)..." >&2
+    if start_forward; then
+      return 0
+    fi
+    if (( retry < 3 )); then
+      sleep 2
+    fi
+  done
+  echo "Unable to restore the service on localhost:4101." >&2
+  return 1
+}
+
+fetch_date() {
+  curl --fail --silent --show-error --connect-timeout 5 -X GET \
+    "http://localhost:4101/api/price/day/$1" \
+    -H "accept: application/json" --write-out '\n'
+}
+
+ensure_service || exit 1
 
 # Generate dates without relying on platform-specific date command options.
 status=0
-for (( month = 1; month <= 12; month++ )); do
-  for (( day = 1; day <= days_in_month[month - 1]; day++ )); do
+for (( month_index = 0; month_index < 12; month_index++ )); do
+  month=$((month_index + 1))
+  if [[ $reverse == true ]]; then
+    month=$((12 - month_index))
+  fi
+  days=${days_in_month[month - 1]}
+  for (( day_index = 0; day_index < days; day_index++ )); do
+    day=$((day_index + 1))
+    if [[ $reverse == true ]]; then
+      day=$((days - day_index))
+    fi
     printf -v date '%s-%02d-%02d' "$year" "$month" "$day"
-    if ! curl --fail --silent --show-error -X GET \
-      "http://localhost:4101/api/price/day/$date" \
-      -H "accept: application/json" --write-out '\n'; then
+    if ! fetch_date "$date"; then
+      if ! service_ready; then
+        ensure_service || exit 1
+        echo "Retrying $date after tunnel recovery..." >&2
+        if fetch_date "$date"; then
+          continue
+        fi
+      fi
       echo "Failed to fetch $date" >&2
       status=1
     fi

src/clj/pmagnus/btcdata/writer.clj

@@ -7,6 +7,15 @@
 
 (def lock-id 728194106)
 
+(defn configure-session! [connection]
+  ;; Client-side network timeouts cannot reap a server session left behind by a
+  ;; dead node. Server TCP keepalives release its fence after ~90s of silence.
+  (doseq [sql ["SET tcp_keepalives_idle = 60"
+               "SET tcp_keepalives_interval = 10"
+               "SET tcp_keepalives_count = 3"
+               "SET tcp_user_timeout = 90000"]]
+    (jdbc/execute-one! connection [sql])))
+
 (defn guarded-query [connection query-fn]
   (fn [query params]
     (locking connection
@@ -20,6 +29,7 @@
       (.setNetworkTimeout connection
         (reify java.util.concurrent.Executor (execute [_ task] (.run ^Runnable task)))
         10000)
+      (configure-session! connection)
       (when-not (:locked (jdbc/execute-one! connection
                            ["select pg_try_advisory_lock(?) as locked" lock-id]))
         (throw (ex-info "Another btcdata writer holds the database lock" {})))

test/clj/pmagnus/btcdata/writer_test.clj

@@ -13,6 +13,18 @@
     (is (= :result (query :insert-price {:price 1})))
     (is (= [connection :insert-price {:price 1}] @seen))))
 
+(deftest server-session-has-bounded-dead-peer-detection
+  (let [connection (Object.)
+        calls (atom [])]
+    (with-redefs [jdbc/execute-one! (fn [conn sql] (swap! calls conj [conn sql]))]
+      (writer/configure-session! connection))
+    (is (= (mapv #(vector connection [%])
+                 ["SET tcp_keepalives_idle = 60"
+                  "SET tcp_keepalives_interval = 10"
+                  "SET tcp_keepalives_count = 3"
+                  "SET tcp_user_timeout = 90000"])
+           @calls))))
+
 (deftest postgres-session-fencing
   ;; Optional disposable PostgreSQL integration test; never use production here.
   (if-let [url (System/getenv "BTCDATA_TEST_JDBC_URL")]
@@ -22,6 +34,13 @@
           first-writer (ig/init-key :writer/query-fn opts)]
       (try
         (is (= 1 (:healthy (first-writer :check {}))))
+        (let [connection (:connection (meta first-writer))]
+          (doseq [[setting expected] [["tcp_keepalives_idle" "60"]
+                                     ["tcp_keepalives_interval" "10"]
+                                     ["tcp_keepalives_count" "3"]
+                                     ["tcp_user_timeout" "90000"]]]
+            (is (= expected (:value (jdbc/execute-one! connection
+                                     ["select current_setting(?) as value" setting]))))))
         (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Another btcdata writer"
               (ig/init-key :writer/query-fn opts)))
         ;; Closing the owning session cannot transparently reconnect for writes.