diff --git a/deps.edn b/deps.edn index bf9097a..2157611 100644 --- a/deps.edn +++ b/deps.edn @@ -26,6 +26,8 @@ ;; Serialization metosin/muuntaja {:mvn/version "0.6.11"} + metosin/jsonista {:mvn/version "0.3.12"} + com.fasterxml.jackson.datatype/jackson-datatype-jsr310 {:mvn/version "2.18.3"} luminus-transit/luminus-transit {:mvn/version "0.1.6"} ;; Database diff --git a/resources/queries.sql b/resources/queries.sql index cdad95a..da916d3 100644 --- a/resources/queries.sql +++ b/resources/queries.sql @@ -102,7 +102,7 @@ SELECT e.id, e.event_type, e.occurred_at, e.recorded_at, FROM events e LEFT JOIN wallets fw ON fw.id = e.from_wallet_id LEFT JOIN wallets tw ON tw.id = e.to_wallet_id -ORDER BY e.occurred_at DESC +ORDER BY e.occurred_at DESC, e.id DESC -- :name get-events-by-type :? :* -- :doc Get events filtered by type @@ -115,7 +115,7 @@ FROM events e LEFT JOIN wallets fw ON fw.id = e.from_wallet_id LEFT JOIN wallets tw ON tw.id = e.to_wallet_id WHERE e.event_type = :event-type -ORDER BY e.occurred_at DESC +ORDER BY e.occurred_at DESC, e.id DESC -- :name get-events-by-wallet :? :* -- :doc Get events involving a specific wallet @@ -128,7 +128,20 @@ FROM events e LEFT JOIN wallets fw ON fw.id = e.from_wallet_id LEFT JOIN wallets tw ON tw.id = e.to_wallet_id WHERE e.from_wallet_id = :wallet-id OR e.to_wallet_id = :wallet-id -ORDER BY e.occurred_at DESC +ORDER BY e.occurred_at DESC, e.id DESC + +-- :name truncate-deposits! :! :n +-- :doc Delete all rows from the deposits read table +TRUNCATE deposits + +-- :name get-deposit-events :? :* +-- :doc Get bank_to_exchange events with exchange name, oldest first +SELECT e.id, e.occurred_at, e.fiat_amount, e.fiat_currency, + e.to_wallet_id, tw.name AS exchange, e.note +FROM events e +LEFT JOIN wallets tw ON tw.id = e.to_wallet_id +WHERE e.event_type = 'bank_to_exchange' +ORDER BY e.occurred_at ASC, e.id ASC -- :name insert-deposit! :! :n -- :doc Insert a projected deposit row @@ -140,7 +153,7 @@ VALUES (:event-id, :occurred-at, :exchange, :fiat-amount, :fiat-currency, :amoun SELECT id, event_id, occurred_at, exchange, fiat_amount, fiat_currency, amount_eur, amount_dkk, amount_usd, note FROM deposits -ORDER BY occurred_at DESC +ORDER BY occurred_at DESC, id DESC -- :name get-wallet-balances :? :* -- :doc Compute sats balance per wallet from events diff --git a/src/clj/pmagnus/btcdata/core.clj b/src/clj/pmagnus/btcdata/core.clj index 0d64070..8d42836 100644 --- a/src/clj/pmagnus/btcdata/core.clj +++ b/src/clj/pmagnus/btcdata/core.clj @@ -2,6 +2,7 @@ (:require [clojure.tools.logging :as log] [integrant.core :as ig] + [next.jdbc.result-set] [pmagnus.btcdata.config :as config] [pmagnus.btcdata.env :refer [defaults]] @@ -20,7 +21,22 @@ [pmagnus.btcdata.kraken.ohlc-daily] [pmagnus.btcdata.frankfurter.rates] [pmagnus.btcdata.strike.ticker]) - (:gen-class)) + (:gen-class) + (:import + [java.sql Date Timestamp])) + +;; Read SQL DATE as LocalDate and TIMESTAMP as Instant (timezone-safe) +(extend-protocol next.jdbc.result-set/ReadableColumn + Date + (read-column-by-label [v _] + (.toLocalDate v)) + (read-column-by-index [v _ _] + (.toLocalDate v)) + Timestamp + (read-column-by-label [v _] + (.toInstant v)) + (read-column-by-index [v _ _] + (.toInstant v))) (defonce system (atom nil)) diff --git a/src/clj/pmagnus/btcdata/web/controllers/transactions.clj b/src/clj/pmagnus/btcdata/web/controllers/transactions.clj index f9a5f4c..31a0642 100644 --- a/src/clj/pmagnus/btcdata/web/controllers/transactions.clj +++ b/src/clj/pmagnus/btcdata/web/controllers/transactions.clj @@ -98,6 +98,36 @@ (defn list-deposits [{:keys [query-fn]} _req] (response/ok (query-fn :get-all-deposits {}))) +(defn rebuild-deposits! [{:keys [query-fn frankfurter]} _req] + (query-fn :truncate-deposits! {}) + (let [events (query-fn :get-deposit-events {}) + n (reduce + (fn [cnt {:keys [id occurred_at fiat_amount fiat_currency exchange note]}] + (if-not fiat_amount + cnt + (try + (let [rates (or (query-fn :get-currency-rates-by-date {:rate-date occurred_at}) + (rates/fetch-and-persist-for-date! + (:client frankfurter) (:url frankfurter) + query-fn occurred_at)) + amounts (if rates + (convert-amount fiat_amount fiat_currency rates) + {:amount-eur nil :amount-dkk nil :amount-usd nil})] + (query-fn :insert-deposit! + (merge {:event-id id + :occurred-at occurred_at + :exchange (or exchange "Unknown") + :fiat-amount fiat_amount + :fiat-currency fiat_currency + :note note} + amounts)) + (inc cnt)) + (catch Exception e + (log/error e "Failed to rebuild deposit for event" id) + cnt)))) + 0 events)] + (response/ok {:rebuilt n}))) + (defn list-events [{:keys [query-fn]} req] (let [event-type (get-in req [:query-params "type"])] (if event-type diff --git a/src/clj/pmagnus/btcdata/web/middleware/formats.clj b/src/clj/pmagnus/btcdata/web/middleware/formats.clj index e84f374..3204eeb 100644 --- a/src/clj/pmagnus/btcdata/web/middleware/formats.clj +++ b/src/clj/pmagnus/btcdata/web/middleware/formats.clj @@ -1,11 +1,23 @@ (ns pmagnus.btcdata.web.middleware.formats (:require + [jsonista.core :as j] [luminus-transit.time :as time] - [muuntaja.core :as m])) + [muuntaja.core :as m]) + (:import + [com.fasterxml.jackson.databind SerializationFeature] + [com.fasterxml.jackson.datatype.jsr310 JavaTimeModule])) + +(def ^:private mapper + (j/object-mapper + {:modules [(JavaTimeModule.)] + :decode-key-fn true + :configure {SerializationFeature/WRITE_DATES_AS_TIMESTAMPS false}})) (def instance (m/create (-> m/default-options + (assoc-in [:formats "application/json" :decoder-opts] {:mapper mapper}) + (assoc-in [:formats "application/json" :encoder-opts] {:mapper mapper}) (update-in [:formats "application/transit+json" :decoder-opts] (partial merge time/time-deserialization-handlers)) (update-in [:formats "application/transit+json" :encoder-opts] diff --git a/src/clj/pmagnus/btcdata/web/routes/api.clj b/src/clj/pmagnus/btcdata/web/routes/api.clj index 910ce13..aa35866 100644 --- a/src/clj/pmagnus/btcdata/web/routes/api.clj +++ b/src/clj/pmagnus/btcdata/web/routes/api.clj @@ -174,7 +174,9 @@ {:get (fn [req] (tx/list-events opts req)) :post (fn [req] (tx/create-event! opts req))}] ["/deposits" - {:get (fn [req] (tx/list-deposits opts req))}]]) + {:get (fn [req] (tx/list-deposits opts req))}] + ["/deposits/rebuild" + {:post (fn [req] (tx/rebuild-deposits! opts req))}]]) (defn route-data [opts] (merge