From d0ff9b485d3858275e2623362ac12ce72ad49fbc Mon Sep 17 00:00:00 2001 From: Per Magnus Petersen Date: Sat, 14 Feb 2026 18:50:41 +0100 Subject: [PATCH] Skip sync for existing data and too-recent dates - Check consumption table before fetching; return early if data exists - Return "no data to fetch" for dates >= yesterday (not yet available) - Add sync-month.sh script to sync all days in a month - Add kill-server.sh script to free port 3000 Co-Authored-By: Claude Opus 4.6 --- kill-server.sh | 13 ++++ resources/queries.sql | 6 ++ .../elprice/web/controllers/eloverblik.clj | 69 +++++++++++-------- src/clj/pmagnus/elprice/web/routes/ui.clj | 8 ++- sync-month.sh | 32 +++++++++ 5 files changed, 99 insertions(+), 29 deletions(-) create mode 100755 kill-server.sh create mode 100755 sync-month.sh diff --git a/kill-server.sh b/kill-server.sh new file mode 100755 index 0000000..ec16489 --- /dev/null +++ b/kill-server.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Kill any process listening on port 3000. +# Usage: ./kill-server.sh +set -euo pipefail + +pids=$(lsof -ti:3000 2>/dev/null || true) + +if [ -z "$pids" ]; then + echo "No process on port 3000" +else + echo "$pids" | xargs kill + echo "Killed: $pids" +fi diff --git a/resources/queries.sql b/resources/queries.sql index 6090995..3596213 100644 --- a/resources/queries.sql +++ b/resources/queries.sql @@ -48,6 +48,12 @@ VALUES (:metering-point-id, :charge-type, :name, :description, :owner, :valid-from-date, :valid-to-date, :period-type, :price, :quantity, :position) ON CONFLICT (metering_point_id, charge_type, name, owner, valid_from_date, position) DO NOTHING; +-- :name count-consumption-for-date :? :1 +-- :doc Count consumption records for a given UTC time_start +SELECT count(*) AS cnt +FROM consumption +WHERE time_start = :time-start; + -- :name insert-production! :! :n -- :doc Insert a production record, skip if already exists INSERT INTO production (time_start, hour, kwh) diff --git a/src/clj/pmagnus/elprice/web/controllers/eloverblik.clj b/src/clj/pmagnus/elprice/web/controllers/eloverblik.clj index 77e3725..2825c57 100644 --- a/src/clj/pmagnus/elprice/web/controllers/eloverblik.clj +++ b/src/clj/pmagnus/elprice/web/controllers/eloverblik.clj @@ -246,31 +246,44 @@ "Fetch all Eloverblik data and save to database. date is a LocalDate for which to fetch time series (one day)." [query-fn ^LocalDate date] - (log/info "Starting Eloverblik fetch for" (.format date date-fmt)) - (let [refresh-token (read-refresh-token) - access-token (get-access-token refresh-token)] - (if-not access-token - (do (log/error "Failed to get Eloverblik access token") - {:error "Failed to get access token"}) - (let [mps (fetch-metering-points access-token) - all-ids (->> mps - (mapcat (fn [mp] - (cons (:meteringPointId mp) - (map :meteringPointId - (:childMeteringPoints mp))))) - (distinct) - (vec)) - from date - to (.plusDays date 1)] - (log/info "Found" (count all-ids) "metering points:" all-ids) - (save-metering-points! query-fn mps) - ;; Time series - (let [ts-resp (fetch-time-series access-token all-ids from to) - readings (parse-time-series ts-resp all-ids)] - (save-time-series! query-fn readings from)) - ;; Charges - (let [ch-resp (fetch-charges access-token all-ids)] - (save-charges! query-fn ch-resp)) - {:metering-points (count all-ids) - :from (.format from date-fmt) - :to (.format to date-fmt)})))) + (let [yesterday (.minusDays (LocalDate/now dk-zone) 1) + time-start (utc-start-for-date date) + cnt (:cnt (query-fn :count-consumption-for-date + {:time-start time-start}))] + (cond + (not (.isBefore date yesterday)) + (do (log/info "No data to fetch for" (.format date date-fmt) "(too recent)") + {:no-data true :date (.format date date-fmt)}) + + (and cnt (pos? cnt)) + (do (log/info "Data already exists for" (.format date date-fmt)) + {:exists true :date (.format date date-fmt)}) + :else + (do + (log/info "Starting Eloverblik fetch for" (.format date date-fmt)) + (let [refresh-token (read-refresh-token) + access-token (get-access-token refresh-token)] + (if-not access-token + (do (log/error "Failed to get Eloverblik access token") + {:error "Failed to get access token"}) + (let [mps (fetch-metering-points access-token) + all-ids (->> mps + (mapcat (fn [mp] + (cons (:meteringPointId mp) + (map :meteringPointId + (:childMeteringPoints mp))))) + (distinct) + (vec)) + from date + to (.plusDays date 1)] + (log/info "Found" (count all-ids) "metering points:" all-ids) + (save-metering-points! query-fn mps) + (let [ts-resp (fetch-time-series access-token all-ids from to) + readings (parse-time-series ts-resp all-ids)] + (save-time-series! query-fn readings from)) + (let [ch-resp (fetch-charges access-token all-ids)] + (save-charges! query-fn ch-resp)) + {:metering-points (count all-ids) + :from (.format from date-fmt) + :to (.format to date-fmt)}))))))) + diff --git a/src/clj/pmagnus/elprice/web/routes/ui.clj b/src/clj/pmagnus/elprice/web/routes/ui.clj index 9210c4e..6f60b6c 100644 --- a/src/clj/pmagnus/elprice/web/routes/ui.clj +++ b/src/clj/pmagnus/elprice/web/routes/ui.clj @@ -182,8 +182,14 @@ (java.time.LocalDate/now dk-zone)) result (eloverblik/fetch-and-save-all! query-fn date)] (fragment - (if (:error result) + (cond + (:error result) [:p {:class "text-red-600 font-medium"} (:error result)] + (:no-data result) + [:p {:class "text-gray-400"} (str "No data to fetch for " (:date result))] + (:exists result) + [:p {:class "text-gray-500"} (str "Data exists for " (:date result))] + :else [:p {:class "text-green-700 font-medium"} (str "Synced " (:metering-points result) " metering points (" (:from result) " to " (:to result) ")")])))) diff --git a/sync-month.sh b/sync-month.sh new file mode 100755 index 0000000..9516bf8 --- /dev/null +++ b/sync-month.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Sync Eloverblik data for a specific month. +# Usage: ./sync-month.sh 2026-02 +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "Usage: $0 YYYY-MM" + exit 1 +fi + +MONTH="$1" +BASE_URL="${BASE_URL:-http://localhost:3000}" + +# Last day of the month (macOS date) +days=$(date -j -v+1m -v-1d -f "%Y-%m-%d" "${MONTH}-01" "+%d") + +echo "Syncing $MONTH ($days days)" + +for day in $(seq 1 "$days"); do + d=$(printf "%s-%02d" "$MONTH" "$day") + resp=$(curl -s -X POST "${BASE_URL}/eloverblik/sync?date=${d}") + text=$(echo "$resp" | sed 's/<[^>]*>//g' | xargs) + + if [ -z "$text" ]; then + echo "$d: no response (server down?)" + exit 1 + fi + + echo "$d: $text" +done + +echo "Done."