Skip sync for existing data and too-recent dates
- Check consumption table before fetching; return early if data exists - Return "no data to fetch" for dates >= yesterday (not yet available) - Add sync-month.sh script to sync all days in a month - Add kill-server.sh script to free port 3000 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Kill any process listening on port 3000.
|
||||||
|
# Usage: ./kill-server.sh
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
pids=$(lsof -ti:3000 2>/dev/null || true)
|
||||||
|
|
||||||
|
if [ -z "$pids" ]; then
|
||||||
|
echo "No process on port 3000"
|
||||||
|
else
|
||||||
|
echo "$pids" | xargs kill
|
||||||
|
echo "Killed: $pids"
|
||||||
|
fi
|
||||||
@@ -48,6 +48,12 @@ VALUES (:metering-point-id, :charge-type, :name, :description, :owner,
|
|||||||
:valid-from-date, :valid-to-date, :period-type, :price, :quantity, :position)
|
:valid-from-date, :valid-to-date, :period-type, :price, :quantity, :position)
|
||||||
ON CONFLICT (metering_point_id, charge_type, name, owner, valid_from_date, position) DO NOTHING;
|
ON CONFLICT (metering_point_id, charge_type, name, owner, valid_from_date, position) DO NOTHING;
|
||||||
|
|
||||||
|
-- :name count-consumption-for-date :? :1
|
||||||
|
-- :doc Count consumption records for a given UTC time_start
|
||||||
|
SELECT count(*) AS cnt
|
||||||
|
FROM consumption
|
||||||
|
WHERE time_start = :time-start;
|
||||||
|
|
||||||
-- :name insert-production! :! :n
|
-- :name insert-production! :! :n
|
||||||
-- :doc Insert a production record, skip if already exists
|
-- :doc Insert a production record, skip if already exists
|
||||||
INSERT INTO production (time_start, hour, kwh)
|
INSERT INTO production (time_start, hour, kwh)
|
||||||
|
|||||||
@@ -246,6 +246,20 @@
|
|||||||
"Fetch all Eloverblik data and save to database.
|
"Fetch all Eloverblik data and save to database.
|
||||||
date is a LocalDate for which to fetch time series (one day)."
|
date is a LocalDate for which to fetch time series (one day)."
|
||||||
[query-fn ^LocalDate date]
|
[query-fn ^LocalDate date]
|
||||||
|
(let [yesterday (.minusDays (LocalDate/now dk-zone) 1)
|
||||||
|
time-start (utc-start-for-date date)
|
||||||
|
cnt (:cnt (query-fn :count-consumption-for-date
|
||||||
|
{:time-start time-start}))]
|
||||||
|
(cond
|
||||||
|
(not (.isBefore date yesterday))
|
||||||
|
(do (log/info "No data to fetch for" (.format date date-fmt) "(too recent)")
|
||||||
|
{:no-data true :date (.format date date-fmt)})
|
||||||
|
|
||||||
|
(and cnt (pos? cnt))
|
||||||
|
(do (log/info "Data already exists for" (.format date date-fmt))
|
||||||
|
{:exists true :date (.format date date-fmt)})
|
||||||
|
:else
|
||||||
|
(do
|
||||||
(log/info "Starting Eloverblik fetch for" (.format date date-fmt))
|
(log/info "Starting Eloverblik fetch for" (.format date date-fmt))
|
||||||
(let [refresh-token (read-refresh-token)
|
(let [refresh-token (read-refresh-token)
|
||||||
access-token (get-access-token refresh-token)]
|
access-token (get-access-token refresh-token)]
|
||||||
@@ -264,13 +278,12 @@
|
|||||||
to (.plusDays date 1)]
|
to (.plusDays date 1)]
|
||||||
(log/info "Found" (count all-ids) "metering points:" all-ids)
|
(log/info "Found" (count all-ids) "metering points:" all-ids)
|
||||||
(save-metering-points! query-fn mps)
|
(save-metering-points! query-fn mps)
|
||||||
;; Time series
|
|
||||||
(let [ts-resp (fetch-time-series access-token all-ids from to)
|
(let [ts-resp (fetch-time-series access-token all-ids from to)
|
||||||
readings (parse-time-series ts-resp all-ids)]
|
readings (parse-time-series ts-resp all-ids)]
|
||||||
(save-time-series! query-fn readings from))
|
(save-time-series! query-fn readings from))
|
||||||
;; Charges
|
|
||||||
(let [ch-resp (fetch-charges access-token all-ids)]
|
(let [ch-resp (fetch-charges access-token all-ids)]
|
||||||
(save-charges! query-fn ch-resp))
|
(save-charges! query-fn ch-resp))
|
||||||
{:metering-points (count all-ids)
|
{:metering-points (count all-ids)
|
||||||
:from (.format from date-fmt)
|
:from (.format from date-fmt)
|
||||||
:to (.format to date-fmt)}))))
|
:to (.format to date-fmt)})))))))
|
||||||
|
|
||||||
|
|||||||
@@ -182,8 +182,14 @@
|
|||||||
(java.time.LocalDate/now dk-zone))
|
(java.time.LocalDate/now dk-zone))
|
||||||
result (eloverblik/fetch-and-save-all! query-fn date)]
|
result (eloverblik/fetch-and-save-all! query-fn date)]
|
||||||
(fragment
|
(fragment
|
||||||
(if (:error result)
|
(cond
|
||||||
|
(:error result)
|
||||||
[:p {:class "text-red-600 font-medium"} (:error result)]
|
[:p {:class "text-red-600 font-medium"} (:error result)]
|
||||||
|
(:no-data result)
|
||||||
|
[:p {:class "text-gray-400"} (str "No data to fetch for " (:date result))]
|
||||||
|
(:exists result)
|
||||||
|
[:p {:class "text-gray-500"} (str "Data exists for " (:date result))]
|
||||||
|
:else
|
||||||
[:p {:class "text-green-700 font-medium"}
|
[:p {:class "text-green-700 font-medium"}
|
||||||
(str "Synced " (:metering-points result)
|
(str "Synced " (:metering-points result)
|
||||||
" metering points (" (:from result) " to " (:to result) ")")]))))
|
" metering points (" (:from result) " to " (:to result) ")")]))))
|
||||||
|
|||||||
Executable
+32
@@ -0,0 +1,32 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Sync Eloverblik data for a specific month.
|
||||||
|
# Usage: ./sync-month.sh 2026-02
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [ $# -ne 1 ]; then
|
||||||
|
echo "Usage: $0 YYYY-MM"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
MONTH="$1"
|
||||||
|
BASE_URL="${BASE_URL:-http://localhost:3000}"
|
||||||
|
|
||||||
|
# Last day of the month (macOS date)
|
||||||
|
days=$(date -j -v+1m -v-1d -f "%Y-%m-%d" "${MONTH}-01" "+%d")
|
||||||
|
|
||||||
|
echo "Syncing $MONTH ($days days)"
|
||||||
|
|
||||||
|
for day in $(seq 1 "$days"); do
|
||||||
|
d=$(printf "%s-%02d" "$MONTH" "$day")
|
||||||
|
resp=$(curl -s -X POST "${BASE_URL}/eloverblik/sync?date=${d}")
|
||||||
|
text=$(echo "$resp" | sed 's/<[^>]*>//g' | xargs)
|
||||||
|
|
||||||
|
if [ -z "$text" ]; then
|
||||||
|
echo "$d: no response (server down?)"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "$d: $text"
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Done."
|
||||||
Reference in New Issue
Block a user