Implement verified weekly archives and SQLite inventory reporting

sql/indexes.sql

@@ -0,0 +1,9 @@
+-- Inspect pg_indexes first; equivalent indexes with other names should be reused.
+-- Run outside a transaction: psql -v ON_ERROR_STOP=1 -f sql/indexes.sql
+-- IF NOT EXISTS only checks names, not index validity or equivalence. Inspect invalid
+-- indexes after an interrupted build before retrying.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS binance_price_recorded_at_id
+    ON public.binance_price (recorded_at, id);
+CREATE INDEX CONCURRENTLY IF NOT EXISTS strike_price_recorded_at_id
+    ON public.strike_price (recorded_at, id);
+-- kraken_minute already has a ts primary-key index.

sql/s3-runtime-policy.json

@@ -0,0 +1,34 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetBucketVersioning"],
+      "Resource": "arn:aws:s3:::btcdata-archive"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"],
+      "Resource": "arn:aws:s3:::btcdata-archive",
+      "Condition": {"StringLike": {"s3:prefix": ["btcprod/weeks/", "btcprod/weeks/*"]}}
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["s3:GetObject", "s3:GetObjectVersion", "s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"],
+      "Resource": "arn:aws:s3:::btcdata-archive/btcprod/weeks/*"
+    },
+    {
+      "Effect": "Deny",
+      "Action": ["admin:*"]
+    },
+    {
+      "Effect": "Deny",
+      "Action": ["sts:*"]
+    },
+    {
+      "Effect": "Deny",
+      "Action": ["s3:DeleteObject", "s3:DeleteObjectVersion", "s3:DeleteBucket", "s3:PutBucketVersioning"],
+      "Resource": ["arn:aws:s3:::*", "arn:aws:s3:::*/*"]
+    }
+  ]
+}

sql/setup.sql

@@ -0,0 +1,30 @@
+-- Operator-run only: psql -v ON_ERROR_STOP=1 -v runtime_role=btcdata_archiver -f sql/setup.sql
+-- Create the login separately with a password supplied through secure operator tooling.
+BEGIN;
+CREATE SCHEMA IF NOT EXISTS btcdata_archive;
+CREATE TABLE IF NOT EXISTS btcdata_archive.weeks (
+    week_start timestamptz PRIMARY KEY,
+    week_end timestamptz NOT NULL,
+    format_version integer NOT NULL CHECK (format_version = 1),
+    status text NOT NULL CHECK (status IN ('exporting','verified','pruning','complete','failed')),
+    object_key text NOT NULL,
+    object_version_id text,
+    file_sha256 text CHECK (file_sha256 ~ '^[0-9a-f]{64}$'),
+    row_counts jsonb,
+    deletion_counts jsonb,
+    attempt_time timestamptz NOT NULL,
+    completion_time timestamptz,
+    last_error text,
+    CHECK (week_end - week_start = interval '168 hours'),
+    CHECK (extract(isodow FROM week_start AT TIME ZONE 'UTC') = 1),
+    CHECK ((week_start AT TIME ZONE 'UTC')::time = time '00:00:00'),
+    CHECK (status NOT IN ('verified','pruning','complete') OR
+           (object_version_id IS NOT NULL AND object_version_id <> 'null' AND file_sha256 IS NOT NULL))
+);
+REVOKE ALL ON SCHEMA btcdata_archive FROM PUBLIC;
+REVOKE ALL ON btcdata_archive.weeks FROM PUBLIC;
+GRANT CONNECT, TEMPORARY ON DATABASE :"DBNAME" TO :"runtime_role";
+GRANT USAGE ON SCHEMA public, btcdata_archive TO :"runtime_role";
+GRANT SELECT, DELETE ON public.binance_price, public.strike_price, public.kraken_minute TO :"runtime_role";
+GRANT SELECT, INSERT, UPDATE ON btcdata_archive.weeks TO :"runtime_role";
+COMMIT;