From 066d866b905b84c792f6ade1e1141dd20b83b2fa Mon Sep 17 00:00:00 2001 From: david kiania Date: Thu, 25 Jun 2026 23:55:17 +0300 Subject: [PATCH] fix(crq): migration 15 creates tickets.crq (live DB never materialized it) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-DB reconciliation before seeding CRQ revealed two divergences: - tickets.crq did NOT exist: 01_tickets_schema.sql was applied 2026-06-15 from a version predating its crq section, so the IF-NOT-EXISTS ledger guard has blocked it ever since (fn_tickets_for_map + resolve_ticket_geoms already reference crq, so they errored if called — masked because the live INC view uses fn_inc_dashboard). - The live ledger carries un-versioned 13_inc_search_fn.sql / 14_inc_filter_options.sql (applied 2026-06-19, absent from this repo). So 13_crq_columns.sql (ALTER-only, number 13) is replaced by 15_crq_table.sql, which CREATEs tickets.crq self-containedly (table + geom trigger + raw/typed indexes) and adds the typed STORED generated columns. Deterministic + idempotent on both the live DB (crq missing) and a fresh DB (crq minimal from 01). Numbered 15 to sit after the live ledger's max. Docs/CLI references updated 13->15. Applied + seeded on the live DB out-of-band (running container, INC image untouched): 39,240 crq rows, 99.99% geocoded (cluster + shared location cache), watermark current, crq now renders on fn_tickets_for_map. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 +- crq/import_crq.py | 2 +- docs/deployment-and-operations.md | 10 +-- docs/implementation.md | 4 +- migrations/13_crq_columns.sql | 55 ---------------- migrations/15_crq_table.sql | 101 ++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 64 deletions(-) delete mode 100644 migrations/13_crq_columns.sql create mode 100644 migrations/15_crq_table.sql diff --git a/README.md b/README.md index f539506..11e6ab2 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ and is driven from the INC entrypoint. | `migrations/08_inc_open_sla_view.sql` | `tickets.inc_open_sla` view — open (`is_actionable`) tickets with **derived SLA** (`hours_open`, `sla_state` vs 48h; clock = `created_at_service` ∥ `first_seen_at`), plus team/cluster/`geog` for dispatch | | `migrations/09_inc_dashboard_fn.sql` | `reporting.fn_inc_dashboard(cluster, status, window, from, to)` — one JSON payload (`window` / `open` GeoJSON / `closed` GeoJSON / `metrics` / `freshness`) powering the FleetOps live INC map. Open=live, closed=windowed (EAT calendar / custom); filters AND | | `migrations/10_inc_history_capture.sql` | History for time-series: `tickets.closure_events` (append-only observed closures) + `tickets.inc_daily_snapshot` (per-EAT-day open backlog + flow), populated by `tickets.capture_history()` each ingest. Unlocks **backlog-over-time** | -| `migrations/13_crq_columns.sql` | CRQ mirror of `03`: unpacks `tickets.crq.raw` into the same **typed STORED generated columns** + indexes (reuses `tickets.eat_ts()`). Brings CRQ to data-layer parity with INC | +| `migrations/15_crq_table.sql` | **Materializes `tickets.crq`** (table + geom trigger + indexes — `01`'s crq section never ran on the live DB) and unpacks `raw` into the same **typed STORED generated columns** as INC's `03` (reuses `tickets.eat_ts()`). Brings CRQ to data-layer parity | | `pipeline.py` | **Shared engine** — the dataset-agnostic CDC loader (drains `automations//changes/.csv` from the `isptickets` bucket, upserts on `ticket_id` oldest→newest, watermark + per-file archive) and the **cross-dataset** geocoder (clusters + actionable inc/crq locations) | | `inc/import_inc.py` | INC entrypoint (`python -m inc.import_inc`) — INC `Dataset` config + CLI; runs `tickets.capture_history()` after each `--apply`; hosts the shared geocode commands | | `crq/import_crq.py` | CRQ entrypoint (`python -m crq.import_crq`) — CRQ `Dataset` config + CLI (ingest only; no history hook yet) | @@ -241,7 +241,7 @@ Live: INC ingestion deployed on Coolify (every 20 min `*/20 6-20 * * *` EAT), sc generated columns + geocoding + the `inc_open_sla` view in `tracksolid_db`. **CRQ (this milestone):** data layer + map — `tickets.crq` fed from -`automations/crq/changes/` by `crq/import_crq.py`, typed columns (migration 13), +`automations/crq/changes/` by `crq/import_crq.py`, the `tickets.crq` table + typed columns (migration 15), cross-dataset geocoding, and visibility on the Tickets map via `fn_tickets_for_map`. One-time seed: drain the isptickets CRQ stream (`python -m crq.import_crq --from-bucket --apply`) — empty watermark + the stream's periodic full-state snapshots converge to diff --git a/crq/import_crq.py b/crq/import_crq.py index 6539bb9..1d8c49e 100644 --- a/crq/import_crq.py +++ b/crq/import_crq.py @@ -23,7 +23,7 @@ Usage (needs DATABASE_URL + RUSTFS_* env; see .env.example): python -m crq.import_crq --crq-csv 2026-06-24T12-55-44.csv --apply Pre-requisite: migrations applied (run_migrations.py) — tickets.crq + its typed -columns (13_crq_columns.sql) + geo_clusters/geo_locations + fn_tickets_for_map. +columns (15_crq_table.sql) + geo_clusters/geo_locations + fn_tickets_for_map. ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ """ diff --git a/docs/deployment-and-operations.md b/docs/deployment-and-operations.md index 306452f..01e52a3 100644 --- a/docs/deployment-and-operations.md +++ b/docs/deployment-and-operations.md @@ -152,12 +152,14 @@ If the provider moves the INC feed to a new bucket (as happened `tickets` → `i ## Bringing CRQ online (one-time seed) -CRQ was added 2026-06-25 (data layer + map). To seed `tickets.crq` from zero on the live -DB — once the code + migration `13_crq_columns.sql` are deployed (`run_migrations.py` -applies it on build): +CRQ was added 2026-06-25 (data layer + map). Migration `15_crq_table.sql` **creates** +`tickets.crq` (the live DB's `01` predated its crq section, so the table never existed) +plus the typed columns. To seed it from zero on the live DB — once the code + migration are +applied (`run_migrations.py`; on the live cutover it was applied out-of-band via the running +container, see below): 1. **Verify** the migration applied: `SELECT 1 FROM tickets.schema_migrations WHERE - filename='13_crq_columns.sql';` and `\d tickets.crq` shows the typed columns. + filename='15_crq_table.sql';` and `\d tickets.crq` shows the table + typed columns. 2. **Seed** from isptickets (empty `crq` watermark → drains all `automations/crq/changes/` files oldest→newest; the stream's periodic full-state snapshots converge to current state — same convergence the INC cutover relied on, so **no `--reseed` needed**): diff --git a/docs/implementation.md b/docs/implementation.md index 25563e5..c917df4 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -54,7 +54,7 @@ from the INC entrypoint. | 09_inc_dashboard_fn | **built** — `reporting.fn_inc_dashboard(cluster, status, window, from, to)`: one JSON payload (open GeoJSON + windowed closed GeoJSON + metrics + freshness) for the FleetOps live INC map. See `docs/phase-2-dashboard.md` | | 10_inc_history_capture | **built** — `tickets.closure_events` (append-only observed closures) + `tickets.inc_daily_snapshot` (per-EAT-day open backlog + flow) + `tickets.capture_history()`; the ingest calls it each `--apply` run. Unlocks backlog-over-time | | 12_inc_dashboard_by_owner | **built** — owner/team breakdown extension to `fn_inc_dashboard` | -| 13_crq_columns | **built** — CRQ mirror of `03`: typed STORED generated columns + indexes on `tickets.crq` (reuses `tickets.eat_ts()`). Data-layer parity for the CRQ tab | +| 15_crq_table | **built** — materializes `tickets.crq` (table + geom trigger + indexes; `01`'s crq section never ran on the live DB) + the typed STORED generated columns from `03` (reuses `tickets.eat_ts()`). Data-layer parity for the CRQ tab | `tickets.inc` columns: `ticket_id` (PK), `raw` (jsonb, source of truth), `normalized_status`/`raw_status`, `bucket`, `is_actionable`, `cluster`/`region`/ @@ -110,7 +110,7 @@ repos; see `docs/dashboard-api-contract.md`), FleetNow **dispatch** off `geog`, **team closure attribution**. **CRQ** (this milestone): the shared engine now feeds `tickets.crq` from -`automations/crq/changes/` (`crq/import_crq.py`), with typed columns (migration 13) and +`automations/crq/changes/` (`crq/import_crq.py`), with the `tickets.crq` table + typed columns (migration 15) and cross-dataset geocoding — CRQ shows on the Tickets map via `fn_tickets_for_map` (which already unions it) and gets its own FleetOps tab. Deferred to a follow-up once installation-lifecycle semantics are confirmed: the CRQ analogues of migrations diff --git a/migrations/13_crq_columns.sql b/migrations/13_crq_columns.sql deleted file mode 100644 index 197e6ca..0000000 --- a/migrations/13_crq_columns.sql +++ /dev/null @@ -1,55 +0,0 @@ --- 13_crq_columns.sql — fleettickets · unpack tickets.crq.raw into typed columns --- ───────────────────────────────────────────────────────────────────────────── --- CRQ (new-installation) mirror of 03_inc_columns.sql. CRQ shares the INC source's --- IDENTICAL 32-column flat-CSV schema, so the same STORED generated columns apply: --- the crq dataset gets real typed, indexable columns while `raw` stays the source --- of truth (drift-safe). STORED generated columns are computed for ALL existing --- rows on creation and auto-recomputed on every future insert/update — no loader --- change needed. --- --- tickets.eat_ts() (EAT wall-clock text -> timestamptz, IMMUTABLE) already exists --- from 03_inc_columns.sql — reuse it, don't redefine. See that file's note on why --- IMMUTABLE is safe for Kenya (fixed UTC+3, no DST). --- --- Idempotent: safe on a fresh DB and re-appliable on the live DB. --- ───────────────────────────────────────────────────────────────────────────── - -SET search_path = tickets, public; - -ALTER TABLE tickets.crq - -- text - ADD COLUMN IF NOT EXISTS service_type text GENERATED ALWAYS AS (raw->>'service_type') STORED, - ADD COLUMN IF NOT EXISTS bucket text GENERATED ALWAYS AS (raw->>'bucket') STORED, - ADD COLUMN IF NOT EXISTS raw_status text GENERATED ALWAYS AS (raw->>'raw_status') STORED, - ADD COLUMN IF NOT EXISTS normalized_status text GENERATED ALWAYS AS (raw->>'normalized_status') STORED, - ADD COLUMN IF NOT EXISTS cluster text GENERATED ALWAYS AS (raw->>'cluster') STORED, - ADD COLUMN IF NOT EXISTS region text GENERATED ALWAYS AS (raw->>'region') STORED, - ADD COLUMN IF NOT EXISTS location_name text GENERATED ALWAYS AS (raw->>'location_name') STORED, - ADD COLUMN IF NOT EXISTS assigned_team text GENERATED ALWAYS AS (raw->>'assigned_team') STORED, - ADD COLUMN IF NOT EXISTS owner text GENERATED ALWAYS AS (raw->>'owner') STORED, - ADD COLUMN IF NOT EXISTS sla_status text GENERATED ALWAYS AS (raw->>'sla_status') STORED, - -- numeric / float - ADD COLUMN IF NOT EXISTS mttr numeric GENERATED ALWAYS AS (NULLIF(raw->>'mttr','')::numeric) STORED, - ADD COLUMN IF NOT EXISTS latitude double precision GENERATED ALWAYS AS (NULLIF(raw->>'latitude','')::double precision) STORED, - ADD COLUMN IF NOT EXISTS longitude double precision GENERATED ALWAYS AS (NULLIF(raw->>'longitude','')::double precision) STORED, - -- boolean - ADD COLUMN IF NOT EXISTS is_actionable boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_actionable','')::boolean) STORED, - ADD COLUMN IF NOT EXISTS is_auto_created boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_auto_created','')::boolean) STORED, - ADD COLUMN IF NOT EXISTS is_auto_closed boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_auto_closed','')::boolean) STORED, - ADD COLUMN IF NOT EXISTS is_alarm boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_alarm','')::boolean) STORED, - -- timestamps (EAT wall-clock -> timestamptz). created_at/updated_at are the - -- EXPORT pipeline's bookkeeping (not ticket lifecycle), hence the source_ prefix. - ADD COLUMN IF NOT EXISTS created_at_service timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'created_at_service')) STORED, - ADD COLUMN IF NOT EXISTS scheduled_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'scheduled_at')) STORED, - ADD COLUMN IF NOT EXISTS closed_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'closed_at')) STORED, - ADD COLUMN IF NOT EXISTS last_seen_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'last_seen_at')) STORED, - ADD COLUMN IF NOT EXISTS first_seen_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'first_seen_at')) STORED, - ADD COLUMN IF NOT EXISTS source_created_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'created_at')) STORED, - ADD COLUMN IF NOT EXISTS source_updated_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'updated_at')) STORED; - --- indexes on the new typed columns (serve cluster / team / closure queries) -CREATE INDEX IF NOT EXISTS ix_crq_norm_status_col ON tickets.crq (normalized_status); -CREATE INDEX IF NOT EXISTS ix_crq_cluster_col ON tickets.crq (cluster); -CREATE INDEX IF NOT EXISTS ix_crq_assigned_team ON tickets.crq (assigned_team); -CREATE INDEX IF NOT EXISTS ix_crq_closed_at ON tickets.crq (closed_at); -CREATE INDEX IF NOT EXISTS ix_crq_actionable_col ON tickets.crq (is_actionable) WHERE is_actionable; diff --git a/migrations/15_crq_table.sql b/migrations/15_crq_table.sql new file mode 100644 index 0000000..f7435a5 --- /dev/null +++ b/migrations/15_crq_table.sql @@ -0,0 +1,101 @@ +-- 15_crq_table.sql — fleettickets · materialize tickets.crq + typed columns +-- ───────────────────────────────────────────────────────────────────────────── +-- Why a NEW migration (not an edit to 01): `01_tickets_schema.sql` was applied to the +-- live DB on 2026-06-15 from a version that PREDATED its `tickets.crq` section, so the +-- IF-NOT-EXISTS ledger guard has kept crq from ever being created there — even though +-- the live `reporting.fn_tickets_for_map` and `tickets.resolve_ticket_geoms` already +-- reference it (they error if called until crq exists). This migration creates +-- `tickets.crq` self-containedly (table + geom trigger + indexes) and adds the same +-- typed STORED generated columns INC got in `03_inc_columns.sql`, bringing CRQ to +-- data-layer parity. +-- +-- Deterministic + idempotent — converges to the same shape on BOTH: +-- • the live DB (crq missing) -> CREATE makes it, ALTER adds typed cols +-- • a fresh DB (crq minimal, from 01) -> CREATE skipped, ALTER adds typed cols +-- Reuses shared objects already present: tickets.tg_ticket_geom() (01), +-- tickets.norm_cluster() (01), tickets.eat_ts() (03). +-- +-- NOTE: the live DB also carries un-versioned migrations 13_inc_search_fn.sql / +-- 14_inc_filter_options.sql (applied 2026-06-19, absent from this repo) — INC dashboard +-- functions, unrelated to CRQ. Numbered 15 here to sit cleanly after the live ledger. +-- ───────────────────────────────────────────────────────────────────────────── + +SET search_path = tickets, public; + +-- ── table (base shape mirrors tickets.inc's original 01 base) ──────────────── +CREATE TABLE IF NOT EXISTS tickets.crq ( + ticket_id text PRIMARY KEY, + raw jsonb NOT NULL, + geom geometry(Point, 4326), + geo_source text, -- 'feed' | 'location' | 'cluster' | 'none' + ingested_at timestamptz NOT NULL DEFAULT now() +); + +-- ── geom trigger — read from raw; shared tickets.tg_ticket_geom() (from 01) ─── +DROP TRIGGER IF EXISTS trg_crq_geom ON tickets.crq; +CREATE TRIGGER trg_crq_geom BEFORE INSERT OR UPDATE ON tickets.crq + FOR EACH ROW EXECUTE FUNCTION tickets.tg_ticket_geom(); + +-- ── raw-based indexes (mirror 01's inc/crq set) ────────────────────────────── +CREATE INDEX IF NOT EXISTS ix_crq_status_raw ON tickets.crq ((raw->>'normalized_status')); +CREATE INDEX IF NOT EXISTS ix_crq_actionable_raw ON tickets.crq (((raw->>'is_actionable')::boolean)) + WHERE (raw->>'is_actionable')::boolean; +CREATE INDEX IF NOT EXISTS ix_crq_cluster_raw ON tickets.crq (tickets.norm_cluster(raw->>'cluster')); +CREATE INDEX IF NOT EXISTS ix_crq_loc_raw ON tickets.crq (tickets.norm_cluster(raw->>'location_name')); +CREATE INDEX IF NOT EXISTS ix_crq_geom ON tickets.crq USING gist (geom); + +-- ── typed STORED generated columns (mirror of 03_inc_columns.sql) ──────────── +-- Computed for ALL existing rows on creation + auto-recomputed on every insert/update; +-- `raw` stays the source of truth. tickets.eat_ts() (EAT->timestamptz, IMMUTABLE) is +-- reused from 03 — see that file's note on why IMMUTABLE is safe for Kenya (UTC+3, no DST). +ALTER TABLE tickets.crq + -- text + ADD COLUMN IF NOT EXISTS service_type text GENERATED ALWAYS AS (raw->>'service_type') STORED, + ADD COLUMN IF NOT EXISTS bucket text GENERATED ALWAYS AS (raw->>'bucket') STORED, + ADD COLUMN IF NOT EXISTS raw_status text GENERATED ALWAYS AS (raw->>'raw_status') STORED, + ADD COLUMN IF NOT EXISTS normalized_status text GENERATED ALWAYS AS (raw->>'normalized_status') STORED, + ADD COLUMN IF NOT EXISTS cluster text GENERATED ALWAYS AS (raw->>'cluster') STORED, + ADD COLUMN IF NOT EXISTS region text GENERATED ALWAYS AS (raw->>'region') STORED, + ADD COLUMN IF NOT EXISTS location_name text GENERATED ALWAYS AS (raw->>'location_name') STORED, + ADD COLUMN IF NOT EXISTS assigned_team text GENERATED ALWAYS AS (raw->>'assigned_team') STORED, + ADD COLUMN IF NOT EXISTS owner text GENERATED ALWAYS AS (raw->>'owner') STORED, + ADD COLUMN IF NOT EXISTS sla_status text GENERATED ALWAYS AS (raw->>'sla_status') STORED, + -- numeric / float + ADD COLUMN IF NOT EXISTS mttr numeric GENERATED ALWAYS AS (NULLIF(raw->>'mttr','')::numeric) STORED, + ADD COLUMN IF NOT EXISTS latitude double precision GENERATED ALWAYS AS (NULLIF(raw->>'latitude','')::double precision) STORED, + ADD COLUMN IF NOT EXISTS longitude double precision GENERATED ALWAYS AS (NULLIF(raw->>'longitude','')::double precision) STORED, + -- boolean + ADD COLUMN IF NOT EXISTS is_actionable boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_actionable','')::boolean) STORED, + ADD COLUMN IF NOT EXISTS is_auto_created boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_auto_created','')::boolean) STORED, + ADD COLUMN IF NOT EXISTS is_auto_closed boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_auto_closed','')::boolean) STORED, + ADD COLUMN IF NOT EXISTS is_alarm boolean GENERATED ALWAYS AS (NULLIF(raw->>'is_alarm','')::boolean) STORED, + -- timestamps (EAT wall-clock -> timestamptz). created_at/updated_at are the EXPORT + -- pipeline's bookkeeping (not ticket lifecycle), hence the source_ prefix. + ADD COLUMN IF NOT EXISTS created_at_service timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'created_at_service')) STORED, + ADD COLUMN IF NOT EXISTS scheduled_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'scheduled_at')) STORED, + ADD COLUMN IF NOT EXISTS closed_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'closed_at')) STORED, + ADD COLUMN IF NOT EXISTS last_seen_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'last_seen_at')) STORED, + ADD COLUMN IF NOT EXISTS first_seen_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'first_seen_at')) STORED, + ADD COLUMN IF NOT EXISTS source_created_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'created_at')) STORED, + ADD COLUMN IF NOT EXISTS source_updated_at timestamptz GENERATED ALWAYS AS (tickets.eat_ts(raw->>'updated_at')) STORED; + +-- ── typed-column indexes (serve cluster / team / closure queries) ──────────── +CREATE INDEX IF NOT EXISTS ix_crq_norm_status_col ON tickets.crq (normalized_status); +CREATE INDEX IF NOT EXISTS ix_crq_cluster_col ON tickets.crq (cluster); +CREATE INDEX IF NOT EXISTS ix_crq_assigned_team ON tickets.crq (assigned_team); +CREATE INDEX IF NOT EXISTS ix_crq_closed_at ON tickets.crq (closed_at); +CREATE INDEX IF NOT EXISTS ix_crq_actionable_col ON tickets.crq (is_actionable) WHERE is_actionable; + +-- ── grants (guarded: roles may not exist on a fresh DB) ────────────────────── +DO $grants$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tracksolid_owner') THEN + GRANT SELECT, INSERT, UPDATE, DELETE ON tickets.crq TO tracksolid_owner; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'dashboard_ro') THEN + GRANT SELECT ON tickets.crq TO dashboard_ro; + END IF; + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro') THEN + GRANT SELECT ON tickets.crq TO grafana_ro; + END IF; +END $grants$;