cae64167eb
- migrations 21->23: dedicated `tickets` schema (tickets.inc / tickets.crq, raw-jsonb-first), geo_clusters + geo_locations gazetteers, geom-resolution trigger (feed -> location -> cluster -> none), reporting.fn_tickets_for_map - dashboard_api: GET /webhook/tickets (INC/CRQ GeoJSON for the FleetOps map) - tools/import_tickets.py: raw-first bucket ingest + cluster/location geocoding (LocationIQ/OpenCage, viewbox-bounded with a cluster-distance sanity guard) - docs/CONNECTIONS.md: geocoder env var names Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
309 lines
14 KiB
PL/PgSQL
309 lines
14 KiB
PL/PgSQL
-- 22_tickets_schema.sql
|
|
-- Promote the ticket layer into its own `tickets` schema and SPLIT by type:
|
|
-- migration-21 tracksolid.tickets (one table, service_type discriminator) -->
|
|
-- tickets.inc + tickets.crq (one table per ticket type)
|
|
-- tracksolid.geo_clusters --> tickets.geo_clusters (shared gazetteer)
|
|
--
|
|
-- The read function reporting.fn_tickets_for_map keeps its SAME signature
|
|
-- (so dashboard_api GET /webhook/tickets and the FleetOps map are unchanged) but
|
|
-- now UNIONs the two per-type tables. The tracksolid.* objects from migration 21
|
|
-- are dropped after the data is copied across. Safe to re-apply.
|
|
--
|
|
-- NOTE: this intentionally introduces a schema beyond `tracksolid`/`reporting`
|
|
-- (a deliberate exception to the single-live-schema convention) to give tickets
|
|
-- their own namespace + access boundary.
|
|
|
|
CREATE SCHEMA IF NOT EXISTS tickets;
|
|
SET search_path = tickets, public;
|
|
|
|
-- ── normalize helper (cluster_key) ────────────────────────────────────────────
|
|
CREATE OR REPLACE FUNCTION tickets.norm_cluster(p text)
|
|
RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE
|
|
AS $fn$ SELECT NULLIF(upper(regexp_replace(trim(COALESCE(p, '')), '\s+', ' ', 'g')), '') $fn$;
|
|
|
|
-- ── gazetteer: move from tracksolid if present, else create fresh ─────────────
|
|
DO $mv$
|
|
BEGIN
|
|
IF to_regclass('tracksolid.geo_clusters') IS NOT NULL
|
|
AND to_regclass('tickets.geo_clusters') IS NULL THEN
|
|
ALTER TABLE tracksolid.geo_clusters SET SCHEMA tickets;
|
|
END IF;
|
|
END $mv$;
|
|
|
|
CREATE TABLE IF NOT EXISTS tickets.geo_clusters (
|
|
cluster_key text PRIMARY KEY,
|
|
region text,
|
|
lat double precision,
|
|
lng double precision,
|
|
geom geometry(Point, 4326),
|
|
source text,
|
|
verified boolean NOT NULL DEFAULT false,
|
|
updated_at timestamptz NOT NULL DEFAULT now()
|
|
);
|
|
|
|
CREATE OR REPLACE FUNCTION tickets.tg_geo_clusters_geom()
|
|
RETURNS trigger LANGUAGE plpgsql AS $fn$
|
|
BEGIN
|
|
IF NEW.lat IS NOT NULL AND NEW.lng IS NOT NULL
|
|
AND NEW.lat BETWEEN -90 AND 90 AND NEW.lng BETWEEN -180 AND 180
|
|
AND NOT (NEW.lat = 0 AND NEW.lng = 0) THEN
|
|
NEW.geom := ST_SetSRID(ST_MakePoint(NEW.lng, NEW.lat), 4326);
|
|
ELSE
|
|
NEW.geom := NULL;
|
|
END IF;
|
|
NEW.updated_at := now();
|
|
RETURN NEW;
|
|
END $fn$;
|
|
DROP TRIGGER IF EXISTS trg_geo_clusters_geom ON tickets.geo_clusters;
|
|
CREATE TRIGGER trg_geo_clusters_geom BEFORE INSERT OR UPDATE ON tickets.geo_clusters
|
|
FOR EACH ROW EXECUTE FUNCTION tickets.tg_geo_clusters_geom();
|
|
|
|
-- ── per-type ticket tables (identical shape; CHECK-locked to their type) ──────
|
|
CREATE TABLE IF NOT EXISTS tickets.inc (
|
|
ticket_id text PRIMARY KEY,
|
|
source_type text,
|
|
service_type text,
|
|
bucket text,
|
|
raw_status text,
|
|
normalized_status text,
|
|
created_at_service timestamptz,
|
|
scheduled_at timestamptz,
|
|
closed_at timestamptz,
|
|
last_seen_at timestamptz,
|
|
first_seen_at timestamptz,
|
|
week_start date,
|
|
week_end date,
|
|
cluster text,
|
|
region text,
|
|
location_name text,
|
|
latitude double precision,
|
|
longitude double precision,
|
|
department text,
|
|
assigned_team text,
|
|
owner text,
|
|
sla_status text,
|
|
mttr double precision,
|
|
is_auto_created boolean,
|
|
is_auto_closed boolean,
|
|
is_alarm boolean,
|
|
is_actionable boolean,
|
|
source_s3_bucket text,
|
|
source_s3_key text,
|
|
source_snapshot_id bigint,
|
|
created_at timestamptz,
|
|
updated_at timestamptz,
|
|
geom geometry(Point, 4326),
|
|
geo_source text,
|
|
raw jsonb,
|
|
ingested_at timestamptz NOT NULL DEFAULT now(),
|
|
CONSTRAINT inc_service_type_chk CHECK (service_type = 'inc')
|
|
);
|
|
|
|
-- crq mirrors inc's columns/defaults; add its own PK + type CHECK.
|
|
CREATE TABLE IF NOT EXISTS tickets.crq (LIKE tickets.inc INCLUDING DEFAULTS);
|
|
DO $crq$
|
|
BEGIN
|
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint
|
|
WHERE conrelid = 'tickets.crq'::regclass AND contype = 'p') THEN
|
|
ALTER TABLE tickets.crq ADD PRIMARY KEY (ticket_id);
|
|
END IF;
|
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint
|
|
WHERE conrelid = 'tickets.crq'::regclass AND conname = 'crq_service_type_chk') THEN
|
|
ALTER TABLE tickets.crq ADD CONSTRAINT crq_service_type_chk CHECK (service_type = 'crq');
|
|
END IF;
|
|
END $crq$;
|
|
|
|
-- ── shared geom-resolution trigger (feed coords → cluster gazetteer → none) ───
|
|
CREATE OR REPLACE FUNCTION tickets.tg_ticket_geom()
|
|
RETURNS trigger LANGUAGE plpgsql AS $fn$
|
|
DECLARE g geometry(Point, 4326);
|
|
BEGIN
|
|
IF NEW.latitude IS NOT NULL AND NEW.longitude IS NOT NULL
|
|
AND NEW.latitude BETWEEN -90 AND 90
|
|
AND NEW.longitude BETWEEN -180 AND 180
|
|
AND NOT (NEW.latitude = 0 AND NEW.longitude = 0) THEN
|
|
NEW.geom := ST_SetSRID(ST_MakePoint(NEW.longitude, NEW.latitude), 4326);
|
|
NEW.geo_source := 'feed';
|
|
ELSE
|
|
SELECT gc.geom INTO g FROM tickets.geo_clusters gc
|
|
WHERE gc.cluster_key = tickets.norm_cluster(NEW.cluster) AND gc.geom IS NOT NULL LIMIT 1;
|
|
IF g IS NOT NULL THEN NEW.geom := g; NEW.geo_source := 'cluster';
|
|
ELSE NEW.geom := NULL; NEW.geo_source := 'none'; END IF;
|
|
END IF;
|
|
RETURN NEW;
|
|
END $fn$;
|
|
|
|
DROP TRIGGER IF EXISTS trg_inc_geom ON tickets.inc;
|
|
CREATE TRIGGER trg_inc_geom BEFORE INSERT OR UPDATE ON tickets.inc
|
|
FOR EACH ROW EXECUTE FUNCTION tickets.tg_ticket_geom();
|
|
DROP TRIGGER IF EXISTS trg_crq_geom ON tickets.crq;
|
|
CREATE TRIGGER trg_crq_geom BEFORE INSERT OR UPDATE ON tickets.crq
|
|
FOR EACH ROW EXECUTE FUNCTION tickets.tg_ticket_geom();
|
|
|
|
-- re-resolve geoms across both tables after the gazetteer is (re)seeded.
|
|
CREATE OR REPLACE FUNCTION tickets.resolve_ticket_geoms()
|
|
RETURNS integer LANGUAGE plpgsql AS $fn$
|
|
DECLARE n integer; m integer;
|
|
BEGIN
|
|
UPDATE tickets.inc t SET geom = gc.geom, geo_source = 'cluster'
|
|
FROM tickets.geo_clusters gc
|
|
WHERE gc.cluster_key = tickets.norm_cluster(t.cluster) AND gc.geom IS NOT NULL
|
|
AND t.geo_source IS DISTINCT FROM 'feed' AND t.geom IS DISTINCT FROM gc.geom;
|
|
GET DIAGNOSTICS n = ROW_COUNT;
|
|
UPDATE tickets.crq t SET geom = gc.geom, geo_source = 'cluster'
|
|
FROM tickets.geo_clusters gc
|
|
WHERE gc.cluster_key = tickets.norm_cluster(t.cluster) AND gc.geom IS NOT NULL
|
|
AND t.geo_source IS DISTINCT FROM 'feed' AND t.geom IS DISTINCT FROM gc.geom;
|
|
GET DIAGNOSTICS m = ROW_COUNT;
|
|
RETURN n + m;
|
|
END $fn$;
|
|
|
|
-- ── indexes (per table) ───────────────────────────────────────────────────────
|
|
CREATE INDEX IF NOT EXISTS ix_inc_status ON tickets.inc (normalized_status);
|
|
CREATE INDEX IF NOT EXISTS ix_inc_actionable ON tickets.inc (is_actionable) WHERE is_actionable;
|
|
CREATE INDEX IF NOT EXISTS ix_inc_geom ON tickets.inc USING gist (geom);
|
|
CREATE INDEX IF NOT EXISTS ix_inc_cluster ON tickets.inc (tickets.norm_cluster(cluster));
|
|
CREATE INDEX IF NOT EXISTS ix_crq_status ON tickets.crq (normalized_status);
|
|
CREATE INDEX IF NOT EXISTS ix_crq_actionable ON tickets.crq (is_actionable) WHERE is_actionable;
|
|
CREATE INDEX IF NOT EXISTS ix_crq_geom ON tickets.crq USING gist (geom);
|
|
CREATE INDEX IF NOT EXISTS ix_crq_cluster ON tickets.crq (tickets.norm_cluster(cluster));
|
|
|
|
-- ── backfill from migration-21 tracksolid.tickets, then retire it ─────────────
|
|
DO $bf$
|
|
BEGIN
|
|
IF to_regclass('tracksolid.tickets') IS NOT NULL THEN
|
|
INSERT INTO tickets.inc (
|
|
ticket_id, source_type, service_type, bucket, raw_status, normalized_status,
|
|
created_at_service, scheduled_at, closed_at, last_seen_at, first_seen_at,
|
|
week_start, week_end, cluster, region, location_name, latitude, longitude,
|
|
department, assigned_team, owner, sla_status, mttr, is_auto_created,
|
|
is_auto_closed, is_alarm, is_actionable, source_s3_bucket, source_s3_key,
|
|
source_snapshot_id, created_at, updated_at, raw)
|
|
SELECT
|
|
ticket_id, source_type, service_type, bucket, raw_status, normalized_status,
|
|
created_at_service, scheduled_at, closed_at, last_seen_at, first_seen_at,
|
|
week_start, week_end, cluster, region, location_name, latitude, longitude,
|
|
department, assigned_team, owner, sla_status, mttr, is_auto_created,
|
|
is_auto_closed, is_alarm, is_actionable, source_s3_bucket, source_s3_key,
|
|
source_snapshot_id, created_at, updated_at, raw
|
|
FROM tracksolid.tickets WHERE service_type = 'inc'
|
|
ON CONFLICT (ticket_id) DO NOTHING;
|
|
|
|
INSERT INTO tickets.crq (
|
|
ticket_id, source_type, service_type, bucket, raw_status, normalized_status,
|
|
created_at_service, scheduled_at, closed_at, last_seen_at, first_seen_at,
|
|
week_start, week_end, cluster, region, location_name, latitude, longitude,
|
|
department, assigned_team, owner, sla_status, mttr, is_auto_created,
|
|
is_auto_closed, is_alarm, is_actionable, source_s3_bucket, source_s3_key,
|
|
source_snapshot_id, created_at, updated_at, raw)
|
|
SELECT
|
|
ticket_id, source_type, service_type, bucket, raw_status, normalized_status,
|
|
created_at_service, scheduled_at, closed_at, last_seen_at, first_seen_at,
|
|
week_start, week_end, cluster, region, location_name, latitude, longitude,
|
|
department, assigned_team, owner, sla_status, mttr, is_auto_created,
|
|
is_auto_closed, is_alarm, is_actionable, source_s3_bucket, source_s3_key,
|
|
source_snapshot_id, created_at, updated_at, raw
|
|
FROM tracksolid.tickets WHERE service_type = 'crq'
|
|
ON CONFLICT (ticket_id) DO NOTHING;
|
|
END IF;
|
|
END $bf$;
|
|
|
|
-- retire migration-21 objects (data now lives in tickets.*)
|
|
DROP TABLE IF EXISTS tracksolid.tickets;
|
|
DROP FUNCTION IF EXISTS tracksolid.resolve_ticket_geoms();
|
|
DROP FUNCTION IF EXISTS tracksolid.tg_tickets_geom();
|
|
DROP FUNCTION IF EXISTS tracksolid.tg_geo_clusters_geom();
|
|
DROP FUNCTION IF EXISTS tracksolid.norm_cluster(text);
|
|
|
|
-- ── read function (same signature; now UNIONs the two tables) ─────────────────
|
|
CREATE OR REPLACE FUNCTION reporting.fn_tickets_for_map(
|
|
p_service_type text DEFAULT NULL,
|
|
p_status text DEFAULT NULL,
|
|
p_open_only boolean DEFAULT true
|
|
)
|
|
RETURNS jsonb LANGUAGE plpgsql STABLE AS $fn$
|
|
DECLARE v_result jsonb;
|
|
BEGIN
|
|
p_service_type := lower(NULLIF(p_service_type, ''));
|
|
p_status := NULLIF(p_status, '');
|
|
WITH filtered AS (
|
|
SELECT ticket_id, service_type, normalized_status, raw_status, cluster, region,
|
|
location_name, department, owner, assigned_team, sla_status, is_actionable,
|
|
geo_source, created_at_service, scheduled_at, geom
|
|
FROM tickets.inc
|
|
WHERE geom IS NOT NULL
|
|
AND (p_service_type IS NULL OR p_service_type = 'inc')
|
|
AND (p_status IS NULL OR normalized_status = p_status)
|
|
AND (NOT p_open_only OR is_actionable IS TRUE)
|
|
UNION ALL
|
|
SELECT ticket_id, service_type, normalized_status, raw_status, cluster, region,
|
|
location_name, department, owner, assigned_team, sla_status, is_actionable,
|
|
geo_source, created_at_service, scheduled_at, geom
|
|
FROM tickets.crq
|
|
WHERE geom IS NOT NULL
|
|
AND (p_service_type IS NULL OR p_service_type = 'crq')
|
|
AND (p_status IS NULL OR normalized_status = p_status)
|
|
AND (NOT p_open_only OR is_actionable IS TRUE)
|
|
)
|
|
SELECT jsonb_build_object(
|
|
'summary', jsonb_build_object(
|
|
'ticket_count', COUNT(*),
|
|
'inc', COUNT(*) FILTER (WHERE service_type = 'inc'),
|
|
'crq', COUNT(*) FILTER (WHERE service_type = 'crq'),
|
|
'open', COUNT(*) FILTER (WHERE is_actionable IS TRUE),
|
|
'by_status', (SELECT jsonb_object_agg(s, c)
|
|
FROM (SELECT normalized_status AS s, COUNT(*) AS c
|
|
FROM filtered GROUP BY normalized_status) z)
|
|
),
|
|
'geojson', jsonb_build_object(
|
|
'type', 'FeatureCollection',
|
|
'features', COALESCE(jsonb_agg(
|
|
jsonb_build_object(
|
|
'type', 'Feature',
|
|
'properties', jsonb_build_object(
|
|
'ticket_id', ticket_id,
|
|
'service_type', service_type,
|
|
'status', normalized_status,
|
|
'raw_status', raw_status,
|
|
'cluster', cluster,
|
|
'region', region,
|
|
'location_name', location_name,
|
|
'department', department,
|
|
'owner', owner,
|
|
'assigned_team', assigned_team,
|
|
'sla_status', sla_status,
|
|
'is_actionable', is_actionable,
|
|
'geo_source', geo_source,
|
|
'created_at', to_char(created_at_service, 'YYYY-MM-DD HH24:MI:SS'),
|
|
'scheduled_at', to_char(scheduled_at, 'YYYY-MM-DD HH24:MI:SS')
|
|
),
|
|
'geometry', ST_AsGeoJSON(geom)::jsonb
|
|
)
|
|
), '[]'::jsonb)
|
|
)
|
|
) INTO v_result FROM filtered;
|
|
RETURN v_result;
|
|
END $fn$;
|
|
|
|
COMMENT ON FUNCTION reporting.fn_tickets_for_map(text, text, boolean) IS
|
|
'INC/CRQ tickets (tickets.inc + tickets.crq) as GeoJSON for the FleetOps map. Migration 22.';
|
|
|
|
-- ── grants ────────────────────────────────────────────────────────────────────
|
|
DO $grants$
|
|
BEGIN
|
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tracksolid_owner') THEN
|
|
GRANT USAGE, CREATE ON SCHEMA tickets TO tracksolid_owner;
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA tickets TO tracksolid_owner;
|
|
GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA tickets TO tracksolid_owner;
|
|
END IF;
|
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'dashboard_ro') THEN
|
|
GRANT USAGE ON SCHEMA tickets TO dashboard_ro;
|
|
GRANT SELECT ON tickets.inc, tickets.crq, tickets.geo_clusters TO dashboard_ro;
|
|
GRANT EXECUTE ON FUNCTION reporting.fn_tickets_for_map(text, text, boolean) TO dashboard_ro;
|
|
END IF;
|
|
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'grafana_ro') THEN
|
|
GRANT USAGE ON SCHEMA tickets TO grafana_ro;
|
|
GRANT SELECT ON tickets.inc, tickets.crq, tickets.geo_clusters TO grafana_ro;
|
|
GRANT EXECUTE ON FUNCTION reporting.fn_tickets_for_map(text, text, boolean) TO grafana_ro;
|
|
END IF;
|
|
END $grants$;
|