tracksolid_timescale_grafana_prod/dwh/260423_dwh_ddl_v1.sql

-- =============================================================
-- TRACKSOLID DWH SETUP & PERMISSIONS
-- Target Database: tracksolid_dwh
-- =============================================================

-- 1. EXTENSIONS
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE EXTENSION IF NOT EXISTS btree_gist;
CREATE EXTENSION IF NOT EXISTS postgis; -- REQUIRED for geometry(Point,4326) columns

-- 2. ROLE CREATION (Idempotent)
-- SECURITY: Passwords below are placeholders. Before applying this file:
--   1. Generate two strong secrets (e.g. `openssl rand -hex 24`)
--   2. Replace both CHANGE_ME_BEFORE_APPLY tokens in-session (do NOT commit real values)
--   3. Store the generated secrets in the n8n / Grafana credential stores only
-- Rotation: `ALTER ROLE <role> PASSWORD '<new secret>'` as a superuser.
DO $$
BEGIN
    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'dwh_owner') THEN
        CREATE ROLE dwh_owner WITH LOGIN PASSWORD 'CHANGE_ME_BEFORE_APPLY';
    END IF;
    IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'grafana_ro') THEN
        CREATE ROLE grafana_ro WITH LOGIN PASSWORD 'CHANGE_ME_BEFORE_APPLY';
    END IF;
END$$;

-- Grant database connection
GRANT CONNECT ON DATABASE tracksolid_dwh TO dwh_owner;
GRANT CONNECT ON DATABASE tracksolid_dwh TO grafana_ro;

-- 3. SCHEMAS
CREATE SCHEMA IF NOT EXISTS bronze AUTHORIZATION dwh_owner;
CREATE SCHEMA IF NOT EXISTS silver AUTHORIZATION dwh_owner;
CREATE SCHEMA IF NOT EXISTS gold AUTHORIZATION dwh_owner;

ALTER DATABASE tracksolid_dwh SET search_path TO bronze, silver, gold, public;

-- 4. PERMISSIONS & DEFAULT PRIVILEGES (Critical for Security & Automation)
-- Schema access
GRANT USAGE, CREATE ON SCHEMA bronze TO dwh_owner;
GRANT USAGE, CREATE ON SCHEMA silver TO dwh_owner;
GRANT USAGE, CREATE ON SCHEMA gold TO dwh_owner;
GRANT USAGE ON SCHEMA bronze TO grafana_ro;
GRANT USAGE ON SCHEMA silver TO grafana_ro;
GRANT USAGE ON SCHEMA gold TO grafana_ro;
GRANT USAGE ON SCHEMA public TO dwh_owner, grafana_ro;

-- Existing table access for Grafana
GRANT SELECT ON ALL TABLES IN SCHEMA bronze TO grafana_ro;
GRANT SELECT ON ALL TABLES IN SCHEMA silver TO grafana_ro;
GRANT SELECT ON ALL TABLES IN SCHEMA gold TO grafana_ro;

-- FUTURE table access: Any table created by dwh_owner will automatically be readable by grafana_ro
ALTER DEFAULT PRIVILEGES FOR ROLE dwh_owner IN SCHEMA bronze GRANT SELECT ON TABLES TO grafana_ro;
ALTER DEFAULT PRIVILEGES FOR ROLE dwh_owner IN SCHEMA silver GRANT SELECT ON TABLES TO grafana_ro;
ALTER DEFAULT PRIVILEGES FOR ROLE dwh_owner IN SCHEMA gold GRANT SELECT ON TABLES TO grafana_ro;

-- 5. BRONZE SCHEMA TABLES
-- Run as dwh_owner to ensure correct ownership & default privileges apply
SET ROLE dwh_owner;
SET search_path TO bronze, public;

-- 5.1 DEVICES (Slowly Changing Dimension - Type 2 handled in Silver)
CREATE TABLE IF NOT EXISTS bronze.devices (
    imei TEXT PRIMARY KEY,
    device_name TEXT,
    mc_type TEXT,
    mc_type_use_scope TEXT,
    vehicle_name TEXT,
    vehicle_number TEXT,
    vehicle_models TEXT,
    vehicle_icon TEXT,
    vin TEXT,
    engine_number TEXT,
    vehicle_brand TEXT,
    fuel_100km NUMERIC(6,2),
    driver_name TEXT,
    driver_phone TEXT,
    sim TEXT,
    iccid TEXT,
    imsi TEXT,
    account TEXT,
    customer_name TEXT,
    device_group_id TEXT,
    device_group TEXT,
    activation_time TIMESTAMPTZ,
    expiration TIMESTAMPTZ,
    enabled_flag SMALLINT DEFAULT 1 NOT NULL,
    status TEXT DEFAULT 'active'::text,
    city TEXT,
    current_mileage_km NUMERIC(12,2),
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    last_synced_at TIMESTAMPTZ,
    vehicle_category TEXT,
    cost_centre TEXT,
    assigned_route TEXT,
    depot_geom geometry(Point,4326),
    depot_address TEXT,
    assigned_city TEXT,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.2 POSITION HISTORY (High-volume fact table)
CREATE TABLE IF NOT EXISTS bronze.position_history (
    imei TEXT NOT NULL,
    gps_time TIMESTAMPTZ NOT NULL,
    geom geometry(Point,4326),
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    speed NUMERIC(7,2),
    direction NUMERIC(6,2),
    acc_status TEXT,
    satellite SMALLINT,
    current_mileage NUMERIC(12,2),
    recorded_at TIMESTAMPTZ DEFAULT now(),
    altitude NUMERIC(8,2),
    post_type SMALLINT,
    source TEXT DEFAULT 'poll'::text,
    ingested_at TIMESTAMPTZ DEFAULT NOW(),
    PRIMARY KEY (imei, gps_time)
);

-- 5.3 TRIPS (Aggregated fact table)
CREATE TABLE IF NOT EXISTS bronze.trips (
    id BIGINT NOT NULL,
    imei TEXT NOT NULL,
    start_time TIMESTAMPTZ NOT NULL,
    end_time TIMESTAMPTZ,
    start_geom geometry(Point,4326),
    end_geom geometry(Point,4326),
    distance_km NUMERIC(12,2),
    avg_speed_kmh NUMERIC(7,2),
    max_speed_kmh NUMERIC(7,2),
    updated_at TIMESTAMPTZ DEFAULT now(),
    fuel_consumed_l NUMERIC(8,2),
    idle_time_s INTEGER,
    driving_time_s INTEGER,
    trip_seq INTEGER,
    source TEXT DEFAULT 'poll'::text,
    ingested_at TIMESTAMPTZ DEFAULT NOW(),
    PRIMARY KEY (id)
);

-- 5.4 ALARMS (Event log fact table)
CREATE TABLE IF NOT EXISTS bronze.alarms (
    id BIGINT PRIMARY KEY,
    imei TEXT,
    alarm_type TEXT,
    alarm_time TIMESTAMPTZ,
    geom geometry(Point,4326),
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    speed NUMERIC(7,2),
    acc_status TEXT,
    updated_at TIMESTAMPTZ DEFAULT now(),
    alarm_name TEXT,
    source TEXT DEFAULT 'poll'::text,
    severity TEXT,
    geofence_id TEXT,
    geofence_name TEXT,
    acknowledged_at TIMESTAMPTZ,
    acknowledged_by TEXT,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.5 DEVICE EVENTS (Connection lifecycle)
CREATE TABLE IF NOT EXISTS bronze.device_events (
    id BIGINT PRIMARY KEY,
    imei TEXT NOT NULL,
    event_type TEXT NOT NULL,
    event_time TIMESTAMPTZ NOT NULL,
    timezone TEXT,
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.6 DISPATCH LOG (Operational/SLA tracking)
CREATE TABLE IF NOT EXISTS bronze.dispatch_log (
    dispatch_id BIGINT PRIMARY KEY,
    ticket_id TEXT NOT NULL,
    imei TEXT NOT NULL,
    driver_name TEXT,
    job_lat DOUBLE PRECISION NOT NULL,
    job_lng DOUBLE PRECISION NOT NULL,
    job_geom geometry(Point,4326),
    assigned_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    first_movement_at TIMESTAMPTZ,
    on_site_at TIMESTAMPTZ,
    resolved_at TIMESTAMPTZ,
    cancelled_at TIMESTAMPTZ,
    distance_km NUMERIC(8,2),
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.7 FAULT CODES (OBD/DTC diagnostics)
CREATE TABLE IF NOT EXISTS bronze.fault_codes (
    id BIGINT PRIMARY KEY,
    imei TEXT NOT NULL,
    reported_at TIMESTAMPTZ NOT NULL,
    fault_code TEXT NOT NULL,
    status_flags INTEGER,
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    geom geometry(Point,4326),
    event_time TIMESTAMPTZ,
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.8 FUEL READINGS
CREATE TABLE IF NOT EXISTS bronze.fuel_readings (
    imei TEXT NOT NULL,
    reading_time TIMESTAMPTZ NOT NULL,
    sensor_path TEXT,
    value NUMERIC(10,3),
    unit TEXT,
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    geom geometry(Point,4326),
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW(),
    PRIMARY KEY (imei, reading_time)
);

-- 5.9 GEOFENCES (Dimension/Reference)
CREATE TABLE IF NOT EXISTS bronze.geofences (
    id BIGINT PRIMARY KEY,
    fence_id TEXT,
    fence_name TEXT NOT NULL,
    fence_type TEXT,
    geom geometry(Geometry,4326),
    radius_m NUMERIC(10,2),
    description TEXT,
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.10 HEARTBEATS (Device health/ping)
CREATE TABLE IF NOT EXISTS bronze.heartbeats (
    imei TEXT NOT NULL,
    gate_time TIMESTAMPTZ NOT NULL,
    power_level SMALLINT,
    gsm_signal SMALLINT,
    acc_status SMALLINT,
    power_status SMALLINT,
    fortify SMALLINT,
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW(),
    PRIMARY KEY (imei, gate_time)
);

-- 5.11 INGESTION LOG (Metadata for tracking loads)
CREATE TABLE IF NOT EXISTS bronze.ingestion_log (
    id BIGINT PRIMARY KEY,
    run_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    endpoint TEXT NOT NULL,
    imei_count INTEGER DEFAULT 0 NOT NULL,
    rows_upserted INTEGER DEFAULT 0 NOT NULL,
    rows_inserted INTEGER DEFAULT 0 NOT NULL,
    duration_ms INTEGER DEFAULT 0 NOT NULL,
    success BOOLEAN DEFAULT true NOT NULL,
    error_code TEXT,
    error_message TEXT,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.12 LBS READINGS (Fallback positioning)
CREATE TABLE IF NOT EXISTS bronze.lbs_readings (
    id BIGINT PRIMARY KEY,
    imei TEXT NOT NULL,
    gate_time TIMESTAMPTZ NOT NULL,
    post_type TEXT,
    lbs_data JSONB,
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.13 LIVE POSITIONS (Current state snapshot)
CREATE TABLE IF NOT EXISTS bronze.live_positions (
    imei TEXT PRIMARY KEY,
    geom geometry(Point,4326),
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    pos_type TEXT,
    confidence SMALLINT,
    gps_time TIMESTAMPTZ,
    hb_time TIMESTAMPTZ,
    speed NUMERIC(7,2),
    direction NUMERIC(6,2),
    acc_status TEXT,
    gps_signal SMALLINT,
    gps_num SMALLINT,
    elec_quantity NUMERIC(5,2),
    power_value NUMERIC(5,2),
    battery_power_val NUMERIC(5,2),
    tracker_oil TEXT,
    temperature NUMERIC(8,2),
    current_mileage NUMERIC(12,2),
    device_status TEXT,
    expire_flag TEXT,
    activation_flag TEXT,
    loc_desc TEXT,
    recorded_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    updated_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.14 OBD READINGS (Vehicle diagnostics)
CREATE TABLE IF NOT EXISTS bronze.obd_readings (
    id BIGINT PRIMARY KEY,
    imei TEXT,
    reading_time TIMESTAMPTZ,
    engine_rpm INTEGER,
    fuel_level_pct NUMERIC(5,2),
    updated_at TIMESTAMPTZ DEFAULT now(),
    car_type SMALLINT,
    acc_state SMALLINT,
    status_flags INTEGER,
    lat DOUBLE PRECISION,
    lng DOUBLE PRECISION,
    geom geometry(Point,4326),
    obd_data JSONB,
    coolant_temp_c NUMERIC(6,2),
    battery_voltage NUMERIC(5,2),
    intake_pressure NUMERIC(6,2),
    throttle_pct NUMERIC(5,2),
    vehicle_speed NUMERIC(7,2),
    engine_load_pct NUMERIC(5,2),
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.15 PARKING EVENTS
CREATE TABLE IF NOT EXISTS bronze.parking_events (
    id BIGINT PRIMARY KEY,
    imei TEXT NOT NULL,
    event_type TEXT,
    start_time TIMESTAMPTZ NOT NULL,
    end_time TIMESTAMPTZ,
    duration_seconds INTEGER,
    geom geometry(Point,4326),
    address TEXT,
    updated_at TIMESTAMPTZ DEFAULT now(),
    ingested_at TIMESTAMPTZ DEFAULT NOW()
);

-- 5.16 TEMPERATURE READINGS (Cold chain sensors)
CREATE TABLE IF NOT EXISTS bronze.temperature_readings (
    imei TEXT NOT NULL,
    reading_time TIMESTAMPTZ NOT NULL,
    temperature NUMERIC(6,2),
    humidity_pct NUMERIC(5,2),
    created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
    ingested_at TIMESTAMPTZ DEFAULT NOW(),
    PRIMARY KEY (imei, reading_time)
);

-- Reset role back to superuser
RESET ROLE;
RESET search_path;

-- 6. VERIFICATION GRANTS (Ensure Grafana can query immediately)
GRANT SELECT ON ALL TABLES IN SCHEMA bronze TO grafana_ro;
GRANT USAGE ON SCHEMA bronze TO grafana_ro;