diff --git a/CLAUDE.md b/CLAUDE.md index 161356b..4b4f81a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,7 +118,7 @@ docs/PLATFORM_OVERVIEW.html # Current-state platform reference (architecture, de docs/OSM_POI_EXPORT.md # Runbook: OSM .pbf → POI GeoJSON → FleetNow map layer (Shell stations) docs/superpowers/ # Pitch specs and implementation plans (not deployed code) scripts/export_osm_pois.py # OSM .pbf → GeoJSON+CSV POI exporter (amenity/brand filter); see OSM_POI_EXPORT.md -migrations/ # Numbered SQL migrations 02–19, applied in order by run_migrations.py +migrations/ # Numbered SQL migrations 02–20, applied in order by run_migrations.py # 02 full schema · 03 webhook · 04 distance fix · 05 enhancements # 06 ops/analytics · 07 views · 08 config · 09 trips enrichment # 10_driver_clock_views.sql · 10_pgbouncer_auth.sql · 11 reporting @@ -126,6 +126,7 @@ migrations/ # Numbered SQL migrations 02–19, applied in order # 14 fleet segment · 15 map exclude cc · 16 live feed vehicle_type # 17 reporting.v_fuel_daily (FleetOps) · 18 grant reporting.* to grafana_ro # 19 reporting.v_ingest_health (pipeline freshness; replaces Grafana panels) + # 20 restore live-feed (re-assert mig-15 exclusion + mig-16 vehicle_type after a mig-11 re-apply clobber) deploy_dashboard_api_staging.sh # Staging dashboard_api bridge (8891, fleetapi.fivetitude.com); see STAGING_FLEETOPS_ARCHITECTURE.md scripts/dashboard_ro_role.sql + bootstrap_dashboard_ro.sh # Dedicated read-only DB role for the staging bridge Dockerfile # Custom image for ingest/webhook containers @@ -157,7 +158,7 @@ tracksolid.alarms -- Alarm events (alarm_type, alarm_name, alarm_time tracksolid.obd_readings -- OBD diagnostics (push only, awaiting webhook registration) tracksolid.device_events -- Power on/off tamper events (push only) tracksolid.ingestion_log -- API call audit trail — 875 runs / 24h, 0 failures at last check (2026-04-19) -tracksolid.schema_migrations -- Applied migrations 02–19 (19 reporting.v_ingest_health applied 2026-06-10) +tracksolid.schema_migrations -- Applied migrations 02–20 (20 restores live-feed exclusion/vehicle_type, 2026-06-10) -- PURGED 2026-06-05 (migrations 12 + 13): the dormant `ops` schema (tickets, service_log, -- odometer_readings, cost_rates, kpi_targets, vw_service_forecast), tracksolid.dispatch_log, -- and the `dwh_gold` schema (dim_vehicles, fact_daily_fleet_metrics, refresh_daily_metrics). diff --git a/migrations/20_restore_live_feed.sql b/migrations/20_restore_live_feed.sql new file mode 100644 index 0000000..dd90d01 --- /dev/null +++ b/migrations/20_restore_live_feed.sql @@ -0,0 +1,186 @@ +-- 20_restore_live_feed.sql +-- Restore the live-map feed's intended final state after a migration-ordering regression. +-- +-- WHAT HAPPENED: reporting.v_live_positions and reporting.fn_live_positions are first +-- defined in migration 11, then MODIFIED by later migrations: +-- * migration 15 added the cost-centre exclusion to v_live_positions (hide +-- personal/management/mtn from the live map), and +-- * migration 16 added 'vehicle_type'/'fleet_segment' to the fn_live_positions GeoJSON. +-- On 2026-06-10 migration 11 was applied for the first time on prod (the reporting objects +-- had originally been hand-created, so 11 was never recorded; 14/15/16 had already run). +-- Re-running 11 recreated both objects at their BASE definitions, and because 15/16 were +-- already marked applied they were skipped — silently reverting the exclusion and the +-- vehicle_type/fleet_segment additions. Symptom: live-map vehicle count jumped 74 -> 80 and +-- markers lost their specialist-icon fields. +-- +-- THE FIX: re-assert both objects' intended final definitions here. As the highest-numbered +-- migration this always runs last, so the correct state wins regardless of apply order. +-- Verbatim union of migration 15 (v_live_positions + exclusion) and migration 16 +-- (fn_live_positions + vehicle_type/fleet_segment). Idempotent — safe to re-apply. + +SET search_path = tracksolid, reporting, public; + +-- ── exclusion config (created by migration 15; re-asserted so this file is self-contained) ── +CREATE TABLE IF NOT EXISTS reporting.map_excluded_cost_centres ( + cost_centre text PRIMARY KEY, -- compared case-insensitively (store lowercase) + note text, + added_at timestamptz NOT NULL DEFAULT now() +); + +INSERT INTO reporting.map_excluded_cost_centres (cost_centre, note) VALUES + ('personal', 'staff/personal vehicles — not operational fleet'), + ('management', 'management vehicles — not operational fleet'), + ('mtn', 'MTN contract / Uganda (Kampala) — outside Kenyan ops') +ON CONFLICT (cost_centre) DO NOTHING; + +-- ── v_live_positions: base definition + cost-centre exclusion (from migration 15) ───────── +CREATE OR REPLACE VIEW reporting.v_live_positions AS + WITH primary_device AS ( + SELECT DISTINCT ON ((reporting.normalize_plate(d_1.vehicle_number))) reporting.normalize_plate(d_1.vehicle_number) AS vehicle_number, + d_1.imei AS primary_imei + FROM devices d_1 + LEFT JOIN live_positions lp_1 ON lp_1.imei = d_1.imei + WHERE d_1.vehicle_number IS NOT NULL AND d_1.enabled_flag = 1 + -- exclude plates whose device(s) carry a non-operational cost centre + AND reporting.normalize_plate(d_1.vehicle_number) NOT IN ( + SELECT reporting.normalize_plate(x.vehicle_number) + FROM devices x + WHERE x.vehicle_number IS NOT NULL + AND lower(trim(x.cost_centre)) IN ( + SELECT cost_centre FROM reporting.map_excluded_cost_centres) + ) + ORDER BY (reporting.normalize_plate(d_1.vehicle_number)), ( + CASE + WHEN (d_1.mc_type = ANY (ARRAY['GT06E'::text, 'X3'::text, 'AT4'::text])) AND lp_1.gps_time >= (now() - '24:00:00'::interval) THEN 0 + ELSE 1 + END), lp_1.gps_time DESC NULLS LAST, ( + CASE d_1.mc_type + WHEN 'GT06E'::text THEN 1 + WHEN 'X3'::text THEN 2 + WHEN 'AT4'::text THEN 3 + WHEN 'JC400P'::text THEN 4 + ELSE 5 + END), d_1.activation_time, d_1.imei + ) + SELECT lp.imei, + pd.vehicle_number, + d.driver_name AS assigned_driver, + d.cost_centre, + d.assigned_city, + d.vehicle_category, + d.vehicle_models, + d.mc_type, + CASE d.mc_type + WHEN 'GT06E'::text THEN 'tracker'::text + WHEN 'X3'::text THEN 'tracker'::text + WHEN 'AT4'::text THEN 'tracker'::text + WHEN 'JC400P'::text THEN 'camera'::text + ELSE 'other'::text + END AS device_kind, + lp.lat, + lp.lng, + lp.speed, + lp.direction, + lp.acc_status, + lp.device_status, + lp.gps_signal, + lp.gps_num, + lp.current_mileage, + lp.loc_desc, + lp.gps_time, + lp.updated_at, + (lp.gps_time AT TIME ZONE 'Africa/Nairobi'::text) AS gps_time_eat, + (lp.updated_at AT TIME ZONE 'Africa/Nairobi'::text) AS updated_at_eat, + round(EXTRACT(epoch FROM now() - lp.gps_time) / 3600::numeric, 2) AS source_age_hours + FROM live_positions lp + JOIN primary_device pd ON pd.primary_imei = lp.imei + JOIN devices d ON d.imei = lp.imei; + +COMMENT ON TABLE reporting.map_excluded_cost_centres IS + 'Cost centres hidden from the live map (reporting.v_live_positions). Edit to hide/restore; ' + 'effective on next query. Seeded: personal, management, mtn. See migration 15.'; + +-- ── fn_live_positions: base definition + vehicle_type/fleet_segment (from migration 16) ─── +CREATE OR REPLACE FUNCTION reporting.fn_live_positions(p_cost_centre text DEFAULT NULL::text, p_acc_status text DEFAULT NULL::text) + RETURNS jsonb + LANGUAGE plpgsql + STABLE +AS $function$ +DECLARE + v_result jsonb; +BEGIN + p_cost_centre := NULLIF(p_cost_centre, ''); + p_acc_status := NULLIF(p_acc_status, ''); + WITH filtered AS ( + SELECT * FROM reporting.v_live_positions + WHERE (p_cost_centre IS NULL OR cost_centre = p_cost_centre) + AND (p_acc_status IS NULL OR acc_status = p_acc_status) + ) + SELECT jsonb_build_object( + 'summary', jsonb_build_object( + 'vehicle_count', COUNT(*), + -- "moving" and "parked" both restrict to devices that have reported + -- within the OFFLINE_THRESHOLD (24 h) so they represent the live + -- fleet, not equipment-failure stragglers. "offline" is its own + -- counter for the > 24 h tail. + 'moving', COUNT(*) FILTER (WHERE acc_status = '1' + AND source_age_hours < 24), + 'parked', COUNT(*) FILTER (WHERE acc_status = '0' + AND source_age_hours < 24), + 'offline', COUNT(*) FILTER (WHERE source_age_hours >= 24), + 'median_speed_moving', percentile_cont(0.5) WITHIN GROUP (ORDER BY speed) + FILTER (WHERE acc_status = '1' + AND source_age_hours < 24 + AND speed > 0), + 'last_batch_at', to_char(MAX(updated_at) AT TIME ZONE 'Africa/Nairobi', + 'YYYY-MM-DD HH24:MI:SS'), + 'oldest_fix_at', to_char(MIN(gps_time) AT TIME ZONE 'Africa/Nairobi', + 'YYYY-MM-DD HH24:MI:SS'), + 'newest_fix_at', to_char(MAX(gps_time) AT TIME ZONE 'Africa/Nairobi', + 'YYYY-MM-DD HH24:MI:SS'), + 'last_batch_utc', MAX(updated_at), + 'newest_fix_utc', MAX(gps_time) + ), + 'geojson', jsonb_build_object( + 'type', 'FeatureCollection', + 'features', COALESCE(jsonb_agg( + jsonb_build_object( + 'type', 'Feature', + 'properties', jsonb_build_object( + 'imei', imei, + 'vehicle_number', vehicle_number, + 'driver', assigned_driver, + 'cost_centre', cost_centre, + 'assigned_city', assigned_city, + 'vehicle_category', vehicle_category, + 'vehicle_type', vehicle_models, + 'fleet_segment', reporting.fn_fleet_segment(vehicle_models), + 'mc_type', mc_type, + 'device_kind', device_kind, + 'source_age_hours', source_age_hours, + 'speed', speed, + 'direction', direction, + 'acc_status', acc_status, + 'device_status', device_status, + 'gps_signal', gps_signal, + 'gps_num', gps_num, + 'current_mileage', current_mileage, + 'loc_desc', loc_desc, + 'gps_time', to_char(gps_time AT TIME ZONE 'Africa/Nairobi', + 'YYYY-MM-DD HH24:MI:SS'), + 'updated_at', to_char(updated_at AT TIME ZONE 'Africa/Nairobi', + 'YYYY-MM-DD HH24:MI:SS'), + 'gps_time_utc', gps_time, + 'updated_at_utc', updated_at + ), + 'geometry', jsonb_build_object( + 'type', 'Point', + 'coordinates', jsonb_build_array(lng, lat) + ) + ) + ), '[]'::jsonb) + ) + ) INTO v_result FROM filtered; + + RETURN v_result; +END $function$; diff --git a/run_migrations.py b/run_migrations.py index 8f9fea7..6a6c7be 100644 --- a/run_migrations.py +++ b/run_migrations.py @@ -43,6 +43,7 @@ MIGRATIONS = [ "17_fleetops_fuel_view.sql", # reporting.v_fuel_daily — FleetOps GET /analytics/fuel source "18_grant_reporting_ro.sql", # grant SELECT on reporting.* to grafana_ro (staging read-only role) "19_v_ingest_health.sql", # reporting.v_ingest_health — pipeline freshness (replaces Grafana panels) + "20_restore_live_feed.sql", # re-assert v_live_positions exclusion + fn_live_positions vehicle_type (migration-order regression fix) ] # ── Tables that must exist before the service is allowed to start ─────────────