fix(reporting): restore live-feed cost-centre exclusion + vehicle_type (migration 20)
Some checks failed
Static Analysis / static (push) Waiting to run
Tests / test (push) Waiting to run
Static Analysis / static (pull_request) Has been cancelled
Tests / test (pull_request) Has been cancelled

Migration 11 was applied out of order on 2026-06-10 (it had never been recorded
applied on prod — the reporting objects were hand-created, then migrations 14/15/16
modified them). Re-running 11 recreated reporting.v_live_positions and
reporting.fn_live_positions at their BASE definitions; 15/16 were skipped as
already-applied, so the live map silently lost migration 15's cost-centre exclusion
(personal/management/mtn) and migration 16's vehicle_type/fleet_segment GeoJSON
fields — the live-map vehicle count jumped 74 -> 80.

Migration 20 re-asserts both objects' intended final definitions (verbatim union of
15 + 16). As the highest-numbered migration it always runs last, so the correct
state wins regardless of apply order. Already hot-fixed on prod by re-running 15+16;
this captures it durably. Idempotent.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
david kiania 2026-06-10 22:36:45 +03:00
parent d5093f0a1c
commit f1c231a737
3 changed files with 190 additions and 2 deletions

View file

@ -118,7 +118,7 @@ docs/PLATFORM_OVERVIEW.html # Current-state platform reference (architecture, de
docs/OSM_POI_EXPORT.md # Runbook: OSM .pbf → POI GeoJSON → FleetNow map layer (Shell stations)
docs/superpowers/ # Pitch specs and implementation plans (not deployed code)
scripts/export_osm_pois.py # OSM .pbf → GeoJSON+CSV POI exporter (amenity/brand filter); see OSM_POI_EXPORT.md
migrations/ # Numbered SQL migrations 0219, applied in order by run_migrations.py
migrations/ # Numbered SQL migrations 0220, applied in order by run_migrations.py
# 02 full schema · 03 webhook · 04 distance fix · 05 enhancements
# 06 ops/analytics · 07 views · 08 config · 09 trips enrichment
# 10_driver_clock_views.sql · 10_pgbouncer_auth.sql · 11 reporting
@ -126,6 +126,7 @@ migrations/ # Numbered SQL migrations 0219, applied in order
# 14 fleet segment · 15 map exclude cc · 16 live feed vehicle_type
# 17 reporting.v_fuel_daily (FleetOps) · 18 grant reporting.* to grafana_ro
# 19 reporting.v_ingest_health (pipeline freshness; replaces Grafana panels)
# 20 restore live-feed (re-assert mig-15 exclusion + mig-16 vehicle_type after a mig-11 re-apply clobber)
deploy_dashboard_api_staging.sh # Staging dashboard_api bridge (8891, fleetapi.fivetitude.com); see STAGING_FLEETOPS_ARCHITECTURE.md
scripts/dashboard_ro_role.sql + bootstrap_dashboard_ro.sh # Dedicated read-only DB role for the staging bridge
Dockerfile # Custom image for ingest/webhook containers
@ -157,7 +158,7 @@ tracksolid.alarms -- Alarm events (alarm_type, alarm_name, alarm_time
tracksolid.obd_readings -- OBD diagnostics (push only, awaiting webhook registration)
tracksolid.device_events -- Power on/off tamper events (push only)
tracksolid.ingestion_log -- API call audit trail — 875 runs / 24h, 0 failures at last check (2026-04-19)
tracksolid.schema_migrations -- Applied migrations 0219 (19 reporting.v_ingest_health applied 2026-06-10)
tracksolid.schema_migrations -- Applied migrations 0220 (20 restores live-feed exclusion/vehicle_type, 2026-06-10)
-- PURGED 2026-06-05 (migrations 12 + 13): the dormant `ops` schema (tickets, service_log,
-- odometer_readings, cost_rates, kpi_targets, vw_service_forecast), tracksolid.dispatch_log,
-- and the `dwh_gold` schema (dim_vehicles, fact_daily_fleet_metrics, refresh_daily_metrics).

View file

@ -0,0 +1,186 @@
-- 20_restore_live_feed.sql
-- Restore the live-map feed's intended final state after a migration-ordering regression.
--
-- WHAT HAPPENED: reporting.v_live_positions and reporting.fn_live_positions are first
-- defined in migration 11, then MODIFIED by later migrations:
-- * migration 15 added the cost-centre exclusion to v_live_positions (hide
-- personal/management/mtn from the live map), and
-- * migration 16 added 'vehicle_type'/'fleet_segment' to the fn_live_positions GeoJSON.
-- On 2026-06-10 migration 11 was applied for the first time on prod (the reporting objects
-- had originally been hand-created, so 11 was never recorded; 14/15/16 had already run).
-- Re-running 11 recreated both objects at their BASE definitions, and because 15/16 were
-- already marked applied they were skipped — silently reverting the exclusion and the
-- vehicle_type/fleet_segment additions. Symptom: live-map vehicle count jumped 74 -> 80 and
-- markers lost their specialist-icon fields.
--
-- THE FIX: re-assert both objects' intended final definitions here. As the highest-numbered
-- migration this always runs last, so the correct state wins regardless of apply order.
-- Verbatim union of migration 15 (v_live_positions + exclusion) and migration 16
-- (fn_live_positions + vehicle_type/fleet_segment). Idempotent — safe to re-apply.
SET search_path = tracksolid, reporting, public;
-- ── exclusion config (created by migration 15; re-asserted so this file is self-contained) ──
CREATE TABLE IF NOT EXISTS reporting.map_excluded_cost_centres (
cost_centre text PRIMARY KEY, -- compared case-insensitively (store lowercase)
note text,
added_at timestamptz NOT NULL DEFAULT now()
);
INSERT INTO reporting.map_excluded_cost_centres (cost_centre, note) VALUES
('personal', 'staff/personal vehicles — not operational fleet'),
('management', 'management vehicles — not operational fleet'),
('mtn', 'MTN contract / Uganda (Kampala) — outside Kenyan ops')
ON CONFLICT (cost_centre) DO NOTHING;
-- ── v_live_positions: base definition + cost-centre exclusion (from migration 15) ─────────
CREATE OR REPLACE VIEW reporting.v_live_positions AS
WITH primary_device AS (
SELECT DISTINCT ON ((reporting.normalize_plate(d_1.vehicle_number))) reporting.normalize_plate(d_1.vehicle_number) AS vehicle_number,
d_1.imei AS primary_imei
FROM devices d_1
LEFT JOIN live_positions lp_1 ON lp_1.imei = d_1.imei
WHERE d_1.vehicle_number IS NOT NULL AND d_1.enabled_flag = 1
-- exclude plates whose device(s) carry a non-operational cost centre
AND reporting.normalize_plate(d_1.vehicle_number) NOT IN (
SELECT reporting.normalize_plate(x.vehicle_number)
FROM devices x
WHERE x.vehicle_number IS NOT NULL
AND lower(trim(x.cost_centre)) IN (
SELECT cost_centre FROM reporting.map_excluded_cost_centres)
)
ORDER BY (reporting.normalize_plate(d_1.vehicle_number)), (
CASE
WHEN (d_1.mc_type = ANY (ARRAY['GT06E'::text, 'X3'::text, 'AT4'::text])) AND lp_1.gps_time >= (now() - '24:00:00'::interval) THEN 0
ELSE 1
END), lp_1.gps_time DESC NULLS LAST, (
CASE d_1.mc_type
WHEN 'GT06E'::text THEN 1
WHEN 'X3'::text THEN 2
WHEN 'AT4'::text THEN 3
WHEN 'JC400P'::text THEN 4
ELSE 5
END), d_1.activation_time, d_1.imei
)
SELECT lp.imei,
pd.vehicle_number,
d.driver_name AS assigned_driver,
d.cost_centre,
d.assigned_city,
d.vehicle_category,
d.vehicle_models,
d.mc_type,
CASE d.mc_type
WHEN 'GT06E'::text THEN 'tracker'::text
WHEN 'X3'::text THEN 'tracker'::text
WHEN 'AT4'::text THEN 'tracker'::text
WHEN 'JC400P'::text THEN 'camera'::text
ELSE 'other'::text
END AS device_kind,
lp.lat,
lp.lng,
lp.speed,
lp.direction,
lp.acc_status,
lp.device_status,
lp.gps_signal,
lp.gps_num,
lp.current_mileage,
lp.loc_desc,
lp.gps_time,
lp.updated_at,
(lp.gps_time AT TIME ZONE 'Africa/Nairobi'::text) AS gps_time_eat,
(lp.updated_at AT TIME ZONE 'Africa/Nairobi'::text) AS updated_at_eat,
round(EXTRACT(epoch FROM now() - lp.gps_time) / 3600::numeric, 2) AS source_age_hours
FROM live_positions lp
JOIN primary_device pd ON pd.primary_imei = lp.imei
JOIN devices d ON d.imei = lp.imei;
COMMENT ON TABLE reporting.map_excluded_cost_centres IS
'Cost centres hidden from the live map (reporting.v_live_positions). Edit to hide/restore; '
'effective on next query. Seeded: personal, management, mtn. See migration 15.';
-- ── fn_live_positions: base definition + vehicle_type/fleet_segment (from migration 16) ───
CREATE OR REPLACE FUNCTION reporting.fn_live_positions(p_cost_centre text DEFAULT NULL::text, p_acc_status text DEFAULT NULL::text)
RETURNS jsonb
LANGUAGE plpgsql
STABLE
AS $function$
DECLARE
v_result jsonb;
BEGIN
p_cost_centre := NULLIF(p_cost_centre, '');
p_acc_status := NULLIF(p_acc_status, '');
WITH filtered AS (
SELECT * FROM reporting.v_live_positions
WHERE (p_cost_centre IS NULL OR cost_centre = p_cost_centre)
AND (p_acc_status IS NULL OR acc_status = p_acc_status)
)
SELECT jsonb_build_object(
'summary', jsonb_build_object(
'vehicle_count', COUNT(*),
-- "moving" and "parked" both restrict to devices that have reported
-- within the OFFLINE_THRESHOLD (24 h) so they represent the live
-- fleet, not equipment-failure stragglers. "offline" is its own
-- counter for the > 24 h tail.
'moving', COUNT(*) FILTER (WHERE acc_status = '1'
AND source_age_hours < 24),
'parked', COUNT(*) FILTER (WHERE acc_status = '0'
AND source_age_hours < 24),
'offline', COUNT(*) FILTER (WHERE source_age_hours >= 24),
'median_speed_moving', percentile_cont(0.5) WITHIN GROUP (ORDER BY speed)
FILTER (WHERE acc_status = '1'
AND source_age_hours < 24
AND speed > 0),
'last_batch_at', to_char(MAX(updated_at) AT TIME ZONE 'Africa/Nairobi',
'YYYY-MM-DD HH24:MI:SS'),
'oldest_fix_at', to_char(MIN(gps_time) AT TIME ZONE 'Africa/Nairobi',
'YYYY-MM-DD HH24:MI:SS'),
'newest_fix_at', to_char(MAX(gps_time) AT TIME ZONE 'Africa/Nairobi',
'YYYY-MM-DD HH24:MI:SS'),
'last_batch_utc', MAX(updated_at),
'newest_fix_utc', MAX(gps_time)
),
'geojson', jsonb_build_object(
'type', 'FeatureCollection',
'features', COALESCE(jsonb_agg(
jsonb_build_object(
'type', 'Feature',
'properties', jsonb_build_object(
'imei', imei,
'vehicle_number', vehicle_number,
'driver', assigned_driver,
'cost_centre', cost_centre,
'assigned_city', assigned_city,
'vehicle_category', vehicle_category,
'vehicle_type', vehicle_models,
'fleet_segment', reporting.fn_fleet_segment(vehicle_models),
'mc_type', mc_type,
'device_kind', device_kind,
'source_age_hours', source_age_hours,
'speed', speed,
'direction', direction,
'acc_status', acc_status,
'device_status', device_status,
'gps_signal', gps_signal,
'gps_num', gps_num,
'current_mileage', current_mileage,
'loc_desc', loc_desc,
'gps_time', to_char(gps_time AT TIME ZONE 'Africa/Nairobi',
'YYYY-MM-DD HH24:MI:SS'),
'updated_at', to_char(updated_at AT TIME ZONE 'Africa/Nairobi',
'YYYY-MM-DD HH24:MI:SS'),
'gps_time_utc', gps_time,
'updated_at_utc', updated_at
),
'geometry', jsonb_build_object(
'type', 'Point',
'coordinates', jsonb_build_array(lng, lat)
)
)
), '[]'::jsonb)
)
) INTO v_result FROM filtered;
RETURN v_result;
END $function$;

View file

@ -43,6 +43,7 @@ MIGRATIONS = [
"17_fleetops_fuel_view.sql", # reporting.v_fuel_daily — FleetOps GET /analytics/fuel source
"18_grant_reporting_ro.sql", # grant SELECT on reporting.* to grafana_ro (staging read-only role)
"19_v_ingest_health.sql", # reporting.v_ingest_health — pipeline freshness (replaces Grafana panels)
"20_restore_live_feed.sql", # re-assert v_live_positions exclusion + fn_live_positions vehicle_type (migration-order regression fix)
]
# ── Tables that must exist before the service is allowed to start ─────────────