fix(reporting): restore live-feed cost-centre exclusion + vehicle_type (migration 20)
Migration 11 was applied out of order on 2026-06-10 (it had never been recorded applied on prod — the reporting objects were hand-created, then migrations 14/15/16 modified them). Re-running 11 recreated reporting.v_live_positions and reporting.fn_live_positions at their BASE definitions; 15/16 were skipped as already-applied, so the live map silently lost migration 15's cost-centre exclusion (personal/management/mtn) and migration 16's vehicle_type/fleet_segment GeoJSON fields — the live-map vehicle count jumped 74 -> 80. Migration 20 re-asserts both objects' intended final definitions (verbatim union of 15 + 16). As the highest-numbered migration it always runs last, so the correct state wins regardless of apply order. Already hot-fixed on prod by re-running 15+16; this captures it durably. Idempotent. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
d5093f0a1c
commit
f1c231a737
3 changed files with 190 additions and 2 deletions
|
|
@ -118,7 +118,7 @@ docs/PLATFORM_OVERVIEW.html # Current-state platform reference (architecture, de
|
||||||
docs/OSM_POI_EXPORT.md # Runbook: OSM .pbf → POI GeoJSON → FleetNow map layer (Shell stations)
|
docs/OSM_POI_EXPORT.md # Runbook: OSM .pbf → POI GeoJSON → FleetNow map layer (Shell stations)
|
||||||
docs/superpowers/ # Pitch specs and implementation plans (not deployed code)
|
docs/superpowers/ # Pitch specs and implementation plans (not deployed code)
|
||||||
scripts/export_osm_pois.py # OSM .pbf → GeoJSON+CSV POI exporter (amenity/brand filter); see OSM_POI_EXPORT.md
|
scripts/export_osm_pois.py # OSM .pbf → GeoJSON+CSV POI exporter (amenity/brand filter); see OSM_POI_EXPORT.md
|
||||||
migrations/ # Numbered SQL migrations 02–19, applied in order by run_migrations.py
|
migrations/ # Numbered SQL migrations 02–20, applied in order by run_migrations.py
|
||||||
# 02 full schema · 03 webhook · 04 distance fix · 05 enhancements
|
# 02 full schema · 03 webhook · 04 distance fix · 05 enhancements
|
||||||
# 06 ops/analytics · 07 views · 08 config · 09 trips enrichment
|
# 06 ops/analytics · 07 views · 08 config · 09 trips enrichment
|
||||||
# 10_driver_clock_views.sql · 10_pgbouncer_auth.sql · 11 reporting
|
# 10_driver_clock_views.sql · 10_pgbouncer_auth.sql · 11 reporting
|
||||||
|
|
@ -126,6 +126,7 @@ migrations/ # Numbered SQL migrations 02–19, applied in order
|
||||||
# 14 fleet segment · 15 map exclude cc · 16 live feed vehicle_type
|
# 14 fleet segment · 15 map exclude cc · 16 live feed vehicle_type
|
||||||
# 17 reporting.v_fuel_daily (FleetOps) · 18 grant reporting.* to grafana_ro
|
# 17 reporting.v_fuel_daily (FleetOps) · 18 grant reporting.* to grafana_ro
|
||||||
# 19 reporting.v_ingest_health (pipeline freshness; replaces Grafana panels)
|
# 19 reporting.v_ingest_health (pipeline freshness; replaces Grafana panels)
|
||||||
|
# 20 restore live-feed (re-assert mig-15 exclusion + mig-16 vehicle_type after a mig-11 re-apply clobber)
|
||||||
deploy_dashboard_api_staging.sh # Staging dashboard_api bridge (8891, fleetapi.fivetitude.com); see STAGING_FLEETOPS_ARCHITECTURE.md
|
deploy_dashboard_api_staging.sh # Staging dashboard_api bridge (8891, fleetapi.fivetitude.com); see STAGING_FLEETOPS_ARCHITECTURE.md
|
||||||
scripts/dashboard_ro_role.sql + bootstrap_dashboard_ro.sh # Dedicated read-only DB role for the staging bridge
|
scripts/dashboard_ro_role.sql + bootstrap_dashboard_ro.sh # Dedicated read-only DB role for the staging bridge
|
||||||
Dockerfile # Custom image for ingest/webhook containers
|
Dockerfile # Custom image for ingest/webhook containers
|
||||||
|
|
@ -157,7 +158,7 @@ tracksolid.alarms -- Alarm events (alarm_type, alarm_name, alarm_time
|
||||||
tracksolid.obd_readings -- OBD diagnostics (push only, awaiting webhook registration)
|
tracksolid.obd_readings -- OBD diagnostics (push only, awaiting webhook registration)
|
||||||
tracksolid.device_events -- Power on/off tamper events (push only)
|
tracksolid.device_events -- Power on/off tamper events (push only)
|
||||||
tracksolid.ingestion_log -- API call audit trail — 875 runs / 24h, 0 failures at last check (2026-04-19)
|
tracksolid.ingestion_log -- API call audit trail — 875 runs / 24h, 0 failures at last check (2026-04-19)
|
||||||
tracksolid.schema_migrations -- Applied migrations 02–19 (19 reporting.v_ingest_health applied 2026-06-10)
|
tracksolid.schema_migrations -- Applied migrations 02–20 (20 restores live-feed exclusion/vehicle_type, 2026-06-10)
|
||||||
-- PURGED 2026-06-05 (migrations 12 + 13): the dormant `ops` schema (tickets, service_log,
|
-- PURGED 2026-06-05 (migrations 12 + 13): the dormant `ops` schema (tickets, service_log,
|
||||||
-- odometer_readings, cost_rates, kpi_targets, vw_service_forecast), tracksolid.dispatch_log,
|
-- odometer_readings, cost_rates, kpi_targets, vw_service_forecast), tracksolid.dispatch_log,
|
||||||
-- and the `dwh_gold` schema (dim_vehicles, fact_daily_fleet_metrics, refresh_daily_metrics).
|
-- and the `dwh_gold` schema (dim_vehicles, fact_daily_fleet_metrics, refresh_daily_metrics).
|
||||||
|
|
|
||||||
186
migrations/20_restore_live_feed.sql
Normal file
186
migrations/20_restore_live_feed.sql
Normal file
|
|
@ -0,0 +1,186 @@
|
||||||
|
-- 20_restore_live_feed.sql
|
||||||
|
-- Restore the live-map feed's intended final state after a migration-ordering regression.
|
||||||
|
--
|
||||||
|
-- WHAT HAPPENED: reporting.v_live_positions and reporting.fn_live_positions are first
|
||||||
|
-- defined in migration 11, then MODIFIED by later migrations:
|
||||||
|
-- * migration 15 added the cost-centre exclusion to v_live_positions (hide
|
||||||
|
-- personal/management/mtn from the live map), and
|
||||||
|
-- * migration 16 added 'vehicle_type'/'fleet_segment' to the fn_live_positions GeoJSON.
|
||||||
|
-- On 2026-06-10 migration 11 was applied for the first time on prod (the reporting objects
|
||||||
|
-- had originally been hand-created, so 11 was never recorded; 14/15/16 had already run).
|
||||||
|
-- Re-running 11 recreated both objects at their BASE definitions, and because 15/16 were
|
||||||
|
-- already marked applied they were skipped — silently reverting the exclusion and the
|
||||||
|
-- vehicle_type/fleet_segment additions. Symptom: live-map vehicle count jumped 74 -> 80 and
|
||||||
|
-- markers lost their specialist-icon fields.
|
||||||
|
--
|
||||||
|
-- THE FIX: re-assert both objects' intended final definitions here. As the highest-numbered
|
||||||
|
-- migration this always runs last, so the correct state wins regardless of apply order.
|
||||||
|
-- Verbatim union of migration 15 (v_live_positions + exclusion) and migration 16
|
||||||
|
-- (fn_live_positions + vehicle_type/fleet_segment). Idempotent — safe to re-apply.
|
||||||
|
|
||||||
|
SET search_path = tracksolid, reporting, public;
|
||||||
|
|
||||||
|
-- ── exclusion config (created by migration 15; re-asserted so this file is self-contained) ──
|
||||||
|
CREATE TABLE IF NOT EXISTS reporting.map_excluded_cost_centres (
|
||||||
|
cost_centre text PRIMARY KEY, -- compared case-insensitively (store lowercase)
|
||||||
|
note text,
|
||||||
|
added_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO reporting.map_excluded_cost_centres (cost_centre, note) VALUES
|
||||||
|
('personal', 'staff/personal vehicles — not operational fleet'),
|
||||||
|
('management', 'management vehicles — not operational fleet'),
|
||||||
|
('mtn', 'MTN contract / Uganda (Kampala) — outside Kenyan ops')
|
||||||
|
ON CONFLICT (cost_centre) DO NOTHING;
|
||||||
|
|
||||||
|
-- ── v_live_positions: base definition + cost-centre exclusion (from migration 15) ─────────
|
||||||
|
CREATE OR REPLACE VIEW reporting.v_live_positions AS
|
||||||
|
WITH primary_device AS (
|
||||||
|
SELECT DISTINCT ON ((reporting.normalize_plate(d_1.vehicle_number))) reporting.normalize_plate(d_1.vehicle_number) AS vehicle_number,
|
||||||
|
d_1.imei AS primary_imei
|
||||||
|
FROM devices d_1
|
||||||
|
LEFT JOIN live_positions lp_1 ON lp_1.imei = d_1.imei
|
||||||
|
WHERE d_1.vehicle_number IS NOT NULL AND d_1.enabled_flag = 1
|
||||||
|
-- exclude plates whose device(s) carry a non-operational cost centre
|
||||||
|
AND reporting.normalize_plate(d_1.vehicle_number) NOT IN (
|
||||||
|
SELECT reporting.normalize_plate(x.vehicle_number)
|
||||||
|
FROM devices x
|
||||||
|
WHERE x.vehicle_number IS NOT NULL
|
||||||
|
AND lower(trim(x.cost_centre)) IN (
|
||||||
|
SELECT cost_centre FROM reporting.map_excluded_cost_centres)
|
||||||
|
)
|
||||||
|
ORDER BY (reporting.normalize_plate(d_1.vehicle_number)), (
|
||||||
|
CASE
|
||||||
|
WHEN (d_1.mc_type = ANY (ARRAY['GT06E'::text, 'X3'::text, 'AT4'::text])) AND lp_1.gps_time >= (now() - '24:00:00'::interval) THEN 0
|
||||||
|
ELSE 1
|
||||||
|
END), lp_1.gps_time DESC NULLS LAST, (
|
||||||
|
CASE d_1.mc_type
|
||||||
|
WHEN 'GT06E'::text THEN 1
|
||||||
|
WHEN 'X3'::text THEN 2
|
||||||
|
WHEN 'AT4'::text THEN 3
|
||||||
|
WHEN 'JC400P'::text THEN 4
|
||||||
|
ELSE 5
|
||||||
|
END), d_1.activation_time, d_1.imei
|
||||||
|
)
|
||||||
|
SELECT lp.imei,
|
||||||
|
pd.vehicle_number,
|
||||||
|
d.driver_name AS assigned_driver,
|
||||||
|
d.cost_centre,
|
||||||
|
d.assigned_city,
|
||||||
|
d.vehicle_category,
|
||||||
|
d.vehicle_models,
|
||||||
|
d.mc_type,
|
||||||
|
CASE d.mc_type
|
||||||
|
WHEN 'GT06E'::text THEN 'tracker'::text
|
||||||
|
WHEN 'X3'::text THEN 'tracker'::text
|
||||||
|
WHEN 'AT4'::text THEN 'tracker'::text
|
||||||
|
WHEN 'JC400P'::text THEN 'camera'::text
|
||||||
|
ELSE 'other'::text
|
||||||
|
END AS device_kind,
|
||||||
|
lp.lat,
|
||||||
|
lp.lng,
|
||||||
|
lp.speed,
|
||||||
|
lp.direction,
|
||||||
|
lp.acc_status,
|
||||||
|
lp.device_status,
|
||||||
|
lp.gps_signal,
|
||||||
|
lp.gps_num,
|
||||||
|
lp.current_mileage,
|
||||||
|
lp.loc_desc,
|
||||||
|
lp.gps_time,
|
||||||
|
lp.updated_at,
|
||||||
|
(lp.gps_time AT TIME ZONE 'Africa/Nairobi'::text) AS gps_time_eat,
|
||||||
|
(lp.updated_at AT TIME ZONE 'Africa/Nairobi'::text) AS updated_at_eat,
|
||||||
|
round(EXTRACT(epoch FROM now() - lp.gps_time) / 3600::numeric, 2) AS source_age_hours
|
||||||
|
FROM live_positions lp
|
||||||
|
JOIN primary_device pd ON pd.primary_imei = lp.imei
|
||||||
|
JOIN devices d ON d.imei = lp.imei;
|
||||||
|
|
||||||
|
COMMENT ON TABLE reporting.map_excluded_cost_centres IS
|
||||||
|
'Cost centres hidden from the live map (reporting.v_live_positions). Edit to hide/restore; '
|
||||||
|
'effective on next query. Seeded: personal, management, mtn. See migration 15.';
|
||||||
|
|
||||||
|
-- ── fn_live_positions: base definition + vehicle_type/fleet_segment (from migration 16) ───
|
||||||
|
CREATE OR REPLACE FUNCTION reporting.fn_live_positions(p_cost_centre text DEFAULT NULL::text, p_acc_status text DEFAULT NULL::text)
|
||||||
|
RETURNS jsonb
|
||||||
|
LANGUAGE plpgsql
|
||||||
|
STABLE
|
||||||
|
AS $function$
|
||||||
|
DECLARE
|
||||||
|
v_result jsonb;
|
||||||
|
BEGIN
|
||||||
|
p_cost_centre := NULLIF(p_cost_centre, '');
|
||||||
|
p_acc_status := NULLIF(p_acc_status, '');
|
||||||
|
WITH filtered AS (
|
||||||
|
SELECT * FROM reporting.v_live_positions
|
||||||
|
WHERE (p_cost_centre IS NULL OR cost_centre = p_cost_centre)
|
||||||
|
AND (p_acc_status IS NULL OR acc_status = p_acc_status)
|
||||||
|
)
|
||||||
|
SELECT jsonb_build_object(
|
||||||
|
'summary', jsonb_build_object(
|
||||||
|
'vehicle_count', COUNT(*),
|
||||||
|
-- "moving" and "parked" both restrict to devices that have reported
|
||||||
|
-- within the OFFLINE_THRESHOLD (24 h) so they represent the live
|
||||||
|
-- fleet, not equipment-failure stragglers. "offline" is its own
|
||||||
|
-- counter for the > 24 h tail.
|
||||||
|
'moving', COUNT(*) FILTER (WHERE acc_status = '1'
|
||||||
|
AND source_age_hours < 24),
|
||||||
|
'parked', COUNT(*) FILTER (WHERE acc_status = '0'
|
||||||
|
AND source_age_hours < 24),
|
||||||
|
'offline', COUNT(*) FILTER (WHERE source_age_hours >= 24),
|
||||||
|
'median_speed_moving', percentile_cont(0.5) WITHIN GROUP (ORDER BY speed)
|
||||||
|
FILTER (WHERE acc_status = '1'
|
||||||
|
AND source_age_hours < 24
|
||||||
|
AND speed > 0),
|
||||||
|
'last_batch_at', to_char(MAX(updated_at) AT TIME ZONE 'Africa/Nairobi',
|
||||||
|
'YYYY-MM-DD HH24:MI:SS'),
|
||||||
|
'oldest_fix_at', to_char(MIN(gps_time) AT TIME ZONE 'Africa/Nairobi',
|
||||||
|
'YYYY-MM-DD HH24:MI:SS'),
|
||||||
|
'newest_fix_at', to_char(MAX(gps_time) AT TIME ZONE 'Africa/Nairobi',
|
||||||
|
'YYYY-MM-DD HH24:MI:SS'),
|
||||||
|
'last_batch_utc', MAX(updated_at),
|
||||||
|
'newest_fix_utc', MAX(gps_time)
|
||||||
|
),
|
||||||
|
'geojson', jsonb_build_object(
|
||||||
|
'type', 'FeatureCollection',
|
||||||
|
'features', COALESCE(jsonb_agg(
|
||||||
|
jsonb_build_object(
|
||||||
|
'type', 'Feature',
|
||||||
|
'properties', jsonb_build_object(
|
||||||
|
'imei', imei,
|
||||||
|
'vehicle_number', vehicle_number,
|
||||||
|
'driver', assigned_driver,
|
||||||
|
'cost_centre', cost_centre,
|
||||||
|
'assigned_city', assigned_city,
|
||||||
|
'vehicle_category', vehicle_category,
|
||||||
|
'vehicle_type', vehicle_models,
|
||||||
|
'fleet_segment', reporting.fn_fleet_segment(vehicle_models),
|
||||||
|
'mc_type', mc_type,
|
||||||
|
'device_kind', device_kind,
|
||||||
|
'source_age_hours', source_age_hours,
|
||||||
|
'speed', speed,
|
||||||
|
'direction', direction,
|
||||||
|
'acc_status', acc_status,
|
||||||
|
'device_status', device_status,
|
||||||
|
'gps_signal', gps_signal,
|
||||||
|
'gps_num', gps_num,
|
||||||
|
'current_mileage', current_mileage,
|
||||||
|
'loc_desc', loc_desc,
|
||||||
|
'gps_time', to_char(gps_time AT TIME ZONE 'Africa/Nairobi',
|
||||||
|
'YYYY-MM-DD HH24:MI:SS'),
|
||||||
|
'updated_at', to_char(updated_at AT TIME ZONE 'Africa/Nairobi',
|
||||||
|
'YYYY-MM-DD HH24:MI:SS'),
|
||||||
|
'gps_time_utc', gps_time,
|
||||||
|
'updated_at_utc', updated_at
|
||||||
|
),
|
||||||
|
'geometry', jsonb_build_object(
|
||||||
|
'type', 'Point',
|
||||||
|
'coordinates', jsonb_build_array(lng, lat)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
), '[]'::jsonb)
|
||||||
|
)
|
||||||
|
) INTO v_result FROM filtered;
|
||||||
|
|
||||||
|
RETURN v_result;
|
||||||
|
END $function$;
|
||||||
|
|
@ -43,6 +43,7 @@ MIGRATIONS = [
|
||||||
"17_fleetops_fuel_view.sql", # reporting.v_fuel_daily — FleetOps GET /analytics/fuel source
|
"17_fleetops_fuel_view.sql", # reporting.v_fuel_daily — FleetOps GET /analytics/fuel source
|
||||||
"18_grant_reporting_ro.sql", # grant SELECT on reporting.* to grafana_ro (staging read-only role)
|
"18_grant_reporting_ro.sql", # grant SELECT on reporting.* to grafana_ro (staging read-only role)
|
||||||
"19_v_ingest_health.sql", # reporting.v_ingest_health — pipeline freshness (replaces Grafana panels)
|
"19_v_ingest_health.sql", # reporting.v_ingest_health — pipeline freshness (replaces Grafana panels)
|
||||||
|
"20_restore_live_feed.sql", # re-assert v_live_positions exclusion + fn_live_positions vehicle_type (migration-order regression fix)
|
||||||
]
|
]
|
||||||
|
|
||||||
# ── Tables that must exist before the service is allowed to start ─────────────
|
# ── Tables that must exist before the service is allowed to start ─────────────
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue