495bb2bd71
The CSV-based roster import (mig 15+16 and scripts/import_csv_roster.py)
merged vehicle rows that differed only by _Track / _CAM suffix, dropping
the active fleet count from 144 to 124. Reverting the whole thing.
Mig 17 in one transaction:
- Re-splits devices by parsed plate from device_name (same regex as
mig 14, preserving _Track as separate vehicle)
- Restores serve.fn_live_view to its v3 body (no d.driver_name/phone
refs that would break once the columns are gone)
- Drops the six CSV-only columns from domain.devices
- Deletes schema_migrations rows for the deleted 15/16
- Logs final counts via RAISE NOTICE
Apply on VPS: psql -f db/migrations/20260601000017_rollback_csv_import.sql
248 lines
11 KiB
PL/PgSQL
248 lines
11 KiB
PL/PgSQL
-- migrate:up
|
|
--
|
|
-- Full rollback of the CSV import (migrations 15 + 16 + the data mutations
|
|
-- performed by scripts/import_csv_roster.py). After this runs, the database
|
|
-- is back to the post-migration-14 shape: ~144 vehicles split per device_name
|
|
-- (preserving _Track / _CAM suffix vehicles), no CSV-only columns, and
|
|
-- serve.fn_live_view back to its v3 body (driver_name from heuristic only).
|
|
--
|
|
-- Why a single forward migration instead of `dbmate rollback`:
|
|
-- - The CSV import mutated row state, not just schema. A schema-only down
|
|
-- would leave 124 merged vehicles in place.
|
|
-- - dbmate rollback runs the down blocks in reverse, but mig 16's down only
|
|
-- drops the function — it doesn't restore the previous body. Doing it
|
|
-- forward lets us be explicit about the target state.
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 1. Re-split vehicles by parsed plate from device_name
|
|
-- ---------------------------------------------------------------------------
|
|
|
|
DO $rollback$
|
|
DECLARE
|
|
rec RECORD;
|
|
new_vid integer;
|
|
BEGIN
|
|
ALTER TABLE domain.vehicles DROP CONSTRAINT IF EXISTS vehicles_plate_key;
|
|
|
|
FOR rec IN
|
|
SELECT
|
|
d.imei,
|
|
d.vehicle_id AS current_vid,
|
|
v.plate AS current_plate,
|
|
regexp_replace(
|
|
(regexp_match(lp.device_name, '^.* - (.+)$'))[1],
|
|
'_(cam|CAM)$', ''
|
|
) AS parsed_plate
|
|
FROM domain.devices d
|
|
JOIN state.live_positions lp ON lp.imei = d.imei
|
|
JOIN domain.vehicles v ON v.vehicle_id = d.vehicle_id
|
|
WHERE lp.device_name LIKE '% - %'
|
|
LOOP
|
|
IF rec.parsed_plate IS NULL OR rec.parsed_plate = '' THEN CONTINUE; END IF;
|
|
IF rec.parsed_plate !~ '[A-Z]' OR rec.parsed_plate !~ '[0-9]' THEN CONTINUE; END IF;
|
|
IF rec.parsed_plate = rec.current_plate THEN CONTINUE; END IF;
|
|
|
|
SELECT vehicle_id INTO new_vid
|
|
FROM domain.vehicles
|
|
WHERE plate = rec.parsed_plate;
|
|
|
|
IF new_vid IS NULL THEN
|
|
INSERT INTO domain.vehicles (plate)
|
|
VALUES (rec.parsed_plate)
|
|
RETURNING vehicle_id INTO new_vid;
|
|
END IF;
|
|
|
|
UPDATE domain.devices SET vehicle_id = new_vid WHERE imei = rec.imei;
|
|
UPDATE state.live_positions SET vehicle_id = new_vid WHERE imei = rec.imei;
|
|
UPDATE state.position_history SET vehicle_id = new_vid WHERE imei = rec.imei;
|
|
END LOOP;
|
|
|
|
-- Delete vehicles that no device or state row references anymore
|
|
DELETE FROM domain.vehicles v
|
|
WHERE NOT EXISTS (SELECT 1 FROM domain.devices d WHERE d.vehicle_id = v.vehicle_id)
|
|
AND NOT EXISTS (SELECT 1 FROM state.live_positions lp WHERE lp.vehicle_id = v.vehicle_id)
|
|
AND NOT EXISTS (SELECT 1 FROM state.position_history ph WHERE ph.vehicle_id = v.vehicle_id);
|
|
|
|
ALTER TABLE domain.vehicles ADD CONSTRAINT vehicles_plate_key UNIQUE (plate);
|
|
END
|
|
$rollback$;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 2. Restore serve.fn_live_view to the v3 body (driver_name heuristic only)
|
|
-- ---------------------------------------------------------------------------
|
|
|
|
DROP FUNCTION IF EXISTS serve.fn_live_view(jsonb);
|
|
|
|
CREATE OR REPLACE FUNCTION serve.fn_live_view(filters jsonb)
|
|
RETURNS jsonb
|
|
LANGUAGE plpgsql STABLE
|
|
AS $$
|
|
DECLARE
|
|
fresh_window interval := COALESCE((filters->>'fresh_window')::interval, interval '24 hours');
|
|
offline_after interval := COALESCE((filters->>'offline_after')::interval, interval '5 minutes');
|
|
move_speed_kmh numeric := COALESCE((filters->>'move_speed_kmh')::numeric, 5);
|
|
p_cost_centre text := filters->>'cost_centre';
|
|
p_assigned_city text := filters->>'assigned_city';
|
|
p_vehicle_numbers text[] := CASE
|
|
WHEN filters ? 'vehicle_numbers'
|
|
THEN ARRAY(SELECT jsonb_array_elements_text(filters->'vehicle_numbers'))
|
|
ELSE NULL
|
|
END;
|
|
result jsonb;
|
|
BEGIN
|
|
WITH candidates AS (
|
|
SELECT
|
|
lp.imei, lp.occurred_at, lp.geom, lp.speed_kmh, lp.direction_deg,
|
|
lp.mc_type, lp.current_mileage_km, lp.gps_signal, lp.satellites,
|
|
lp.device_name, lp.pos_type,
|
|
d.device_type, d.activation_at,
|
|
v.vehicle_id, v.plate, v.cost_centre, v.assigned_city
|
|
FROM state.live_positions lp
|
|
JOIN domain.devices d ON d.imei = lp.imei
|
|
JOIN domain.vehicles v ON v.vehicle_id = d.vehicle_id
|
|
WHERE d.lifecycle = 'active'
|
|
AND (p_cost_centre IS NULL OR v.cost_centre = p_cost_centre)
|
|
AND (p_assigned_city IS NULL OR v.assigned_city = p_assigned_city)
|
|
AND (p_vehicle_numbers IS NULL OR v.plate = ANY (p_vehicle_numbers))
|
|
),
|
|
ranked AS (
|
|
SELECT c.*, ROW_NUMBER() OVER (
|
|
PARTITION BY c.vehicle_id
|
|
ORDER BY
|
|
CASE c.device_type WHEN 'tracker' THEN 0 ELSE 1 END,
|
|
CASE WHEN c.occurred_at > now() - fresh_window THEN 0 ELSE 1 END,
|
|
c.occurred_at DESC,
|
|
c.activation_at DESC NULLS LAST
|
|
) AS rn
|
|
FROM candidates c
|
|
),
|
|
deduped AS (SELECT * FROM ranked WHERE rn = 1),
|
|
enriched AS (
|
|
SELECT d.*,
|
|
CASE
|
|
WHEN d.occurred_at <= now() - offline_after THEN 'offline'
|
|
WHEN d.speed_kmh IS NOT NULL AND d.speed_kmh > move_speed_kmh THEN 'moving'
|
|
ELSE 'parked'
|
|
END AS operational_state,
|
|
serve._cost_centre_color(d.cost_centre) AS cost_centre_color,
|
|
EXTRACT(EPOCH FROM (now() - d.occurred_at))::int AS age_sec,
|
|
round(ST_Y(d.geom)::numeric, 4) AS lat_rounded,
|
|
round(ST_X(d.geom)::numeric, 4) AS lng_rounded
|
|
FROM deduped d
|
|
),
|
|
with_addr AS (
|
|
SELECT e.*, g.address, g.address_short
|
|
FROM enriched e
|
|
LEFT JOIN state.geocoded_positions g
|
|
ON g.lat_rounded = e.lat_rounded
|
|
AND g.lng_rounded = e.lng_rounded
|
|
),
|
|
summary AS (
|
|
SELECT jsonb_build_object(
|
|
'total_active', count(*),
|
|
'moving', count(*) FILTER (WHERE operational_state = 'moving'),
|
|
'parked', count(*) FILTER (WHERE operational_state = 'parked'),
|
|
'offline', count(*) FILTER (WHERE operational_state = 'offline'),
|
|
'below_freshness_slo', count(*) FILTER (
|
|
WHERE occurred_at <= now() - interval '90 seconds'
|
|
),
|
|
'as_of', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')
|
|
) AS s
|
|
FROM with_addr
|
|
),
|
|
features AS (
|
|
SELECT COALESCE(jsonb_agg(
|
|
jsonb_build_object(
|
|
'type', 'Feature',
|
|
'geometry', ST_AsGeoJSON(e.geom)::jsonb,
|
|
'properties', jsonb_build_object(
|
|
'vehicle_id', e.vehicle_id,
|
|
'plate', e.plate,
|
|
'plate_short', serve._label_short(e.device_name, e.plate),
|
|
'driver_name', serve._driver_name(e.device_name),
|
|
'imei', e.imei,
|
|
'device_type', e.device_type,
|
|
'device_name', e.device_name,
|
|
'mc_type', e.mc_type,
|
|
'pos_type', e.pos_type,
|
|
'cost_centre', e.cost_centre,
|
|
'cost_centre_color', e.cost_centre_color,
|
|
'assigned_city', e.assigned_city,
|
|
'address', e.address,
|
|
'address_short', e.address_short,
|
|
'occurred_at', to_char(e.occurred_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'),
|
|
'age_sec', e.age_sec,
|
|
'speed_kmh', e.speed_kmh,
|
|
'heading_deg', e.direction_deg,
|
|
'gps_signal', e.gps_signal,
|
|
'satellites', e.satellites,
|
|
'current_mileage_km', e.current_mileage_km,
|
|
'operational_state', e.operational_state,
|
|
'style_class', 'vehicle-' || e.operational_state,
|
|
'marker_color', CASE WHEN e.operational_state = 'moving'
|
|
THEN e.cost_centre_color
|
|
ELSE '#9ca3af' END,
|
|
'show_arrow', (e.operational_state = 'moving' AND e.direction_deg IS NOT NULL)
|
|
)
|
|
)
|
|
), '[]'::jsonb) AS feats
|
|
FROM with_addr e
|
|
),
|
|
slo_block AS (
|
|
SELECT COALESCE(jsonb_object_agg(
|
|
metric,
|
|
jsonb_build_object('threshold', threshold, 'current', current_value, 'status', status)
|
|
), '{}'::jsonb) AS ss
|
|
FROM slo.v_current_status
|
|
)
|
|
SELECT jsonb_build_object(
|
|
'summary', (SELECT s FROM summary),
|
|
'geojson', jsonb_build_object('type', 'FeatureCollection', 'features', (SELECT feats FROM features)),
|
|
'slo_status', (SELECT ss FROM slo_block)
|
|
)
|
|
INTO result;
|
|
RETURN result;
|
|
END;
|
|
$$;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 3. Drop the CSV-only columns from domain.devices
|
|
-- ---------------------------------------------------------------------------
|
|
|
|
ALTER TABLE domain.devices
|
|
DROP COLUMN IF EXISTS driver_name,
|
|
DROP COLUMN IF EXISTS driver_phone,
|
|
DROP COLUMN IF EXISTS iccid,
|
|
DROP COLUMN IF EXISTS expiration_at,
|
|
DROP COLUMN IF EXISTS device_group,
|
|
DROP COLUMN IF EXISTS roster_synced_at;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 4. Remove the dead schema_migrations rows for the deleted migrations
|
|
-- ---------------------------------------------------------------------------
|
|
--
|
|
-- 20260601000015 (devices_csv_columns) and 20260601000016 (prefer_csv_driver)
|
|
-- have been deleted from db/migrations/ — drop their tracker rows so dbmate
|
|
-- status stays consistent with the files on disk.
|
|
|
|
DELETE FROM public.schema_migrations
|
|
WHERE version IN ('20260601000015', '20260601000016');
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 5. Sanity check — log the resulting counts
|
|
-- ---------------------------------------------------------------------------
|
|
|
|
DO $check$
|
|
DECLARE
|
|
v_count integer;
|
|
d_count integer;
|
|
BEGIN
|
|
SELECT count(*) INTO v_count FROM domain.vehicles;
|
|
SELECT count(*) INTO d_count FROM domain.devices;
|
|
RAISE NOTICE 'rollback complete: % vehicles, % devices', v_count, d_count;
|
|
END
|
|
$check$;
|
|
|
|
-- migrate:down
|
|
-- No-op: this migration is itself a rollback. To re-apply the CSV import,
|
|
-- re-run migrations 15 + 16 and scripts/import_csv_roster.py.
|