diff --git a/app/projectors/live_positions.py b/app/projectors/live_positions.py index 6f7ee4a..d79d712 100644 --- a/app/projectors/live_positions.py +++ b/app/projectors/live_positions.py @@ -223,16 +223,26 @@ async def _project_one( ), ) + # De-dupe at write time: a parked device re-reports the same gpsTime on + # every poll, which would otherwise pile identical (imei, occurred_at) rows + # into the history hypertable and inflate the trip "fix count". Single + # writer + the (imei, occurred_at) index make the NOT EXISTS check cheap, + # and it needs no unique constraint โ€” so it can't fail if deployed before + # the dedup migration. await cur.execute( """ INSERT INTO state.position_history ( vehicle_id, imei, occurred_at, geom, speed_kmh, direction_deg, acc_state, altitude_m, satellites, source, parser_version, mc_type, current_mileage_km, gps_signal, pos_type - ) VALUES ( + ) + SELECT %s, %s, %s, ST_SetSRID(ST_GeomFromText(%s), 4326), %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s + WHERE NOT EXISTS ( + SELECT 1 FROM state.position_history p + WHERE p.imei = %s AND p.occurred_at = %s ) """, ( @@ -248,6 +258,7 @@ async def _project_one( payload.get("current_mileage_km"), payload.get("gps_signal"), payload.get("pos_type"), + imei, occurred_at, ), ) return True diff --git a/db/migrations/20260601000023_position_history_dedup.sql b/db/migrations/20260601000023_position_history_dedup.sql new file mode 100644 index 0000000..81c27e9 --- /dev/null +++ b/db/migrations/20260601000023_position_history_dedup.sql @@ -0,0 +1,32 @@ +-- migrate:up +-- +-- One-time cleanup + guard for state.position_history duplicates. +-- +-- A parked device re-reports the same gpsTime on every 60s poll, so before the +-- projector's write-time NOT EXISTS guard (shipped alongside this migration) +-- the history hypertable accumulated identical (imei, occurred_at) rows โ€” +-- bloating storage and inflating the "fix count" shown in the trip dock. +-- +-- Step 1 delete duplicates, keeping the lowest history_id per +-- (imei, occurred_at). +-- Step 2 add a unique index so duplicates can never reappear. It includes +-- occurred_at (the hypertable partition column), which TimescaleDB +-- requires for a unique index on a hypertable. +-- +-- NOTE: the DELETE scans the whole hypertable; run it in a quiet window. The +-- migration is independent of the projector deploy order โ€” the code's +-- NOT EXISTS guard needs no constraint, so applying this before/after a +-- redeploy is both safe. + +DELETE FROM state.position_history a + USING state.position_history b + WHERE a.imei = b.imei + AND a.occurred_at = b.occurred_at + AND a.history_id > b.history_id; + +CREATE UNIQUE INDEX IF NOT EXISTS position_history_imei_occurred_uq + ON state.position_history (imei, occurred_at); + +-- migrate:down + +DROP INDEX IF EXISTS state.position_history_imei_occurred_uq; diff --git a/web/index-live.html b/web/index-live.html index 8c58448..31dce63 100644 --- a/web/index-live.html +++ b/web/index-live.html @@ -4,7 +4,7 @@ Live Fleet ยท rahamafresh - +