From 6a0ceb78dd4c03a08baee90343807a5b4bc3e4f8 Mon Sep 17 00:00:00 2001 From: David Kiania Date: Sun, 12 Apr 2026 00:06:57 +0300 Subject: [PATCH] =?UTF-8?q?Fix=20trip=20distance=20unit=20(metres=E2=86=92?= =?UTF-8?q?km)=20and=20full=20device=20sync=20on=20upsert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [FIX-M16] jimi.device.track.mileage returns distance in metres despite docs claiming km. Confirmed: avgSpeed × runTimeSecond / 3600 = distance/1000. poll_trips() now divides raw value by 1000 before storing as distance_km. 3 existing bad rows corrected in prod DB (distance_km / 1000). [FIX-M17] sync_devices() ON CONFLICT clause was only updating 5 of 26 fields, silently dropping driver_phone, sim, iccid, vehicle_name, status etc. on subsequent syncs. Expanded to update all device fields so driver assignments made in Tracksolid Pro UI propagate to DB on next daily sync. Add sync_driver_audit.py: one-shot script to compare API vs DB device registry, report driver/IMEI gaps, and force a full field upsert. Co-Authored-By: Claude Sonnet 4.6 --- ingest_movement_rev.py | 40 ++++++-- sync_driver_audit.py | 205 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 8 deletions(-) create mode 100644 sync_driver_audit.py diff --git a/ingest_movement_rev.py b/ingest_movement_rev.py index 95ba13d..54af9c4 100644 --- a/ingest_movement_rev.py +++ b/ingest_movement_rev.py @@ -93,12 +93,33 @@ def sync_devices(): %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() ) ON CONFLICT (imei) DO UPDATE SET - device_name = EXCLUDED.device_name, - vehicle_number = EXCLUDED.vehicle_number, - driver_name = EXCLUDED.driver_name, - enabled_flag = EXCLUDED.enabled_flag, + device_name = EXCLUDED.device_name, + mc_type = EXCLUDED.mc_type, + mc_type_use_scope = EXCLUDED.mc_type_use_scope, + vehicle_name = EXCLUDED.vehicle_name, + vehicle_number = EXCLUDED.vehicle_number, + vehicle_models = EXCLUDED.vehicle_models, + vehicle_icon = EXCLUDED.vehicle_icon, + vin = EXCLUDED.vin, + engine_number = EXCLUDED.engine_number, + vehicle_brand = EXCLUDED.vehicle_brand, + fuel_100km = EXCLUDED.fuel_100km, + driver_name = EXCLUDED.driver_name, + driver_phone = EXCLUDED.driver_phone, + sim = EXCLUDED.sim, + iccid = EXCLUDED.iccid, + imsi = EXCLUDED.imsi, + account = EXCLUDED.account, + customer_name = EXCLUDED.customer_name, + device_group_id = EXCLUDED.device_group_id, + device_group = EXCLUDED.device_group, + activation_time = EXCLUDED.activation_time, + expiration = EXCLUDED.expiration, + enabled_flag = EXCLUDED.enabled_flag, + status = EXCLUDED.status, current_mileage_km = EXCLUDED.current_mileage_km, - last_synced_at = NOW(), updated_at = NOW() + last_synced_at = NOW(), + updated_at = NOW() """, ( imei, clean(d.get("deviceName")), clean(d.get("mcType")), clean(d.get("mcTypeUseScope")), clean(d.get("vehicleName")), clean(d.get("vehicleNumber")), clean(d.get("vehicleModels")), clean(d.get("vehicleIcon")), @@ -191,9 +212,12 @@ def poll_trips(): with get_conn() as conn: with conn.cursor() as cur: for t in trips: - # [FIX-M11] API returns distance in km. Store directly as distance_km. - # Previous code multiplied by 1000 (→ mm), which was wrong. - dist_km = clean_num(t.get("distance")) + # [FIX-M16] API returns distance in METRES despite documentation saying km. + # Confirmed via: avgSpeed(km/h) × runTimeSecond / 3600 == distance/1000. + # startMileage/endMileage are cumulative odometer in metres (same unit). + # Divide by 1000 to store as distance_km. + raw_dist = clean_num(t.get("distance")) + dist_km = round(raw_dist / 1000.0, 4) if raw_dist is not None else None cur.execute(""" INSERT INTO tracksolid.trips ( imei, start_time, end_time, distance_km, diff --git a/sync_driver_audit.py b/sync_driver_audit.py new file mode 100644 index 0000000..62edc7f --- /dev/null +++ b/sync_driver_audit.py @@ -0,0 +1,205 @@ +""" +sync_driver_audit.py — Fireside Communications · Driver & IMEI Audit Sync +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +One-shot script: fetches ALL devices from Tracksolid API, compares driver +and IMEI details against the DB, reports gaps, and populates missing data. + +Run inside the container: + docker exec -it python sync_driver_audit.py + +Or via Coolify terminal with env vars loaded. +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +""" + +import time +from ts_shared_rev import ( + TARGET_ACCOUNT, + api_post, + get_conn, + get_token, + clean, + clean_num, + clean_int, + clean_ts, + get_logger, +) + +log = get_logger("driver_audit") + + +def run_audit(): + log.info("=== Driver & IMEI Audit Sync ===") + t0 = time.time() + token = get_token() + if not token: + log.error("Could not obtain API token. Check credentials.") + return + + # 1. Fetch all devices from API + resp = api_post("jimi.user.device.list", {"target": TARGET_ACCOUNT}, token) + if resp.get("code") != 0: + log.error("API error: %s", resp) + return + + api_devices = resp.get("result") or [] + log.info("API returned %d devices.", len(api_devices)) + + # 2. Fetch current DB state + with get_conn() as conn: + with conn.cursor() as cur: + cur.execute(""" + SELECT imei, device_name, driver_name, driver_phone, sim, status + FROM tracksolid.devices + ORDER BY imei + """) + db_rows = {row[0]: { + "device_name": row[1], + "driver_name": row[2], + "driver_phone": row[3], + "sim": row[4], + "status": row[5], + } for row in cur.fetchall()} + + log.info("DB has %d devices registered.", len(db_rows)) + + # 3. Compare and report gaps + api_imeis = set() + missing_from_db = [] + driver_gaps = [] + driver_phone_gaps = [] + + for d in api_devices: + imei = d.get("imei") + if not imei: + continue + api_imeis.add(imei) + + if imei not in db_rows: + missing_from_db.append(imei) + else: + db = db_rows[imei] + if not db["driver_name"] and clean(d.get("driverName")): + driver_gaps.append((imei, clean(d.get("driverName")))) + if not db["driver_phone"] and clean(d.get("driverPhone")): + driver_phone_gaps.append((imei, clean(d.get("driverPhone")))) + + orphaned_in_db = set(db_rows.keys()) - api_imeis + + # 4. Print gap report + print("\n" + "="*60) + print("AUDIT REPORT") + print("="*60) + print(f" API devices : {len(api_imeis)}") + print(f" DB devices : {len(db_rows)}") + print(f" New (API only): {len(missing_from_db)}") + print(f" Orphaned (DB) : {len(orphaned_in_db)}") + print(f" Missing driver_name (API has, DB null): {len(driver_gaps)}") + print(f" Missing driver_phone (API has, DB null): {len(driver_phone_gaps)}") + + if missing_from_db: + print(f"\nIMEIs NOT in DB ({len(missing_from_db)}):") + for imei in missing_from_db: + print(f" {imei}") + + if driver_gaps: + print(f"\nDevices missing driver_name in DB ({len(driver_gaps)}):") + for imei, name in driver_gaps: + print(f" {imei} → '{name}'") + + if driver_phone_gaps: + print(f"\nDevices missing driver_phone in DB ({len(driver_phone_gaps)}):") + for imei, phone in driver_phone_gaps: + print(f" {imei} → '{phone}'") + + if orphaned_in_db: + print(f"\nIMEIs in DB but NOT in API (orphaned/deactivated) ({len(orphaned_in_db)}):") + for imei in sorted(orphaned_in_db): + print(f" {imei}") + + print("="*60) + + # 5. Upsert ALL devices with full field sync (including driver info) + log.info("Starting full upsert of %d devices...", len(api_devices)) + upserted = 0 + + with get_conn() as conn: + with conn.cursor() as cur: + for d in api_devices: + imei = d.get("imei") + if not imei: + continue + + # Fetch detailed info for driver phone, SIM, ICCID etc. + detail_resp = api_post("jimi.track.device.detail", {"imei": imei}, token) + dtl = detail_resp.get("result") or {} if detail_resp.get("code") == 0 else {} + + cur.execute(""" + INSERT INTO tracksolid.devices ( + imei, device_name, mc_type, mc_type_use_scope, + vehicle_name, vehicle_number, vehicle_models, vehicle_icon, + vin, engine_number, vehicle_brand, fuel_100km, + driver_name, driver_phone, sim, iccid, imsi, + account, customer_name, device_group_id, device_group, + activation_time, expiration, enabled_flag, status, + current_mileage_km, last_synced_at + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW() + ) + ON CONFLICT (imei) DO UPDATE SET + device_name = EXCLUDED.device_name, + mc_type = EXCLUDED.mc_type, + mc_type_use_scope = EXCLUDED.mc_type_use_scope, + vehicle_name = EXCLUDED.vehicle_name, + vehicle_number = EXCLUDED.vehicle_number, + vehicle_models = EXCLUDED.vehicle_models, + vehicle_icon = EXCLUDED.vehicle_icon, + vin = EXCLUDED.vin, + engine_number = EXCLUDED.engine_number, + vehicle_brand = EXCLUDED.vehicle_brand, + fuel_100km = EXCLUDED.fuel_100km, + driver_name = EXCLUDED.driver_name, + driver_phone = EXCLUDED.driver_phone, + sim = EXCLUDED.sim, + iccid = EXCLUDED.iccid, + imsi = EXCLUDED.imsi, + account = EXCLUDED.account, + customer_name = EXCLUDED.customer_name, + device_group_id = EXCLUDED.device_group_id, + device_group = EXCLUDED.device_group, + activation_time = EXCLUDED.activation_time, + expiration = EXCLUDED.expiration, + enabled_flag = EXCLUDED.enabled_flag, + status = EXCLUDED.status, + current_mileage_km = EXCLUDED.current_mileage_km, + last_synced_at = NOW(), + updated_at = NOW() + """, ( + imei, + clean(d.get("deviceName")), clean(d.get("mcType")), + clean(d.get("mcTypeUseScope")), clean(d.get("vehicleName")), + clean(d.get("vehicleNumber")), clean(d.get("vehicleModels")), + clean(d.get("vehicleIcon")), + clean(dtl.get("vin")), clean(dtl.get("engineNumber")), + clean(dtl.get("vehicleBrand")), clean_num(dtl.get("fuel_100km")), + clean(d.get("driverName")), clean(d.get("driverPhone")), + clean(d.get("sim")), clean(dtl.get("iccid")), + clean(dtl.get("imsi")), + clean(dtl.get("account")), clean(dtl.get("customerName")), + clean(d.get("deviceGroupId")), clean(d.get("deviceGroup")), + clean_ts(d.get("activationTime")), clean_ts(d.get("expiration")), + clean_int(d.get("enabledFlag", 1)), + clean(dtl.get("status", "active")), + clean_num(dtl.get("currentMileage")), + )) + upserted += 1 + + conn.commit() + + elapsed = int((time.time() - t0) * 1000) + log.info("Done. Upserted %d devices in %dms.", upserted, elapsed) + print(f"\nSync complete: {upserted} devices upserted in {elapsed}ms.") + + +if __name__ == "__main__": + run_audit()