Fix trip distance unit (metres→km) and full device sync on upsert
[FIX-M16] jimi.device.track.mileage returns distance in metres despite docs claiming km. Confirmed: avgSpeed × runTimeSecond / 3600 = distance/1000. poll_trips() now divides raw value by 1000 before storing as distance_km. 3 existing bad rows corrected in prod DB (distance_km / 1000). [FIX-M17] sync_devices() ON CONFLICT clause was only updating 5 of 26 fields, silently dropping driver_phone, sim, iccid, vehicle_name, status etc. on subsequent syncs. Expanded to update all device fields so driver assignments made in Tracksolid Pro UI propagate to DB on next daily sync. Add sync_driver_audit.py: one-shot script to compare API vs DB device registry, report driver/IMEI gaps, and force a full field upsert. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
David Kiania
parent
fcc745f09d
commit
6a0ceb78dd
2 changed files with 237 additions and 8 deletions
|
|
@ -93,12 +93,33 @@ def sync_devices():
|
|||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()
|
||||
)
|
||||
ON CONFLICT (imei) DO UPDATE SET
|
||||
device_name = EXCLUDED.device_name,
|
||||
vehicle_number = EXCLUDED.vehicle_number,
|
||||
driver_name = EXCLUDED.driver_name,
|
||||
enabled_flag = EXCLUDED.enabled_flag,
|
||||
device_name = EXCLUDED.device_name,
|
||||
mc_type = EXCLUDED.mc_type,
|
||||
mc_type_use_scope = EXCLUDED.mc_type_use_scope,
|
||||
vehicle_name = EXCLUDED.vehicle_name,
|
||||
vehicle_number = EXCLUDED.vehicle_number,
|
||||
vehicle_models = EXCLUDED.vehicle_models,
|
||||
vehicle_icon = EXCLUDED.vehicle_icon,
|
||||
vin = EXCLUDED.vin,
|
||||
engine_number = EXCLUDED.engine_number,
|
||||
vehicle_brand = EXCLUDED.vehicle_brand,
|
||||
fuel_100km = EXCLUDED.fuel_100km,
|
||||
driver_name = EXCLUDED.driver_name,
|
||||
driver_phone = EXCLUDED.driver_phone,
|
||||
sim = EXCLUDED.sim,
|
||||
iccid = EXCLUDED.iccid,
|
||||
imsi = EXCLUDED.imsi,
|
||||
account = EXCLUDED.account,
|
||||
customer_name = EXCLUDED.customer_name,
|
||||
device_group_id = EXCLUDED.device_group_id,
|
||||
device_group = EXCLUDED.device_group,
|
||||
activation_time = EXCLUDED.activation_time,
|
||||
expiration = EXCLUDED.expiration,
|
||||
enabled_flag = EXCLUDED.enabled_flag,
|
||||
status = EXCLUDED.status,
|
||||
current_mileage_km = EXCLUDED.current_mileage_km,
|
||||
last_synced_at = NOW(), updated_at = NOW()
|
||||
last_synced_at = NOW(),
|
||||
updated_at = NOW()
|
||||
""", (
|
||||
imei, clean(d.get("deviceName")), clean(d.get("mcType")), clean(d.get("mcTypeUseScope")),
|
||||
clean(d.get("vehicleName")), clean(d.get("vehicleNumber")), clean(d.get("vehicleModels")), clean(d.get("vehicleIcon")),
|
||||
|
|
@ -191,9 +212,12 @@ def poll_trips():
|
|||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
for t in trips:
|
||||
# [FIX-M11] API returns distance in km. Store directly as distance_km.
|
||||
# Previous code multiplied by 1000 (→ mm), which was wrong.
|
||||
dist_km = clean_num(t.get("distance"))
|
||||
# [FIX-M16] API returns distance in METRES despite documentation saying km.
|
||||
# Confirmed via: avgSpeed(km/h) × runTimeSecond / 3600 == distance/1000.
|
||||
# startMileage/endMileage are cumulative odometer in metres (same unit).
|
||||
# Divide by 1000 to store as distance_km.
|
||||
raw_dist = clean_num(t.get("distance"))
|
||||
dist_km = round(raw_dist / 1000.0, 4) if raw_dist is not None else None
|
||||
cur.execute("""
|
||||
INSERT INTO tracksolid.trips (
|
||||
imei, start_time, end_time, distance_km,
|
||||
|
|
|
|||
205
sync_driver_audit.py
Normal file
205
sync_driver_audit.py
Normal file
|
|
@ -0,0 +1,205 @@
|
|||
"""
|
||||
sync_driver_audit.py — Fireside Communications · Driver & IMEI Audit Sync
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
One-shot script: fetches ALL devices from Tracksolid API, compares driver
|
||||
and IMEI details against the DB, reports gaps, and populates missing data.
|
||||
|
||||
Run inside the container:
|
||||
docker exec -it <ingest_movement_container> python sync_driver_audit.py
|
||||
|
||||
Or via Coolify terminal with env vars loaded.
|
||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||
"""
|
||||
|
||||
import time
|
||||
from ts_shared_rev import (
|
||||
TARGET_ACCOUNT,
|
||||
api_post,
|
||||
get_conn,
|
||||
get_token,
|
||||
clean,
|
||||
clean_num,
|
||||
clean_int,
|
||||
clean_ts,
|
||||
get_logger,
|
||||
)
|
||||
|
||||
log = get_logger("driver_audit")
|
||||
|
||||
|
||||
def run_audit():
|
||||
log.info("=== Driver & IMEI Audit Sync ===")
|
||||
t0 = time.time()
|
||||
token = get_token()
|
||||
if not token:
|
||||
log.error("Could not obtain API token. Check credentials.")
|
||||
return
|
||||
|
||||
# 1. Fetch all devices from API
|
||||
resp = api_post("jimi.user.device.list", {"target": TARGET_ACCOUNT}, token)
|
||||
if resp.get("code") != 0:
|
||||
log.error("API error: %s", resp)
|
||||
return
|
||||
|
||||
api_devices = resp.get("result") or []
|
||||
log.info("API returned %d devices.", len(api_devices))
|
||||
|
||||
# 2. Fetch current DB state
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT imei, device_name, driver_name, driver_phone, sim, status
|
||||
FROM tracksolid.devices
|
||||
ORDER BY imei
|
||||
""")
|
||||
db_rows = {row[0]: {
|
||||
"device_name": row[1],
|
||||
"driver_name": row[2],
|
||||
"driver_phone": row[3],
|
||||
"sim": row[4],
|
||||
"status": row[5],
|
||||
} for row in cur.fetchall()}
|
||||
|
||||
log.info("DB has %d devices registered.", len(db_rows))
|
||||
|
||||
# 3. Compare and report gaps
|
||||
api_imeis = set()
|
||||
missing_from_db = []
|
||||
driver_gaps = []
|
||||
driver_phone_gaps = []
|
||||
|
||||
for d in api_devices:
|
||||
imei = d.get("imei")
|
||||
if not imei:
|
||||
continue
|
||||
api_imeis.add(imei)
|
||||
|
||||
if imei not in db_rows:
|
||||
missing_from_db.append(imei)
|
||||
else:
|
||||
db = db_rows[imei]
|
||||
if not db["driver_name"] and clean(d.get("driverName")):
|
||||
driver_gaps.append((imei, clean(d.get("driverName"))))
|
||||
if not db["driver_phone"] and clean(d.get("driverPhone")):
|
||||
driver_phone_gaps.append((imei, clean(d.get("driverPhone"))))
|
||||
|
||||
orphaned_in_db = set(db_rows.keys()) - api_imeis
|
||||
|
||||
# 4. Print gap report
|
||||
print("\n" + "="*60)
|
||||
print("AUDIT REPORT")
|
||||
print("="*60)
|
||||
print(f" API devices : {len(api_imeis)}")
|
||||
print(f" DB devices : {len(db_rows)}")
|
||||
print(f" New (API only): {len(missing_from_db)}")
|
||||
print(f" Orphaned (DB) : {len(orphaned_in_db)}")
|
||||
print(f" Missing driver_name (API has, DB null): {len(driver_gaps)}")
|
||||
print(f" Missing driver_phone (API has, DB null): {len(driver_phone_gaps)}")
|
||||
|
||||
if missing_from_db:
|
||||
print(f"\nIMEIs NOT in DB ({len(missing_from_db)}):")
|
||||
for imei in missing_from_db:
|
||||
print(f" {imei}")
|
||||
|
||||
if driver_gaps:
|
||||
print(f"\nDevices missing driver_name in DB ({len(driver_gaps)}):")
|
||||
for imei, name in driver_gaps:
|
||||
print(f" {imei} → '{name}'")
|
||||
|
||||
if driver_phone_gaps:
|
||||
print(f"\nDevices missing driver_phone in DB ({len(driver_phone_gaps)}):")
|
||||
for imei, phone in driver_phone_gaps:
|
||||
print(f" {imei} → '{phone}'")
|
||||
|
||||
if orphaned_in_db:
|
||||
print(f"\nIMEIs in DB but NOT in API (orphaned/deactivated) ({len(orphaned_in_db)}):")
|
||||
for imei in sorted(orphaned_in_db):
|
||||
print(f" {imei}")
|
||||
|
||||
print("="*60)
|
||||
|
||||
# 5. Upsert ALL devices with full field sync (including driver info)
|
||||
log.info("Starting full upsert of %d devices...", len(api_devices))
|
||||
upserted = 0
|
||||
|
||||
with get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
for d in api_devices:
|
||||
imei = d.get("imei")
|
||||
if not imei:
|
||||
continue
|
||||
|
||||
# Fetch detailed info for driver phone, SIM, ICCID etc.
|
||||
detail_resp = api_post("jimi.track.device.detail", {"imei": imei}, token)
|
||||
dtl = detail_resp.get("result") or {} if detail_resp.get("code") == 0 else {}
|
||||
|
||||
cur.execute("""
|
||||
INSERT INTO tracksolid.devices (
|
||||
imei, device_name, mc_type, mc_type_use_scope,
|
||||
vehicle_name, vehicle_number, vehicle_models, vehicle_icon,
|
||||
vin, engine_number, vehicle_brand, fuel_100km,
|
||||
driver_name, driver_phone, sim, iccid, imsi,
|
||||
account, customer_name, device_group_id, device_group,
|
||||
activation_time, expiration, enabled_flag, status,
|
||||
current_mileage_km, last_synced_at
|
||||
) VALUES (
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
|
||||
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW()
|
||||
)
|
||||
ON CONFLICT (imei) DO UPDATE SET
|
||||
device_name = EXCLUDED.device_name,
|
||||
mc_type = EXCLUDED.mc_type,
|
||||
mc_type_use_scope = EXCLUDED.mc_type_use_scope,
|
||||
vehicle_name = EXCLUDED.vehicle_name,
|
||||
vehicle_number = EXCLUDED.vehicle_number,
|
||||
vehicle_models = EXCLUDED.vehicle_models,
|
||||
vehicle_icon = EXCLUDED.vehicle_icon,
|
||||
vin = EXCLUDED.vin,
|
||||
engine_number = EXCLUDED.engine_number,
|
||||
vehicle_brand = EXCLUDED.vehicle_brand,
|
||||
fuel_100km = EXCLUDED.fuel_100km,
|
||||
driver_name = EXCLUDED.driver_name,
|
||||
driver_phone = EXCLUDED.driver_phone,
|
||||
sim = EXCLUDED.sim,
|
||||
iccid = EXCLUDED.iccid,
|
||||
imsi = EXCLUDED.imsi,
|
||||
account = EXCLUDED.account,
|
||||
customer_name = EXCLUDED.customer_name,
|
||||
device_group_id = EXCLUDED.device_group_id,
|
||||
device_group = EXCLUDED.device_group,
|
||||
activation_time = EXCLUDED.activation_time,
|
||||
expiration = EXCLUDED.expiration,
|
||||
enabled_flag = EXCLUDED.enabled_flag,
|
||||
status = EXCLUDED.status,
|
||||
current_mileage_km = EXCLUDED.current_mileage_km,
|
||||
last_synced_at = NOW(),
|
||||
updated_at = NOW()
|
||||
""", (
|
||||
imei,
|
||||
clean(d.get("deviceName")), clean(d.get("mcType")),
|
||||
clean(d.get("mcTypeUseScope")), clean(d.get("vehicleName")),
|
||||
clean(d.get("vehicleNumber")), clean(d.get("vehicleModels")),
|
||||
clean(d.get("vehicleIcon")),
|
||||
clean(dtl.get("vin")), clean(dtl.get("engineNumber")),
|
||||
clean(dtl.get("vehicleBrand")), clean_num(dtl.get("fuel_100km")),
|
||||
clean(d.get("driverName")), clean(d.get("driverPhone")),
|
||||
clean(d.get("sim")), clean(dtl.get("iccid")),
|
||||
clean(dtl.get("imsi")),
|
||||
clean(dtl.get("account")), clean(dtl.get("customerName")),
|
||||
clean(d.get("deviceGroupId")), clean(d.get("deviceGroup")),
|
||||
clean_ts(d.get("activationTime")), clean_ts(d.get("expiration")),
|
||||
clean_int(d.get("enabledFlag", 1)),
|
||||
clean(dtl.get("status", "active")),
|
||||
clean_num(dtl.get("currentMileage")),
|
||||
))
|
||||
upserted += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
elapsed = int((time.time() - t0) * 1000)
|
||||
log.info("Done. Upserted %d devices in %dms.", upserted, elapsed)
|
||||
print(f"\nSync complete: {upserted} devices upserted in {elapsed}ms.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_audit()
|
||||
Loading…
Reference in a new issue