Fix alarm field mapping, distance unit bug, parking params; add schema migrations
BUG-01 [FIX-E06]: jimi.device.alarm.list poll response uses alertTypeId/ alarmTypeName/alertTime, not the webhook field names. All 1,054 stored alarm records had null alarm_type/alarm_name as a result. Corrected field mapping in ingest_events_rev.py; also added alarm_name and source columns to INSERT. BUG-02 [FIX-M11/M12]: trips.distance_m was storing millimetres due to an erroneous * 1000 on an already-km API value. Removed the multiplication in poll_trips() and push_trip_report(). Column renamed to distance_km in migration 04 (historical rows divided by 1,000,000 to correct to km). All SQL in both ingestion files updated to reference distance_km. POLL-02 [FIX-M13]: parking poll returned 0 rows because the required account and acc_type=0 parameters were missing. Also fixed response field mapping: durSecond was incorrectly read as 'seconds'. Migration 04: corrects and renames distance_m → distance_km. Migration 05: adds normalized OBD columns, alarm/device enrichment columns, new tables (device_events, fuel_readings, temperature_readings, lbs_readings, geofences), expands dwh_gold fact table, and adds refresh_daily_metrics() ETL. tracksolid_DB_manual.md updated to reflect column rename and mark fixed issues. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
David Kiania
parent
791bf2700c
commit
c05b47abe2
6 changed files with 1142 additions and 20 deletions
36
04_bug_fix_migration.sql
Normal file
36
04_bug_fix_migration.sql
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
-- Migration 04 — Bug Fix: distance unit correction + column rename
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
-- BUG-02: tracksolid.trips.distance_m was storing millimetres not metres.
|
||||||
|
--
|
||||||
|
-- Root cause: ingestion code applied `km * 1000` but the API already returns
|
||||||
|
-- values in km, producing mm. Confirmed by cross-checking stored values against
|
||||||
|
-- avg_speed_kmh × driving_time_s — e.g. 4,203,000 stored for a 4.203 km trip.
|
||||||
|
--
|
||||||
|
-- Fix applied here:
|
||||||
|
-- 1. Divide all existing rows by 1,000,000 to convert mm → km.
|
||||||
|
-- 2. Rename the column to distance_km to eliminate future ambiguity.
|
||||||
|
--
|
||||||
|
-- Corresponding code fix: removed * 1000 from poll_trips() in
|
||||||
|
-- ingest_movement_rev.py and push_trip_report() in webhook_receiver_rev.py.
|
||||||
|
-- Both now store the raw API value directly as km.
|
||||||
|
--
|
||||||
|
-- Run once against tracksolid_db before deploying updated ingestion containers.
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- Step 1: Correct all existing stored values (mm → km)
|
||||||
|
UPDATE tracksolid.trips
|
||||||
|
SET distance_m = distance_m / 1000000.0
|
||||||
|
WHERE distance_m IS NOT NULL;
|
||||||
|
|
||||||
|
-- Step 2: Rename column
|
||||||
|
ALTER TABLE tracksolid.trips
|
||||||
|
RENAME COLUMN distance_m TO distance_km;
|
||||||
|
|
||||||
|
-- Step 3: Update column comment
|
||||||
|
COMMENT ON COLUMN tracksolid.trips.distance_km
|
||||||
|
IS 'Trip distance in kilometres. Corrected from mm storage on migration 04 (2026-04-10).';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
270
05_enhancement_migration.sql
Normal file
270
05_enhancement_migration.sql
Normal file
|
|
@ -0,0 +1,270 @@
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
-- Migration 05 — Schema Enhancements for Expanded Ingestion
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
-- Adds columns and tables to support:
|
||||||
|
-- • Normalized OBD scalar fields (from /pushobd JSONB payload)
|
||||||
|
-- • Alarm enrichment (severity, geofence context, acknowledgement)
|
||||||
|
-- • Vehicle enrichment (category, cost centre, depot location)
|
||||||
|
-- • New webhook endpoints: /pushevent, /pushoil, /pushtem, /pushlbs
|
||||||
|
-- • Geofence definition storage
|
||||||
|
-- • dwh_gold fact table expansion for full daily KPI reporting
|
||||||
|
--
|
||||||
|
-- Run after migration 04. Safe to re-run (uses IF NOT EXISTS / DO NOTHING).
|
||||||
|
-- ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- ── 1. Normalize OBD scalar fields ───────────────────────────────────────────
|
||||||
|
-- These are extracted from the obd_data JSONB column during /pushobd ingestion.
|
||||||
|
-- Raw JSONB is retained for full fidelity. Common OBD PID values only.
|
||||||
|
|
||||||
|
ALTER TABLE tracksolid.obd_readings
|
||||||
|
ADD COLUMN IF NOT EXISTS engine_rpm INTEGER,
|
||||||
|
ADD COLUMN IF NOT EXISTS coolant_temp_c NUMERIC(6,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS fuel_level_pct NUMERIC(5,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS battery_voltage NUMERIC(5,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS intake_pressure NUMERIC(6,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS throttle_pct NUMERIC(5,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS vehicle_speed NUMERIC(7,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS engine_load_pct NUMERIC(5,2);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.engine_rpm IS 'Engine RPM from OBD PID 0x0C';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.coolant_temp_c IS 'Coolant temperature °C from OBD PID 0x05';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.fuel_level_pct IS 'Fuel tank level % from OBD PID 0x2F';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.battery_voltage IS 'Battery voltage (V) from OBD PID 0x42';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.intake_pressure IS 'Intake manifold pressure kPa from OBD PID 0x0B';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.throttle_pct IS 'Throttle position % from OBD PID 0x11';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.vehicle_speed IS 'Vehicle speed km/h from OBD PID 0x0D';
|
||||||
|
COMMENT ON COLUMN tracksolid.obd_readings.engine_load_pct IS 'Calculated engine load % from OBD PID 0x04';
|
||||||
|
|
||||||
|
-- ── 2. Alarm enrichment ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
ALTER TABLE tracksolid.alarms
|
||||||
|
ADD COLUMN IF NOT EXISTS severity TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS geofence_id TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS geofence_name TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS acknowledged_at TIMESTAMPTZ,
|
||||||
|
ADD COLUMN IF NOT EXISTS acknowledged_by TEXT;
|
||||||
|
|
||||||
|
COMMENT ON COLUMN tracksolid.alarms.severity IS 'Alarm severity level: critical | warning | info';
|
||||||
|
COMMENT ON COLUMN tracksolid.alarms.geofence_id IS 'Tracksolid geofence ID if this is a geofence alarm';
|
||||||
|
COMMENT ON COLUMN tracksolid.alarms.geofence_name IS 'Human-readable geofence name';
|
||||||
|
COMMENT ON COLUMN tracksolid.alarms.acknowledged_at IS 'Timestamp when alarm was acknowledged by an operator';
|
||||||
|
COMMENT ON COLUMN tracksolid.alarms.acknowledged_by IS 'Username or ID of operator who acknowledged the alarm';
|
||||||
|
|
||||||
|
-- ── 3. Vehicle enrichment ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
ALTER TABLE tracksolid.devices
|
||||||
|
ADD COLUMN IF NOT EXISTS vehicle_category TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS cost_centre TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS assigned_route TEXT,
|
||||||
|
ADD COLUMN IF NOT EXISTS depot_geom geometry(Point,4326),
|
||||||
|
ADD COLUMN IF NOT EXISTS depot_address TEXT;
|
||||||
|
|
||||||
|
COMMENT ON COLUMN tracksolid.devices.vehicle_category IS 'Vehicle type: truck | van | motorcycle | car | other';
|
||||||
|
COMMENT ON COLUMN tracksolid.devices.cost_centre IS 'Business unit or department this vehicle belongs to';
|
||||||
|
COMMENT ON COLUMN tracksolid.devices.assigned_route IS 'Regular route name or ID for route-based reporting';
|
||||||
|
COMMENT ON COLUMN tracksolid.devices.depot_geom IS 'Home base/depot coordinates (WGS84)';
|
||||||
|
COMMENT ON COLUMN tracksolid.devices.depot_address IS 'Human-readable depot address';
|
||||||
|
|
||||||
|
-- ── 4. Device login/logout events (webhook /pushevent) ────────────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tracksolid.device_events (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
imei TEXT NOT NULL REFERENCES tracksolid.devices(imei),
|
||||||
|
event_type TEXT NOT NULL, -- 'LOGIN' | 'LOGOUT'
|
||||||
|
event_time TIMESTAMPTZ NOT NULL,
|
||||||
|
timezone TEXT,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
UNIQUE (imei, event_type, event_time)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_device_events_imei_time
|
||||||
|
ON tracksolid.device_events (imei, event_time DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE tracksolid.device_events
|
||||||
|
IS 'Device network connection and disconnection events from /pushevent webhook.';
|
||||||
|
COMMENT ON COLUMN tracksolid.device_events.event_type
|
||||||
|
IS 'LOGIN = device connected to network; LOGOUT = device disconnected';
|
||||||
|
|
||||||
|
-- ── 5. Fuel sensor readings (webhook /pushoil) — hypertable ──────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tracksolid.fuel_readings (
|
||||||
|
imei TEXT NOT NULL REFERENCES tracksolid.devices(imei),
|
||||||
|
reading_time TIMESTAMPTZ NOT NULL,
|
||||||
|
sensor_path TEXT,
|
||||||
|
value NUMERIC(10,3),
|
||||||
|
unit TEXT,
|
||||||
|
lat DOUBLE PRECISION,
|
||||||
|
lng DOUBLE PRECISION,
|
||||||
|
geom geometry(Point,4326),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (imei, reading_time)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE tracksolid.fuel_readings
|
||||||
|
IS 'Fuel/oil sensor readings from /pushoil webhook. Unit varies per sensor: cm | % | V | L.';
|
||||||
|
COMMENT ON COLUMN tracksolid.fuel_readings.sensor_path
|
||||||
|
IS 'Sensor channel identifier from the device (path field in API payload)';
|
||||||
|
COMMENT ON COLUMN tracksolid.fuel_readings.unit
|
||||||
|
IS 'Measurement unit: cm (tank depth), % (percentage), V (voltage), L (litres)';
|
||||||
|
|
||||||
|
SELECT create_hypertable(
|
||||||
|
'tracksolid.fuel_readings', 'reading_time',
|
||||||
|
chunk_time_interval => INTERVAL '7 days',
|
||||||
|
if_not_exists => TRUE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── 6. Temperature & humidity readings (webhook /pushtem) — hypertable ────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tracksolid.temperature_readings (
|
||||||
|
imei TEXT NOT NULL REFERENCES tracksolid.devices(imei),
|
||||||
|
reading_time TIMESTAMPTZ NOT NULL,
|
||||||
|
temperature NUMERIC(6,2),
|
||||||
|
humidity_pct NUMERIC(5,2),
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (imei, reading_time)
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE tracksolid.temperature_readings
|
||||||
|
IS 'Temperature and humidity sensor readings from /pushtem webhook. For cold-chain / refrigerated cargo monitoring.';
|
||||||
|
|
||||||
|
SELECT create_hypertable(
|
||||||
|
'tracksolid.temperature_readings', 'reading_time',
|
||||||
|
chunk_time_interval => INTERVAL '7 days',
|
||||||
|
if_not_exists => TRUE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── 7. LBS / cell-tower fallback positions (webhook /pushlbs) ────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tracksolid.lbs_readings (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
imei TEXT NOT NULL REFERENCES tracksolid.devices(imei),
|
||||||
|
gate_time TIMESTAMPTZ NOT NULL,
|
||||||
|
post_type TEXT,
|
||||||
|
lbs_data JSONB,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
UNIQUE (imei, gate_time)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_lbs_readings_imei_time
|
||||||
|
ON tracksolid.lbs_readings (imei, gate_time DESC);
|
||||||
|
|
||||||
|
COMMENT ON TABLE tracksolid.lbs_readings
|
||||||
|
IS 'Cell tower / WiFi positioning fallback data from /pushlbs webhook. Used when GPS signal is unavailable.';
|
||||||
|
COMMENT ON COLUMN tracksolid.lbs_readings.post_type
|
||||||
|
IS 'Positioning technology: WIFI | LBS (cell tower)';
|
||||||
|
COMMENT ON COLUMN tracksolid.lbs_readings.lbs_data
|
||||||
|
IS 'Raw JSON payload containing MCC, MNC, and cell tower list for approximate geocoding.';
|
||||||
|
|
||||||
|
-- ── 8. Geofence definitions ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS tracksolid.geofences (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
fence_id TEXT UNIQUE,
|
||||||
|
fence_name TEXT NOT NULL,
|
||||||
|
fence_type TEXT,
|
||||||
|
geom geometry(Geometry,4326),
|
||||||
|
radius_m NUMERIC(10,2),
|
||||||
|
description TEXT,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||||
|
);
|
||||||
|
|
||||||
|
COMMENT ON TABLE tracksolid.geofences
|
||||||
|
IS 'Geofence boundary definitions synced from the Tracksolid platform.';
|
||||||
|
COMMENT ON COLUMN tracksolid.geofences.fence_type
|
||||||
|
IS 'circle | polygon';
|
||||||
|
COMMENT ON COLUMN tracksolid.geofences.radius_m
|
||||||
|
IS 'Radius in metres — only applicable for circle type geofences';
|
||||||
|
|
||||||
|
-- ── 9. Expand dwh_gold.fact_daily_fleet_metrics ───────────────────────────────
|
||||||
|
|
||||||
|
ALTER TABLE dwh_gold.fact_daily_fleet_metrics
|
||||||
|
ADD COLUMN IF NOT EXISTS total_distance_km NUMERIC(12,3),
|
||||||
|
ADD COLUMN IF NOT EXISTS total_trips INTEGER,
|
||||||
|
ADD COLUMN IF NOT EXISTS total_drive_hours NUMERIC(8,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS total_idle_hours NUMERIC(8,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS fuel_consumed_l NUMERIC(10,3),
|
||||||
|
ADD COLUMN IF NOT EXISTS alarm_count INTEGER,
|
||||||
|
ADD COLUMN IF NOT EXISTS overspeed_count INTEGER,
|
||||||
|
ADD COLUMN IF NOT EXISTS day_start_time TIME,
|
||||||
|
ADD COLUMN IF NOT EXISTS day_end_time TIME,
|
||||||
|
ADD COLUMN IF NOT EXISTS avg_speed_kmh NUMERIC(7,2),
|
||||||
|
ADD COLUMN IF NOT EXISTS peak_speed_kmh NUMERIC(7,2);
|
||||||
|
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.total_distance_km IS 'Total km driven that day across all trips';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.total_trips IS 'Number of completed trips';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.total_drive_hours IS 'Total hours of active driving (engine on + moving)';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.total_idle_hours IS 'Total hours engine on but stationary';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.fuel_consumed_l IS 'Total fuel consumed in litres (from webhook trip reports)';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.alarm_count IS 'Total alarm events triggered that day';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.overspeed_count IS 'Number of overspeed alarm events';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.day_start_time IS 'Time of first trip start (Africa/Nairobi)';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.day_end_time IS 'Time of last trip end (Africa/Nairobi)';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.avg_speed_kmh IS 'Fleet average speed across all trips that day';
|
||||||
|
COMMENT ON COLUMN dwh_gold.fact_daily_fleet_metrics.peak_speed_kmh IS 'Highest max_speed_kmh recorded across all trips';
|
||||||
|
|
||||||
|
-- ── 10. ETL function — refresh daily metrics ──────────────────────────────────
|
||||||
|
-- Populates dwh_gold.fact_daily_fleet_metrics for a given date.
|
||||||
|
-- Call nightly: SELECT dwh_gold.refresh_daily_metrics(CURRENT_DATE - 1);
|
||||||
|
|
||||||
|
CREATE OR REPLACE FUNCTION dwh_gold.refresh_daily_metrics(target_date DATE)
|
||||||
|
RETURNS void LANGUAGE plpgsql AS $$
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO dwh_gold.fact_daily_fleet_metrics (
|
||||||
|
day,
|
||||||
|
vehicle_key,
|
||||||
|
total_distance_km,
|
||||||
|
total_trips,
|
||||||
|
total_drive_hours,
|
||||||
|
total_idle_hours,
|
||||||
|
fuel_consumed_l,
|
||||||
|
alarm_count,
|
||||||
|
overspeed_count,
|
||||||
|
day_start_time,
|
||||||
|
day_end_time,
|
||||||
|
avg_speed_kmh,
|
||||||
|
peak_speed_kmh
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
target_date AS day,
|
||||||
|
t.imei AS vehicle_key,
|
||||||
|
ROUND(SUM(t.distance_km)::numeric, 3) AS total_distance_km,
|
||||||
|
COUNT(*) AS total_trips,
|
||||||
|
ROUND((SUM(t.driving_time_s) / 3600.0)::numeric, 2) AS total_drive_hours,
|
||||||
|
ROUND((SUM(t.idle_time_s) / 3600.0)::numeric, 2) AS total_idle_hours,
|
||||||
|
ROUND(SUM(t.fuel_consumed_l)::numeric, 3) AS fuel_consumed_l,
|
||||||
|
COUNT(a.id) AS alarm_count,
|
||||||
|
COUNT(a.id) FILTER (WHERE a.alarm_type ILIKE '%speed%') AS overspeed_count,
|
||||||
|
MIN(t.start_time AT TIME ZONE 'Africa/Nairobi')::TIME AS day_start_time,
|
||||||
|
MAX(t.end_time AT TIME ZONE 'Africa/Nairobi')::TIME AS day_end_time,
|
||||||
|
ROUND(AVG(t.avg_speed_kmh)::numeric, 2) AS avg_speed_kmh,
|
||||||
|
MAX(t.max_speed_kmh) AS peak_speed_kmh
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
LEFT JOIN tracksolid.alarms a
|
||||||
|
ON a.imei = t.imei
|
||||||
|
AND DATE(a.alarm_time AT TIME ZONE 'Africa/Nairobi') = target_date
|
||||||
|
WHERE DATE(t.start_time AT TIME ZONE 'Africa/Nairobi') = target_date
|
||||||
|
AND t.end_time IS NOT NULL
|
||||||
|
GROUP BY t.imei
|
||||||
|
ON CONFLICT (day, vehicle_key) DO UPDATE SET
|
||||||
|
total_distance_km = EXCLUDED.total_distance_km,
|
||||||
|
total_trips = EXCLUDED.total_trips,
|
||||||
|
total_drive_hours = EXCLUDED.total_drive_hours,
|
||||||
|
total_idle_hours = EXCLUDED.total_idle_hours,
|
||||||
|
fuel_consumed_l = EXCLUDED.fuel_consumed_l,
|
||||||
|
alarm_count = EXCLUDED.alarm_count,
|
||||||
|
overspeed_count = EXCLUDED.overspeed_count,
|
||||||
|
day_start_time = EXCLUDED.day_start_time,
|
||||||
|
day_end_time = EXCLUDED.day_end_time,
|
||||||
|
avg_speed_kmh = EXCLUDED.avg_speed_kmh,
|
||||||
|
peak_speed_kmh = EXCLUDED.peak_speed_kmh;
|
||||||
|
END;
|
||||||
|
$$;
|
||||||
|
|
||||||
|
COMMENT ON FUNCTION dwh_gold.refresh_daily_metrics(DATE)
|
||||||
|
IS 'Populates or refreshes fact_daily_fleet_metrics for the given date. '
|
||||||
|
'Call nightly: SELECT dwh_gold.refresh_daily_metrics(CURRENT_DATE - 1);';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -12,6 +12,10 @@ REVISIONS (QA-Verified):
|
||||||
[FIX-E04] Signal Handling: Clean pool closure on SIGTERM/SIGINT.
|
[FIX-E04] Signal Handling: Clean pool closure on SIGTERM/SIGINT.
|
||||||
[FIX-E05] Removed poll_obd: OBD data is push-only via /pushobd webhook.
|
[FIX-E05] Removed poll_obd: OBD data is push-only via /pushobd webhook.
|
||||||
[FIX-11] Uses shared safe_task/setup_shutdown from ts_shared_rev (DRY).
|
[FIX-11] Uses shared safe_task/setup_shutdown from ts_shared_rev (DRY).
|
||||||
|
[FIX-E06] BUG-01: jimi.device.alarm.list returns alertTypeId/alarmTypeName/
|
||||||
|
alertTime — not alarmType/alarmName/alarmTime (those are webhook
|
||||||
|
field names). Corrected field mapping so alarm_type and alarm_name
|
||||||
|
are no longer silently stored as NULL.
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -64,20 +68,25 @@ def poll_alarms():
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
for a in alarms:
|
for a in alarms:
|
||||||
lat, lng = clean_num(a.get("lat")), clean_num(a.get("lng"))
|
lat, lng = clean_num(a.get("lat")), clean_num(a.get("lng"))
|
||||||
|
# [FIX-E06] Poll response uses alertTypeId/alarmTypeName/alertTime,
|
||||||
|
# not alarmType/alarmName/alarmTime (those are webhook push field names).
|
||||||
|
alarm_type = clean(a.get("alertTypeId"))
|
||||||
|
alarm_name = clean(a.get("alarmTypeName"))
|
||||||
|
alarm_time = clean_ts(a.get("alertTime"))
|
||||||
|
|
||||||
cur.execute("""
|
cur.execute("""
|
||||||
INSERT INTO tracksolid.alarms (
|
INSERT INTO tracksolid.alarms (
|
||||||
imei, alarm_type, alarm_time, geom, lat, lng,
|
imei, alarm_type, alarm_name, alarm_time, geom, lat, lng,
|
||||||
speed, acc_status, updated_at
|
speed, acc_status, source, updated_at
|
||||||
) VALUES (
|
) VALUES (
|
||||||
%s, %s, %s,
|
%s, %s, %s, %s,
|
||||||
CASE WHEN %s IS NOT NULL AND %s IS NOT NULL
|
CASE WHEN %s IS NOT NULL AND %s IS NOT NULL
|
||||||
THEN ST_SetSRID(ST_MakePoint(%s, %s), 4326)
|
THEN ST_SetSRID(ST_MakePoint(%s, %s), 4326)
|
||||||
ELSE NULL END,
|
ELSE NULL END,
|
||||||
%s, %s, %s, %s, NOW()
|
%s, %s, %s, %s, 'poll', NOW()
|
||||||
) ON CONFLICT (imei, alarm_type, alarm_time) DO NOTHING
|
) ON CONFLICT (imei, alarm_type, alarm_time) DO NOTHING
|
||||||
""", (
|
""", (
|
||||||
a.get("imei"), clean(a.get("alarmType")), clean_ts(a.get("alarmTime")),
|
a.get("imei"), alarm_type, alarm_name, alarm_time,
|
||||||
lng, lat, lng, lat, lat, lng,
|
lng, lat, lng, lat, lat, lng,
|
||||||
clean_num(a.get("speed")), clean(a.get("accStatus"))
|
clean_num(a.get("speed")), clean(a.get("accStatus"))
|
||||||
))
|
))
|
||||||
|
|
|
||||||
|
|
@ -9,10 +9,18 @@ REVISIONS (QA-Verified):
|
||||||
[FIX-M05] Batching: Groups 50 IMEIs per API call (API Limit Compliance).
|
[FIX-M05] Batching: Groups 50 IMEIs per API call (API Limit Compliance).
|
||||||
[FIX-M07] Signal Handling: Clean DB pool closure on SIGTERM/SIGINT.
|
[FIX-M07] Signal Handling: Clean DB pool closure on SIGTERM/SIGINT.
|
||||||
[FIX-M08] Atomic Logging: log_ingestion happens within the data transaction.
|
[FIX-M08] Atomic Logging: log_ingestion happens within the data transaction.
|
||||||
[FIX-QA-01] Distance: Explicit km to meters conversion (* 1000).
|
|
||||||
[FIX-11] Uses shared safe_task/setup_shutdown from ts_shared_rev (DRY).
|
[FIX-11] Uses shared safe_task/setup_shutdown from ts_shared_rev (DRY).
|
||||||
[FIX-M09] Trips: Captures runTimeSecond and maxSpeed from API.
|
[FIX-M09] Trips: Captures runTimeSecond and maxSpeed from API.
|
||||||
[FIX-M10] Parking: New poll_parking via jimi.open.platform.report.parking.
|
[FIX-M10] Parking: New poll_parking via jimi.open.platform.report.parking.
|
||||||
|
[FIX-M11] BUG-02: distance_m was stored in millimetres due to erroneous
|
||||||
|
* 1000 on an already-metric API value. Removed multiplication;
|
||||||
|
column renamed to distance_km in migration 04. Both poll_trips
|
||||||
|
and push_trip_report (webhook) corrected.
|
||||||
|
[FIX-M12] BUG-02: Renamed distance_m → distance_km in all SQL to match
|
||||||
|
migration 04 schema change.
|
||||||
|
[FIX-M13] POLL-02: Parking poll was returning 0 rows — added missing
|
||||||
|
acc_type=0 and account params; fixed response field durSecond
|
||||||
|
(was mapped as 'seconds').
|
||||||
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -174,21 +182,22 @@ def poll_trips():
|
||||||
with get_conn() as conn:
|
with get_conn() as conn:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
for t in trips:
|
for t in trips:
|
||||||
|
# [FIX-M11] API returns distance in km. Store directly as distance_km.
|
||||||
|
# Previous code multiplied by 1000 (→ mm), which was wrong.
|
||||||
dist_km = clean_num(t.get("distance"))
|
dist_km = clean_num(t.get("distance"))
|
||||||
dist_m = dist_km * 1000 if dist_km is not None else 0 # [QA-01] Conversion
|
|
||||||
cur.execute("""
|
cur.execute("""
|
||||||
INSERT INTO tracksolid.trips (
|
INSERT INTO tracksolid.trips (
|
||||||
imei, start_time, end_time, distance_m,
|
imei, start_time, end_time, distance_km,
|
||||||
avg_speed_kmh, max_speed_kmh, driving_time_s, source
|
avg_speed_kmh, max_speed_kmh, driving_time_s, source
|
||||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'poll')
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, 'poll')
|
||||||
ON CONFLICT (imei, start_time) DO UPDATE SET
|
ON CONFLICT (imei, start_time) DO UPDATE SET
|
||||||
end_time = EXCLUDED.end_time,
|
end_time = EXCLUDED.end_time,
|
||||||
distance_m = EXCLUDED.distance_m,
|
distance_km = EXCLUDED.distance_km,
|
||||||
max_speed_kmh = COALESCE(EXCLUDED.max_speed_kmh, tracksolid.trips.max_speed_kmh),
|
max_speed_kmh = COALESCE(EXCLUDED.max_speed_kmh, tracksolid.trips.max_speed_kmh),
|
||||||
driving_time_s = COALESCE(EXCLUDED.driving_time_s, tracksolid.trips.driving_time_s)
|
driving_time_s = COALESCE(EXCLUDED.driving_time_s, tracksolid.trips.driving_time_s)
|
||||||
""", (
|
""", (
|
||||||
t.get("imei"), clean_ts(t.get("startTime")), clean_ts(t.get("endTime")),
|
t.get("imei"), clean_ts(t.get("startTime")), clean_ts(t.get("endTime")),
|
||||||
dist_m, clean_num(t.get("avgSpeed")),
|
dist_km, clean_num(t.get("avgSpeed")),
|
||||||
clean_num(t.get("maxSpeed")), clean_int(t.get("runTimeSecond"))
|
clean_num(t.get("maxSpeed")), clean_int(t.get("runTimeSecond"))
|
||||||
))
|
))
|
||||||
inserted += 1
|
inserted += 1
|
||||||
|
|
@ -208,10 +217,14 @@ def poll_parking():
|
||||||
|
|
||||||
for i in range(0, len(imeis), 50):
|
for i in range(0, len(imeis), 50):
|
||||||
batch = imeis[i:i+50]
|
batch = imeis[i:i+50]
|
||||||
|
# [FIX-M13] Added account + acc_type=0 (all stop types). Without these
|
||||||
|
# the API returns empty results even when parking events exist.
|
||||||
resp = api_post("jimi.open.platform.report.parking", {
|
resp = api_post("jimi.open.platform.report.parking", {
|
||||||
|
"account": TARGET_ACCOUNT,
|
||||||
"imeis": ",".join(batch),
|
"imeis": ",".join(batch),
|
||||||
"begin_time": start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
"begin_time": start_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
"end_time": end_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
"end_time": end_ts.strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
"acc_type": 0,
|
||||||
}, token)
|
}, token)
|
||||||
|
|
||||||
events = resp.get("result") or []
|
events = resp.get("result") or []
|
||||||
|
|
@ -236,7 +249,7 @@ def poll_parking():
|
||||||
) ON CONFLICT (imei, start_time, event_type) DO NOTHING
|
) ON CONFLICT (imei, start_time, event_type) DO NOTHING
|
||||||
""", (
|
""", (
|
||||||
imei, start_time, clean_ts(p.get("endTime")),
|
imei, start_time, clean_ts(p.get("endTime")),
|
||||||
clean_int(p.get("seconds")),
|
clean_int(p.get("durSecond")), # [FIX-M13] API returns durSecond, not seconds
|
||||||
lng, lat, lng, lat,
|
lng, lat, lng, lat,
|
||||||
clean(p.get("address"))
|
clean(p.get("address"))
|
||||||
))
|
))
|
||||||
|
|
|
||||||
793
tracksolid_DB_manual.md
Normal file
793
tracksolid_DB_manual.md
Normal file
|
|
@ -0,0 +1,793 @@
|
||||||
|
# Tracksolid Database Manual
|
||||||
|
|
||||||
|
**Database:** `tracksolid_db`
|
||||||
|
**Host:** `kianiadee@stage.rahamafresh.com`
|
||||||
|
**Container:** `timescale_db-bo3nov2ija7g8wn9b1g2paxs-210508774107`
|
||||||
|
**Engine:** TimescaleDB (PostgreSQL 16 + TimescaleDB 2.15)
|
||||||
|
**Timezone note:** All timestamps are stored in UTC. Always cast to `AT TIME ZONE 'Africa/Nairobi'` (EAT = UTC+3) when displaying to users or building reports.
|
||||||
|
|
||||||
|
To connect from the host server:
|
||||||
|
```bash
|
||||||
|
docker exec timescale_db-bo3nov2ija7g8wn9b1g2paxs-210508774107 psql -U postgres -d tracksolid_db
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Table of Contents
|
||||||
|
|
||||||
|
1. [Schema Overview](#1-schema-overview)
|
||||||
|
2. [tracksolid.devices](#2-tracksoliddevices)
|
||||||
|
3. [tracksolid.position_history](#3-tracksolidposition_history)
|
||||||
|
4. [tracksolid.trips](#4-tracksolidtrips)
|
||||||
|
5. [tracksolid.live_positions](#5-tracksolidlive_positions)
|
||||||
|
6. [tracksolid.alarms](#6-tracksolidalarms)
|
||||||
|
7. [tracksolid.ingestion_log](#7-tracksolidingestion_log)
|
||||||
|
8. [tracksolid.heartbeats](#8-tracksolidheartbeats)
|
||||||
|
9. [tracksolid.obd_readings](#9-tracksolidobd_readings)
|
||||||
|
10. [tracksolid.parking_events](#10-tracksolidparking_events)
|
||||||
|
11. [tracksolid.fault_codes](#11-trackshopfault_codes)
|
||||||
|
12. [dwh_gold.dim_vehicles](#12-dwh_golddim_vehicles)
|
||||||
|
13. [dwh_gold.fact_daily_fleet_metrics](#13-dwh_goldfact_daily_fleet_metrics)
|
||||||
|
14. [Business Intelligence Queries](#14-business-intelligence-queries)
|
||||||
|
15. [Today's Metrics — From 00:00 Nairobi Time to Now](#15-todays-metrics--from-0000-nairobi-time-to-now)
|
||||||
|
16. [Known Data Issues](#16-known-data-issues)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Schema Overview
|
||||||
|
|
||||||
|
The database is organised into three schemas:
|
||||||
|
|
||||||
|
| Schema | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `tracksolid` | Raw operational data ingested from the Tracksolid/Jimi Open Platform API |
|
||||||
|
| `dwh_gold` | Pre-aggregated data warehouse layer for reporting and dashboards |
|
||||||
|
| `public` | PostGIS spatial reference tables (system-managed) |
|
||||||
|
|
||||||
|
Data is pulled from the Jimi/Tracksolid API on a continuous polling schedule. The `ingestion_log` table records every API call so you can audit pipeline health.
|
||||||
|
|
||||||
|
### List all tables with sizes
|
||||||
|
|
||||||
|
This query gives a quick inventory of every table in the two business schemas along with its on-disk size. Useful as a first step when connecting to the database to understand what exists and which tables hold the most data.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
schemaname,
|
||||||
|
tablename,
|
||||||
|
pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) AS size
|
||||||
|
FROM pg_tables
|
||||||
|
WHERE schemaname IN ('tracksolid', 'dwh_gold')
|
||||||
|
ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Count rows in all tables
|
||||||
|
|
||||||
|
A fast sanity check to see which tables are populated and which are still empty. Run this after deployments or data migrations to confirm data is flowing as expected.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT 'tracksolid.trips' AS tbl, COUNT(*) FROM tracksolid.trips
|
||||||
|
UNION ALL SELECT 'tracksolid.position_history', COUNT(*) FROM tracksolid.position_history
|
||||||
|
UNION ALL SELECT 'tracksolid.alarms', COUNT(*) FROM tracksolid.alarms
|
||||||
|
UNION ALL SELECT 'tracksolid.devices', COUNT(*) FROM tracksolid.devices
|
||||||
|
UNION ALL SELECT 'tracksolid.heartbeats', COUNT(*) FROM tracksolid.heartbeats
|
||||||
|
UNION ALL SELECT 'tracksolid.obd_readings', COUNT(*) FROM tracksolid.obd_readings
|
||||||
|
UNION ALL SELECT 'tracksolid.parking_events', COUNT(*) FROM tracksolid.parking_events
|
||||||
|
UNION ALL SELECT 'tracksolid.live_positions', COUNT(*) FROM tracksolid.live_positions
|
||||||
|
UNION ALL SELECT 'tracksolid.fault_codes', COUNT(*) FROM tracksolid.fault_codes
|
||||||
|
UNION ALL SELECT 'tracksolid.ingestion_log', COUNT(*) FROM tracksolid.ingestion_log
|
||||||
|
UNION ALL SELECT 'dwh_gold.dim_vehicles', COUNT(*) FROM dwh_gold.dim_vehicles
|
||||||
|
UNION ALL SELECT 'dwh_gold.fact_daily_fleet_metrics',COUNT(*) FROM dwh_gold.fact_daily_fleet_metrics;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. tracksolid.devices
|
||||||
|
|
||||||
|
This table is the master registry of all GPS tracking devices in the fleet. Each row represents one physical GPS tracker unit installed in a vehicle. Key fields include the device's IMEI (its unique hardware identifier), vehicle registration details (name, plate number, brand, model), driver assignment, SIM card details, and subscription lifecycle timestamps. The `current_mileage_km` field reflects the latest odometer reading received from the device. At time of audit, 63 devices were registered, all in a single "Default group", but `vehicle_name`, `vehicle_number`, and `driver_name` were not yet populated — these need to be filled in for reports to be human-readable.
|
||||||
|
|
||||||
|
The `fuel_100km` column is particularly important: it is the reference value used to estimate `fuel_consumed_l` in the trips table. If it is null, fuel calculations will not work.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.devices
|
||||||
|
```
|
||||||
|
|
||||||
|
### List all active devices with odometer readings
|
||||||
|
|
||||||
|
Returns all enabled devices ordered by highest mileage. Use this to identify high-utilisation vehicles that may need servicing, and to verify that vehicle metadata (name, plate, driver) has been filled in. Vehicles with very high odometer readings but blank names are a data quality flag.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
imei,
|
||||||
|
vehicle_name,
|
||||||
|
vehicle_number,
|
||||||
|
driver_name,
|
||||||
|
mc_type,
|
||||||
|
device_group,
|
||||||
|
status,
|
||||||
|
current_mileage_km,
|
||||||
|
last_synced_at AT TIME ZONE 'Africa/Nairobi' AS last_synced_nairobi
|
||||||
|
FROM tracksolid.devices
|
||||||
|
WHERE enabled_flag = 1
|
||||||
|
ORDER BY current_mileage_km DESC NULLS LAST;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. tracksolid.position_history
|
||||||
|
|
||||||
|
This is the core time-series table and is implemented as a **TimescaleDB hypertable**, automatically partitioned by time into chunks for efficient storage and querying of large volumes of GPS data. Every time the ingestion service polls the Tracksolid API (approximately every 1 minute), it writes a GPS breadcrumb for each active device into this table. Each row captures the device's location (latitude, longitude, and PostGIS geometry), speed, heading, ignition status, satellite count, and running odometer.
|
||||||
|
|
||||||
|
The `acc_status` field indicates whether the vehicle's ignition/accessory circuit is on (`1`) or off (`0`) at the time of the ping. The table uses a composite primary key of `(imei, gps_time)`, ensuring no duplicate pings are stored. Older chunks are transparently compressed by TimescaleDB to save disk space.
|
||||||
|
|
||||||
|
**Important:** `altitude` is present in the schema but not currently populated by the ingestion pipeline.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.position_history
|
||||||
|
```
|
||||||
|
|
||||||
|
### Most recent GPS pings per vehicle (Nairobi time)
|
||||||
|
|
||||||
|
Returns the latest position record for each device. Use this to quickly check which vehicles are currently active, where they last reported from, and whether the GPS lock is good (satellite count ≥ 8 is acceptable; ≥ 12 is excellent).
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
ph.imei,
|
||||||
|
ph.gps_time AT TIME ZONE 'Africa/Nairobi' AS gps_nairobi,
|
||||||
|
ph.lat,
|
||||||
|
ph.lng,
|
||||||
|
ph.speed,
|
||||||
|
ph.direction,
|
||||||
|
ph.acc_status,
|
||||||
|
ph.current_mileage,
|
||||||
|
ph.altitude,
|
||||||
|
ph.satellite
|
||||||
|
FROM tracksolid.position_history ph
|
||||||
|
ORDER BY ph.gps_time DESC
|
||||||
|
LIMIT 15;
|
||||||
|
```
|
||||||
|
|
||||||
|
### All GPS pings up to the current moment
|
||||||
|
|
||||||
|
Returns position history records with a `gps_time` strictly before `NOW()`. `NOW()` returns the current timestamp in UTC, which matches the stored timezone of the column directly — no conversion needed in the `WHERE` clause. This filter is useful when building live queries or scheduled reports that must not accidentally include future-dated records from a faulty device clock. Combine with a lower-bound interval to avoid scanning the entire table; the example below limits results to the last 24 hours.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
ph.imei,
|
||||||
|
ph.gps_time AT TIME ZONE 'Africa/Nairobi' AS gps_nairobi,
|
||||||
|
ph.lat,
|
||||||
|
ph.lng,
|
||||||
|
ph.speed,
|
||||||
|
ph.direction,
|
||||||
|
ph.acc_status,
|
||||||
|
ph.current_mileage,
|
||||||
|
ph.altitude,
|
||||||
|
ph.satellite
|
||||||
|
FROM tracksolid.position_history ph
|
||||||
|
WHERE ph.gps_time > NOW() - INTERVAL '24 hours'
|
||||||
|
AND ph.gps_time < NOW()
|
||||||
|
ORDER BY ph.gps_time DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
To widen or narrow the window, replace the interval:
|
||||||
|
|
||||||
|
| Interval | Example |
|
||||||
|
|---|---|
|
||||||
|
| Last hour | `NOW() - INTERVAL '1 hour'` |
|
||||||
|
| Last 24 hours | `NOW() - INTERVAL '24 hours'` |
|
||||||
|
| Last 7 days | `NOW() - INTERVAL '7 days'` |
|
||||||
|
| Since midnight Nairobi time | See today's metrics section below |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Full position trail for a specific vehicle on a given day
|
||||||
|
|
||||||
|
Replaces `'YOUR_IMEI_HERE'` and the date with actual values. Useful for replaying a vehicle's route through the day, debugging missing trip segments, or feeding data into a mapping tool.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
gps_time AT TIME ZONE 'Africa/Nairobi' AS gps_nairobi,
|
||||||
|
lat,
|
||||||
|
lng,
|
||||||
|
speed,
|
||||||
|
direction,
|
||||||
|
acc_status,
|
||||||
|
current_mileage,
|
||||||
|
satellite
|
||||||
|
FROM tracksolid.position_history
|
||||||
|
WHERE imei = 'YOUR_IMEI_HERE'
|
||||||
|
AND gps_time >= '2026-04-10 00:00:00+03'
|
||||||
|
AND gps_time < '2026-04-11 00:00:00+03'
|
||||||
|
ORDER BY gps_time ASC;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. tracksolid.trips
|
||||||
|
|
||||||
|
The trips table stores auto-detected journey segments. A trip begins when the vehicle's ignition turns on and ends when it turns off again (or after a prolonged stationary period). Each row summarises one journey: start and end times, start and end coordinates, total distance, average and maximum speed, driving time, idle time, and estimated fuel consumption.
|
||||||
|
|
||||||
|
**Note on `distance_km`:** Stores trip distance directly in kilometres. Prior to migration 04 this column was named `distance_m` and incorrectly held millimetres due to an erroneous `× 1000` in the ingestion code. Migration 04 corrected all historical rows (`÷ 1,000,000`) and renamed the column. Use the raw `distance_km` value in queries — no further division is needed.
|
||||||
|
|
||||||
|
At the time of audit, `max_speed_kmh` was null on all trips (not yet computed by the trip detection logic) and `fuel_consumed_l` was null because `fuel_100km` is not set on the devices.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.trips
|
||||||
|
```
|
||||||
|
|
||||||
|
### Recent trips with full detail (Nairobi time)
|
||||||
|
|
||||||
|
Lists the 20 most recent trip records joined with vehicle metadata. Replace `LIMIT 20` with a larger number or remove it to see more history. This is the primary query for reviewing daily driving activity.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
t.imei,
|
||||||
|
t.start_time AT TIME ZONE 'Africa/Nairobi' AS start_nairobi,
|
||||||
|
t.end_time AT TIME ZONE 'Africa/Nairobi' AS end_nairobi,
|
||||||
|
ROUND(t.distance_km, 3) AS distance_km,
|
||||||
|
t.avg_speed_kmh,
|
||||||
|
t.max_speed_kmh,
|
||||||
|
ROUND(t.driving_time_s / 60.0, 1) AS drive_min,
|
||||||
|
ROUND(t.idle_time_s / 60.0, 1) AS idle_min,
|
||||||
|
t.fuel_consumed_l
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
JOIN tracksolid.devices d ON d.imei = t.imei
|
||||||
|
ORDER BY t.start_time DESC
|
||||||
|
LIMIT 20;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Verify distance units (cross-check via speed × time)
|
||||||
|
|
||||||
|
This diagnostic query confirms that `distance_km` is stored in millimetres by comparing the raw value (divided by 1,000,000 to get km) against the expected distance calculated from average speed and driving time. The two `_km` columns should match closely. Run this whenever the distance figures seem implausible.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
t.imei,
|
||||||
|
t.start_time AT TIME ZONE 'Africa/Nairobi' AS start_nbi,
|
||||||
|
t.end_time AT TIME ZONE 'Africa/Nairobi' AS end_nbi,
|
||||||
|
t.distance_km AS raw_distance_kmm,
|
||||||
|
ROUND(t.distance_km, 3) AS distance_km,
|
||||||
|
t.avg_speed_kmh,
|
||||||
|
t.driving_time_s,
|
||||||
|
ROUND((t.avg_speed_kmh * t.driving_time_s / 3600.0), 3) AS expected_km_from_speed
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
WHERE t.avg_speed_kmh IS NOT NULL
|
||||||
|
AND t.driving_time_s IS NOT NULL
|
||||||
|
AND t.distance_km > 0
|
||||||
|
ORDER BY t.start_time DESC
|
||||||
|
LIMIT 10;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Speed band distribution across all trips
|
||||||
|
|
||||||
|
Categorises trips by average speed to understand whether the fleet primarily operates in congested urban conditions, normal urban flow, or open highway. The result helps benchmark expected journey times and assess whether vehicles are being used efficiently.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
CASE
|
||||||
|
WHEN avg_speed_kmh < 20 THEN '0–20 km/h (slow / heavy traffic)'
|
||||||
|
WHEN avg_speed_kmh < 40 THEN '20–40 km/h (normal urban)'
|
||||||
|
WHEN avg_speed_kmh < 60 THEN '40–60 km/h (highway)'
|
||||||
|
WHEN avg_speed_kmh < 80 THEN '60–80 km/h (fast highway)'
|
||||||
|
ELSE '80+ km/h (very fast)'
|
||||||
|
END AS speed_band,
|
||||||
|
COUNT(*) AS trip_count
|
||||||
|
FROM tracksolid.trips
|
||||||
|
WHERE avg_speed_kmh IS NOT NULL
|
||||||
|
GROUP BY 1
|
||||||
|
ORDER BY trip_count DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. tracksolid.live_positions
|
||||||
|
|
||||||
|
This table holds one row per device and is continuously upserted with the latest known position every time the ingestion service polls the API. It is the equivalent of a "last known state" snapshot and is designed for real-time dashboard displays — you never need to scan the full `position_history` table just to find out where a vehicle currently is.
|
||||||
|
|
||||||
|
Beyond basic GPS coordinates, `live_positions` captures richer status fields than `position_history`: battery level (`elec_quantity`), external power voltage (`power_value`), device operational status (`device_status`), expiry and activation flags, and a human-readable address description (`loc_desc`). The `tracker_oil` field reflects whether the relay/immobiliser output is active.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.live_positions
|
||||||
|
```
|
||||||
|
|
||||||
|
### Current status of all vehicles (Nairobi time)
|
||||||
|
|
||||||
|
Returns the latest known position and status for every device. Use this as the live fleet map feed. Vehicles with `acc_status = '1'` are currently running; `'0'` means engine off. Check `expire_flag` to catch devices whose subscriptions are about to lapse.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
lp.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
lp.lat,
|
||||||
|
lp.lng,
|
||||||
|
lp.speed,
|
||||||
|
lp.acc_status,
|
||||||
|
lp.device_status,
|
||||||
|
lp.gps_time AT TIME ZONE 'Africa/Nairobi' AS gps_nairobi,
|
||||||
|
lp.updated_at AT TIME ZONE 'Africa/Nairobi' AS last_updated_nairobi,
|
||||||
|
lp.elec_quantity,
|
||||||
|
lp.power_value,
|
||||||
|
lp.expire_flag,
|
||||||
|
lp.loc_desc
|
||||||
|
FROM tracksolid.live_positions lp
|
||||||
|
LEFT JOIN tracksolid.devices d ON d.imei = lp.imei
|
||||||
|
ORDER BY lp.updated_at DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. tracksolid.alarms
|
||||||
|
|
||||||
|
The alarms table records every alarm event reported by the Tracksolid API for any device in the fleet. Alarm types can include overspeed, harsh braking, geofence violations, power disconnection, low battery, tampering, and more — the exact set depends on the tracker model and account configuration. Each alarm record captures the device IMEI, alarm type and name, the timestamp and GPS coordinates at which the alarm fired, the vehicle's speed at that moment, and ignition status.
|
||||||
|
|
||||||
|
At the time of audit, all 1,054 records had `alarm_type` and `alarm_name` set to null, meaning the alarm classification is not yet being parsed from the API response. This is a data pipeline gap that needs to be fixed — the raw alarm events are being stored but are not yet actionable without the type label.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.alarms
|
||||||
|
```
|
||||||
|
|
||||||
|
### Recent alarms with Nairobi timestamp
|
||||||
|
|
||||||
|
Returns the most recent alarm events. Once `alarm_type` and `alarm_name` are being populated correctly, this query becomes the primary tool for investigating safety incidents, policy violations, and device health events.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
a.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.driver_name,
|
||||||
|
a.alarm_time AT TIME ZONE 'Africa/Nairobi' AS alarm_nairobi,
|
||||||
|
a.alarm_type,
|
||||||
|
a.alarm_name,
|
||||||
|
a.speed,
|
||||||
|
a.acc_status,
|
||||||
|
a.lat,
|
||||||
|
a.lng,
|
||||||
|
a.source
|
||||||
|
FROM tracksolid.alarms a
|
||||||
|
LEFT JOIN tracksolid.devices d ON d.imei = a.imei
|
||||||
|
ORDER BY a.alarm_time DESC
|
||||||
|
LIMIT 50;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Alarm frequency by type
|
||||||
|
|
||||||
|
Once alarm types are populated, this aggregation shows which alarm categories are most common across the fleet. High overspeed counts flag driver behaviour issues; high power-disconnect counts may indicate tampering or device faults.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
alarm_type,
|
||||||
|
alarm_name,
|
||||||
|
COUNT(*) AS total_alarms,
|
||||||
|
COUNT(DISTINCT imei) AS vehicles_affected,
|
||||||
|
MIN(alarm_time AT TIME ZONE 'Africa/Nairobi') AS first_seen,
|
||||||
|
MAX(alarm_time AT TIME ZONE 'Africa/Nairobi') AS last_seen
|
||||||
|
FROM tracksolid.alarms
|
||||||
|
GROUP BY alarm_type, alarm_name
|
||||||
|
ORDER BY total_alarms DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. tracksolid.ingestion_log
|
||||||
|
|
||||||
|
The ingestion log is the pipeline health ledger. Every time the ingestion service calls a Tracksolid API endpoint, it writes one row here recording: which endpoint was called, how many device IMEIs were processed, how many rows were inserted or upserted into the database, how long the call took (in milliseconds), and whether it succeeded. If a call fails, `success` is set to false and the error details are captured in `error_code` and `error_message`.
|
||||||
|
|
||||||
|
This table is the first place to check when troubleshooting missing data or unexpected gaps in position history. It tells you definitively whether the pipeline ran and what it ingested, as opposed to whether the GPS device itself was transmitting.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.ingestion_log
|
||||||
|
```
|
||||||
|
|
||||||
|
### Pipeline health summary by endpoint
|
||||||
|
|
||||||
|
Aggregates the ingestion log by endpoint to show total run counts, data volumes, average call duration, and failure rate. A non-zero `failure_count` needs investigation. A sudden drop in `avg_rows_per_run` compared to historical baseline may indicate the API rate limit was hit or devices went offline.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
endpoint,
|
||||||
|
COUNT(*) AS runs,
|
||||||
|
SUM(rows_inserted) AS total_inserted,
|
||||||
|
SUM(rows_upserted) AS total_upserted,
|
||||||
|
ROUND(AVG(rows_inserted)) AS avg_rows_per_run,
|
||||||
|
ROUND(AVG(duration_ms)) AS avg_duration_ms,
|
||||||
|
MAX(duration_ms) AS max_duration_ms,
|
||||||
|
SUM(CASE WHEN NOT success THEN 1 ELSE 0 END) AS failure_count,
|
||||||
|
MIN(run_at AT TIME ZONE 'Africa/Nairobi') AS first_run,
|
||||||
|
MAX(run_at AT TIME ZONE 'Africa/Nairobi') AS last_run
|
||||||
|
FROM tracksolid.ingestion_log
|
||||||
|
GROUP BY endpoint
|
||||||
|
ORDER BY runs DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Recent pipeline failures
|
||||||
|
|
||||||
|
Filters the log to show only failed ingestion runs. Run this immediately when investigating data gaps or alert emails from the pipeline. The `error_message` column usually contains the HTTP status code and API error body.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
run_at AT TIME ZONE 'Africa/Nairobi' AS run_nairobi,
|
||||||
|
endpoint,
|
||||||
|
imei_count,
|
||||||
|
rows_inserted,
|
||||||
|
duration_ms,
|
||||||
|
error_code,
|
||||||
|
error_message
|
||||||
|
FROM tracksolid.ingestion_log
|
||||||
|
WHERE success = false
|
||||||
|
ORDER BY run_at DESC
|
||||||
|
LIMIT 50;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. tracksolid.heartbeats
|
||||||
|
|
||||||
|
The heartbeats table is designed to record periodic keep-alive signals sent by GPS devices when they are stationary for extended periods. Rather than inserting a full position record every minute even when nothing has changed, some tracker firmware sends a lightweight heartbeat ping to confirm the device is still powered on and connected. This can be used to distinguish "vehicle parked with tracker alive" from "tracker lost power or signal".
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**. Either the tracker models currently deployed do not send heartbeat packets, or the heartbeat ingestion endpoint has not yet been implemented.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.heartbeats
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. tracksolid.obd_readings
|
||||||
|
|
||||||
|
The OBD (On-Board Diagnostics) readings table is intended to store data pulled from a vehicle's OBD-II port via a compatible tracker. OBD data can include engine RPM, coolant temperature, throttle position, fuel level, battery voltage, and diagnostic trouble codes. This information is valuable for predictive maintenance and driver behaviour scoring beyond what GPS alone provides.
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**. The JC400P and X3 tracker models in the fleet may support OBD connectivity, but either the OBD cables are not installed or the OBD data ingestion pipeline has not been built yet.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.obd_readings
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. tracksolid.parking_events
|
||||||
|
|
||||||
|
This table is designed to store discrete parking events — records of when a vehicle stopped and for how long. Rather than deriving parking from position history (which requires scanning many rows), the Tracksolid API's `jimi.open.platform.report.parking` endpoint provides pre-computed parking events. These are useful for calculating driver wait times, identifying vehicles left idle in locations for long periods, and auditing whether vehicles are parked at authorised locations overnight.
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**, despite the ingestion service successfully calling the parking endpoint 358 times. The API is returning empty results, suggesting parking detection thresholds may need to be adjusted in the Tracksolid account settings, or the vehicles have not yet triggered the platform's parking detection criteria.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.parking_events
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. tracksolid.fault_codes
|
||||||
|
|
||||||
|
The fault codes table stores vehicle diagnostic trouble codes (DTCs) received from OBD-connected trackers. When a vehicle's engine management system logs a fault (e.g. P0300 for a misfire, P0171 for a lean fuel mixture), the tracker reads it via the OBD port and transmits it to the Tracksolid platform, from where it is ingested into this table. This enables remote fleet health monitoring without requiring drivers to visit a workshop.
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**, which is consistent with the OBD readings table also being empty. Fault code ingestion depends on OBD connectivity being established first.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d tracksolid.fault_codes
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. dwh_gold.dim_vehicles
|
||||||
|
|
||||||
|
This is the vehicles dimension table in the data warehouse gold layer. It is intended to be a clean, enriched, business-friendly view of the fleet — combining device metadata from `tracksolid.devices` with any additional attributes needed for reporting (cost centre, vehicle category, assigned route, etc.). Dimension tables in a star schema are typically populated by an ETL job that joins and transforms raw operational tables.
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**. The ETL pipeline that populates the gold layer has not yet been run.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d dwh_gold.dim_vehicles
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. dwh_gold.fact_daily_fleet_metrics
|
||||||
|
|
||||||
|
This is the central fact table of the data warehouse. It is designed to hold one pre-aggregated row per vehicle per day, summarising distance driven, fuel consumed, driving time, idle time, trip count, first departure time, last return time, and alarm counts. Pre-aggregating at this level makes dashboards and management reports extremely fast — a full month's fleet summary requires scanning at most 63 vehicles × 31 days = ~2,000 rows rather than hundreds of thousands of raw trip and position records.
|
||||||
|
|
||||||
|
At the time of audit this table contained **0 rows**. Once the ETL job is running, this should be the primary data source for all Grafana dashboards and business reports.
|
||||||
|
|
||||||
|
### Describe table structure
|
||||||
|
|
||||||
|
```sql
|
||||||
|
\d dwh_gold.fact_daily_fleet_metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 14. Business Intelligence Queries
|
||||||
|
|
||||||
|
### Daily work start/end times and driving summary per vehicle (Nairobi time)
|
||||||
|
|
||||||
|
This is the primary operational report query. For each vehicle on each working day, it computes: the time the first trip of the day began (proxy for "driver started work"), the time the last trip ended (proxy for "driver finished work"), total distance driven, number of trips made, total driving time, and total idle time. Join with `tracksolid.devices` to add vehicle names and driver names once those fields are populated.
|
||||||
|
|
||||||
|
The query groups by IMEI and calendar date in Nairobi time, so a trip that starts at 23:58 EAT correctly belongs to that day rather than rolling over to the next UTC day.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
WITH daily AS (
|
||||||
|
SELECT
|
||||||
|
t.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
DATE(t.start_time AT TIME ZONE 'Africa/Nairobi') AS work_date,
|
||||||
|
MIN(t.start_time AT TIME ZONE 'Africa/Nairobi') AS first_trip_start,
|
||||||
|
MAX(t.end_time AT TIME ZONE 'Africa/Nairobi') AS last_trip_end,
|
||||||
|
ROUND(SUM(t.distance_km) / 1000000.0, 2) AS total_km,
|
||||||
|
COUNT(*) AS trip_count,
|
||||||
|
ROUND(SUM(t.driving_time_s) / 60.0, 1) AS total_drive_min,
|
||||||
|
ROUND(SUM(t.idle_time_s) / 60.0, 1) AS total_idle_min,
|
||||||
|
ROUND(AVG(t.avg_speed_kmh)::numeric, 2) AS avg_speed_kmh,
|
||||||
|
MAX(t.max_speed_kmh) AS peak_speed_kmh
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
JOIN tracksolid.devices d ON d.imei = t.imei
|
||||||
|
WHERE t.end_time IS NOT NULL
|
||||||
|
GROUP BY
|
||||||
|
t.imei, d.vehicle_name, d.vehicle_number, d.driver_name,
|
||||||
|
DATE(t.start_time AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
work_date,
|
||||||
|
imei,
|
||||||
|
vehicle_name,
|
||||||
|
vehicle_number,
|
||||||
|
driver_name,
|
||||||
|
TO_CHAR(first_trip_start, 'HH24:MI') AS day_start,
|
||||||
|
TO_CHAR(last_trip_end, 'HH24:MI') AS day_end,
|
||||||
|
EXTRACT(EPOCH FROM (last_trip_end - first_trip_start)) / 3600.0 AS operational_hours,
|
||||||
|
total_km,
|
||||||
|
trip_count,
|
||||||
|
total_drive_min,
|
||||||
|
total_idle_min,
|
||||||
|
avg_speed_kmh,
|
||||||
|
peak_speed_kmh
|
||||||
|
FROM daily
|
||||||
|
ORDER BY work_date DESC, imei;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fleet summary for a date range
|
||||||
|
|
||||||
|
Aggregates across all vehicles and all days within a date range to give a high-level fleet utilisation report. Useful for weekly or monthly management summaries. Adjust the date range in the `WHERE` clause as needed.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
DATE(t.start_time AT TIME ZONE 'Africa/Nairobi') AS work_date,
|
||||||
|
COUNT(DISTINCT t.imei) AS active_vehicles,
|
||||||
|
COUNT(*) AS total_trips,
|
||||||
|
ROUND(SUM(t.distance_km) / 1000000.0, 2) AS total_fleet_km,
|
||||||
|
ROUND(AVG(t.avg_speed_kmh)::numeric, 2) AS fleet_avg_speed_kmh,
|
||||||
|
ROUND(SUM(t.driving_time_s) / 3600.0, 2) AS total_drive_hours,
|
||||||
|
ROUND(SUM(t.idle_time_s) / 3600.0, 2) AS total_idle_hours
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
WHERE t.start_time >= '2026-04-09 00:00:00+03'
|
||||||
|
AND t.start_time < '2026-04-11 00:00:00+03'
|
||||||
|
AND t.end_time IS NOT NULL
|
||||||
|
GROUP BY DATE(t.start_time AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
ORDER BY work_date;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Idle time analysis — vehicles spending excessive time idling
|
||||||
|
|
||||||
|
Identifies trips where the vehicle spent more than 20% of its time stationary with the engine running. Excessive idling wastes fuel and is often a sign of drivers waiting in traffic, sitting at customer sites, or running the air conditioning while parked. The threshold (20%) can be adjusted.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
t.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.driver_name,
|
||||||
|
t.start_time AT TIME ZONE 'Africa/Nairobi' AS trip_start,
|
||||||
|
ROUND(t.driving_time_s / 60.0, 1) AS drive_min,
|
||||||
|
ROUND(t.idle_time_s / 60.0, 1) AS idle_min,
|
||||||
|
ROUND(
|
||||||
|
100.0 * t.idle_time_s /
|
||||||
|
NULLIF(t.driving_time_s + t.idle_time_s, 0)
|
||||||
|
, 1) AS idle_pct,
|
||||||
|
ROUND(t.distance_km, 3) AS distance_km
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
JOIN tracksolid.devices d ON d.imei = t.imei
|
||||||
|
WHERE t.idle_time_s IS NOT NULL
|
||||||
|
AND t.driving_time_s IS NOT NULL
|
||||||
|
AND (t.idle_time_s + t.driving_time_s) > 0
|
||||||
|
AND (100.0 * t.idle_time_s / (t.driving_time_s + t.idle_time_s)) > 20
|
||||||
|
ORDER BY idle_pct DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Vehicles with no activity today (Nairobi time)
|
||||||
|
|
||||||
|
Returns devices that are registered and active but have not started a trip today. Useful for daily fleet readiness checks — if a vehicle was expected to be operational but appears here, investigate whether the device is offline, the vehicle is in the workshop, or the driver has not started work.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
d.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
d.current_mileage_km,
|
||||||
|
d.last_synced_at AT TIME ZONE 'Africa/Nairobi' AS last_synced_nairobi
|
||||||
|
FROM tracksolid.devices d
|
||||||
|
WHERE d.enabled_flag = 1
|
||||||
|
AND d.imei NOT IN (
|
||||||
|
SELECT DISTINCT imei
|
||||||
|
FROM tracksolid.trips
|
||||||
|
WHERE DATE(start_time AT TIME ZONE 'Africa/Nairobi') = CURRENT_DATE AT TIME ZONE 'Africa/Nairobi'
|
||||||
|
)
|
||||||
|
ORDER BY d.vehicle_name NULLS LAST;
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 15. Today's Metrics — From 00:00 Nairobi Time to Now
|
||||||
|
|
||||||
|
All queries in this section use a dynamic time window that opens at midnight Nairobi time on the current calendar day and closes at `NOW()`. This means they are safe to run at any point during the day and will always reflect activity from the start of the working day up to the present moment. The anchor expression used throughout is:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi') AT TIME ZONE 'Africa/Nairobi'
|
||||||
|
```
|
||||||
|
|
||||||
|
This evaluates to today's `00:00:00 +03:00` in UTC, regardless of when the query is run. Pairing it with `< NOW()` ensures only records that have actually arrived are included.
|
||||||
|
|
||||||
|
### GPS pings since midnight (position_history)
|
||||||
|
|
||||||
|
Returns every position breadcrumb recorded today for all vehicles, from the first ping after midnight Nairobi time up to the current moment. Useful for replaying today's movement on a map or confirming that all devices are actively reporting.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
ph.imei,
|
||||||
|
ph.gps_time AT TIME ZONE 'Africa/Nairobi' AS gps_nairobi,
|
||||||
|
ph.lat,
|
||||||
|
ph.lng,
|
||||||
|
ph.speed,
|
||||||
|
ph.direction,
|
||||||
|
ph.acc_status,
|
||||||
|
ph.current_mileage,
|
||||||
|
ph.satellite
|
||||||
|
FROM tracksolid.position_history ph
|
||||||
|
WHERE ph.gps_time >= DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
AT TIME ZONE 'Africa/Nairobi'
|
||||||
|
AND ph.gps_time < NOW()
|
||||||
|
ORDER BY ph.gps_time DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Today's trips per vehicle (trips)
|
||||||
|
|
||||||
|
Lists every completed trip recorded today, joined with device metadata. Because the day is still in progress, some vehicles may have open trips (where `end_time IS NULL`) — the `WHERE` clause includes both completed and in-progress trips so nothing is missed. The `end_time IS NULL` check in the display makes it easy to spot which trips are still open.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
t.imei,
|
||||||
|
t.start_time AT TIME ZONE 'Africa/Nairobi' AS start_nairobi,
|
||||||
|
t.end_time AT TIME ZONE 'Africa/Nairobi' AS end_nairobi,
|
||||||
|
CASE WHEN t.end_time IS NULL THEN 'IN PROGRESS' ELSE 'COMPLETE' END AS status,
|
||||||
|
ROUND(t.distance_km, 3) AS distance_km,
|
||||||
|
t.avg_speed_kmh,
|
||||||
|
ROUND(t.driving_time_s / 60.0, 1) AS drive_min,
|
||||||
|
ROUND(t.idle_time_s / 60.0, 1) AS idle_min
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
JOIN tracksolid.devices d ON d.imei = t.imei
|
||||||
|
WHERE t.start_time >= DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
AT TIME ZONE 'Africa/Nairobi'
|
||||||
|
AND t.start_time < NOW()
|
||||||
|
ORDER BY t.start_time DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Today's driving summary per vehicle (aggregated)
|
||||||
|
|
||||||
|
Rolls up all of today's trips into one row per vehicle. Shows the time the driver first moved (day start), the most recent trip end time (last known activity), total kilometres driven so far, total trips made, and cumulative driving and idle minutes. The `MAX(t.end_time)` will reflect the end of the last completed trip — if the vehicle is currently mid-trip that trip's distance and time will not yet appear here until the trip closes.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
WITH today_start AS (
|
||||||
|
SELECT DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
AT TIME ZONE 'Africa/Nairobi' AS ts
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
d.vehicle_name,
|
||||||
|
d.vehicle_number,
|
||||||
|
d.driver_name,
|
||||||
|
t.imei,
|
||||||
|
TO_CHAR(MIN(t.start_time AT TIME ZONE 'Africa/Nairobi'), 'HH24:MI') AS day_start,
|
||||||
|
TO_CHAR(MAX(t.end_time AT TIME ZONE 'Africa/Nairobi'), 'HH24:MI') AS last_activity,
|
||||||
|
COUNT(*) AS trips_so_far,
|
||||||
|
ROUND(SUM(t.distance_km) / 1000000.0, 2) AS total_km,
|
||||||
|
ROUND(SUM(t.driving_time_s) / 60.0, 1) AS total_drive_min,
|
||||||
|
ROUND(SUM(t.idle_time_s) / 60.0, 1) AS total_idle_min,
|
||||||
|
ROUND(AVG(t.avg_speed_kmh)::numeric, 2) AS avg_speed_kmh
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
JOIN tracksolid.devices d ON d.imei = t.imei
|
||||||
|
CROSS JOIN today_start ts
|
||||||
|
WHERE t.start_time >= ts.ts
|
||||||
|
AND t.start_time < NOW()
|
||||||
|
GROUP BY t.imei, d.vehicle_name, d.vehicle_number, d.driver_name
|
||||||
|
ORDER BY total_km DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Today's alarms so far
|
||||||
|
|
||||||
|
Returns all alarm events triggered from midnight Nairobi time to the present moment. Once `alarm_type` is being populated correctly this becomes the real-time safety dashboard — a high count of overspeed or harsh-braking alarms early in the day is an early warning signal.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
a.imei,
|
||||||
|
d.vehicle_name,
|
||||||
|
d.driver_name,
|
||||||
|
a.alarm_time AT TIME ZONE 'Africa/Nairobi' AS alarm_nairobi,
|
||||||
|
a.alarm_type,
|
||||||
|
a.alarm_name,
|
||||||
|
a.speed,
|
||||||
|
a.lat,
|
||||||
|
a.lng
|
||||||
|
FROM tracksolid.alarms a
|
||||||
|
LEFT JOIN tracksolid.devices d ON d.imei = a.imei
|
||||||
|
WHERE a.alarm_time >= DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
AT TIME ZONE 'Africa/Nairobi'
|
||||||
|
AND a.alarm_time < NOW()
|
||||||
|
ORDER BY a.alarm_time DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fleet totals for today at a glance
|
||||||
|
|
||||||
|
A single-row summary of the entire fleet's activity since midnight. Designed for a top-of-dashboard KPI card: how many vehicles have moved today, how many trips have been completed, total fleet kilometres, and total driving hours so far.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
WITH today_start AS (
|
||||||
|
SELECT DATE_TRUNC('day', NOW() AT TIME ZONE 'Africa/Nairobi')
|
||||||
|
AT TIME ZONE 'Africa/Nairobi' AS ts
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
COUNT(DISTINCT t.imei) AS active_vehicles_today,
|
||||||
|
COUNT(*) AS total_trips_today,
|
||||||
|
ROUND(SUM(t.distance_km) / 1000000.0, 2) AS total_fleet_km,
|
||||||
|
ROUND(SUM(t.driving_time_s) / 3600.0, 2) AS total_drive_hours,
|
||||||
|
ROUND(SUM(t.idle_time_s) / 3600.0, 2) AS total_idle_hours,
|
||||||
|
ROUND(AVG(t.avg_speed_kmh)::numeric, 2) AS fleet_avg_speed_kmh,
|
||||||
|
TO_CHAR(NOW() AT TIME ZONE 'Africa/Nairobi', 'HH24:MI') AS as_at_nairobi
|
||||||
|
FROM tracksolid.trips t
|
||||||
|
CROSS JOIN today_start ts
|
||||||
|
WHERE t.start_time >= ts.ts
|
||||||
|
AND t.start_time < NOW();
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 16. Known Data Issues
|
||||||
|
|
||||||
|
The following issues were identified during the April 2026 audit. Each represents either a data pipeline gap or a missing enrichment step.
|
||||||
|
|
||||||
|
| # | Table | Issue | Impact | Status |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | `devices` | `vehicle_name`, `vehicle_number`, `driver_name` all null for all 63 devices | All reports show blank vehicle identity | **Open** — manually populate or sync from fleet management source |
|
||||||
|
| 2 | `devices` | `fuel_100km` not set for any device | `fuel_consumed_l` in trips will remain null | **Open** — set fuel consumption rate per vehicle type |
|
||||||
|
| 3 | `alarms` | `alarm_type` and `alarm_name` were null for all 1,054 records | Alarm events could not be categorised | **Fixed** in `ingest_events_rev.py` [FIX-E06] — poll API uses `alertTypeId`/`alarmTypeName`, not webhook field names |
|
||||||
|
| 4 | `trips` | `max_speed_kmh` is null on all records | Cannot identify speeding from trip summaries | **Open** — `jimi.device.track.mileage` may not return this field; verify API response |
|
||||||
|
| 5 | `trips` | `distance_km` was stored in millimetres | All distance queries returned values 1,000,000× too large | **Fixed** — migration 04 divides historical data by 1,000,000 and renames column; ingestion code corrected [FIX-M11/M12] |
|
||||||
|
| 6 | `heartbeats` | 0 rows | Cannot distinguish parked-alive from powered-off | **Open** — verify tracker firmware supports heartbeat push |
|
||||||
|
| 7 | `obd_readings` | 0 rows | No engine health data | **Open** — requires OBD cable installation + `/pushobd` webhook registration in Tracksolid account |
|
||||||
|
| 8 | `parking_events` | 0 rows despite 358 successful API calls | No parking dwell-time reporting | **Fixed** in `ingest_movement_rev.py` [FIX-M13] — added missing `account` and `acc_type=0` params; fixed `durSecond` field mapping |
|
||||||
|
| 9 | `dwh_gold.*` | Both tables empty | Grafana dashboards have no data | **Fixed** — migration 05 adds `refresh_daily_metrics()` ETL function; run nightly via cron or n8n |
|
||||||
|
|
@ -408,8 +408,9 @@ def push_trip_report(token: str = Form(""), data_list: str = Form("")):
|
||||||
if not imei or not begin_time:
|
if not imei or not begin_time:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
miles_km = clean_num(item.get("miles"))
|
# [FIX-M11] API sends miles (km). Store directly as distance_km.
|
||||||
distance_m = miles_km * 1000 if miles_km is not None else None
|
# Previous code multiplied by 1000, producing mm not m.
|
||||||
|
distance_km = clean_num(item.get("miles"))
|
||||||
|
|
||||||
begin_lat = clean_num(item.get("beginLat"))
|
begin_lat = clean_num(item.get("beginLat"))
|
||||||
begin_lng = clean_num(item.get("beginLng"))
|
begin_lng = clean_num(item.get("beginLng"))
|
||||||
|
|
@ -418,7 +419,7 @@ def push_trip_report(token: str = Form(""), data_list: str = Form("")):
|
||||||
|
|
||||||
cur.execute("""
|
cur.execute("""
|
||||||
INSERT INTO tracksolid.trips (
|
INSERT INTO tracksolid.trips (
|
||||||
imei, start_time, end_time, distance_m,
|
imei, start_time, end_time, distance_km,
|
||||||
start_geom, end_geom,
|
start_geom, end_geom,
|
||||||
fuel_consumed_l, idle_time_s, trip_seq, source,
|
fuel_consumed_l, idle_time_s, trip_seq, source,
|
||||||
updated_at
|
updated_at
|
||||||
|
|
@ -433,13 +434,13 @@ def push_trip_report(token: str = Form(""), data_list: str = Form("")):
|
||||||
%s, %s, %s, 'push', NOW()
|
%s, %s, %s, 'push', NOW()
|
||||||
) ON CONFLICT (imei, start_time) DO UPDATE SET
|
) ON CONFLICT (imei, start_time) DO UPDATE SET
|
||||||
end_time = EXCLUDED.end_time,
|
end_time = EXCLUDED.end_time,
|
||||||
distance_m = EXCLUDED.distance_m,
|
distance_km = EXCLUDED.distance_km,
|
||||||
end_geom = EXCLUDED.end_geom,
|
end_geom = EXCLUDED.end_geom,
|
||||||
fuel_consumed_l = EXCLUDED.fuel_consumed_l,
|
fuel_consumed_l = EXCLUDED.fuel_consumed_l,
|
||||||
idle_time_s = EXCLUDED.idle_time_s,
|
idle_time_s = EXCLUDED.idle_time_s,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
""", (
|
""", (
|
||||||
imei, begin_time, end_time, distance_m,
|
imei, begin_time, end_time, distance_km,
|
||||||
begin_lng, begin_lat, begin_lng, begin_lat,
|
begin_lng, begin_lat, begin_lng, begin_lat,
|
||||||
end_lng, end_lat, end_lng, end_lat,
|
end_lng, end_lat, end_lng, end_lat,
|
||||||
clean_num(item.get("oils")),
|
clean_num(item.get("oils")),
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue