fleettickets/run_ingest.sh

#!/usr/bin/env bash
# run_ingest.sh — fleettickets · INC + CRQ ingest wrapper for cron (plain host/VM).
#
# Loads env from the local .env (DATABASE_URL + RUSTFS_* + GEOCODER_*) and drains
# both ticket change streams with --apply (watermark skip-if-unchanged + per-file
# archive are built in, so a run with no new files is a cheap no-op).
#
# Install on the instance (every 20 min, 06:00–20:40 EAT):
#   */20 6-20 * * *  /opt/fleettickets/run_ingest.sh >> /var/log/fleettickets.log 2>&1
# Ensure the crontab runs in the Africa/Nairobi timezone (CRON_TZ=Africa/Nairobi or
# the host/container TZ), since the export filenames and the schedule are EAT.
#
# On Coolify the two ingests run as separate Scheduled Tasks instead (see Dockerfile
# + docs/deployment-and-operations.md); this wrapper is the plain-host fallback.
set -euo pipefail

cd "$(dirname "$0")"

# Load .env if present (KEY=VALUE lines); never commit the real .env.
if [ -f .env ]; then
  set -a
  # shellcheck disable=SC1091
  . ./.env
  set +a
fi

# Prefer the project venv if it exists, else the python on PATH (e.g. in-container).
PY="python"
[ -x ".venv/bin/python" ] && PY=".venv/bin/python"

# Run from the repo root (cwd above) so `-m inc.import_inc` / `-m crq.import_crq`
# resolve the packages alongside pipeline.py + shared.py.
"$PY" -m inc.import_inc --from-bucket --apply
"$PY" -m crq.import_crq --from-bucket --apply

# Incremental cluster geocode (FT-BUG-02): NOT-EXISTS-guarded, so a run with no
# new clusters makes zero geocoder calls. Location-level geocoding stays a manual
# command (budget control): python -m inc.import_inc --geocode-locations --apply
"$PY" -m inc.import_inc --geocode-clusters --apply