From 3b79d5a62e39343f8cf46d8d3ef0b02d461a8cfc Mon Sep 17 00:00:00 2001 From: David Kiania Date: Fri, 8 May 2026 00:34:10 +0300 Subject: [PATCH] revert(infra): remove pgAdmin4 sidecar and configs Reverts the Phase 2 pgAdmin web sidecar from bc020cb. pgbouncer (Phase 1) stays in place. On the instance the pgadmin container has been stopped and removed and the pgadmin-data volume dropped; Coolify subdomain and PGADMIN_DEFAULT_* env vars to be removed in the UI separately. Files: - docker-compose.yaml: drop pgadmin service block + pgadmin-data volume - pgadmin/servers.json: delete (directory removed) - 260507_pgbouncer_deployment.md: strip Phase 2, runbook is pgbouncer-only Co-Authored-By: Claude Opus 4.7 --- 260507_pgbouncer_deployment.md | 121 ++++----------------------------- docker-compose.yaml | 22 ------ pgadmin/servers.json | 17 ----- 3 files changed, 14 insertions(+), 146 deletions(-) delete mode 100644 pgadmin/servers.json diff --git a/260507_pgbouncer_deployment.md b/260507_pgbouncer_deployment.md index fd0329a..726b288 100644 --- a/260507_pgbouncer_deployment.md +++ b/260507_pgbouncer_deployment.md @@ -1,8 +1,8 @@ -# pgbouncer + pgAdmin4 sidecar deployment +# pgbouncer sidecar deployment **Date:** 2026-05-07 **Branch:** `quality-program-2026-04-12` -**Status:** Plan approved; implementation pending +**Status:** Phase 1 deployed. Phase 2 (pgAdmin4 sidecar) was rolled back on 2026-05-08 — see git history (`bc020cb`, then reverted). --- @@ -15,11 +15,9 @@ connections from the ingest pipeline, the budget tips over and cascades — pgcli (and anything else trying to connect) starts failing. **Goal:** Add pgbouncer in front of `timescale_db` to enforce a connection -budget via transaction-mode pooling, and deploy pgAdmin4 as a Coolify-managed -sidecar that connects through pgbouncer over the Docker network. Net effect: -pgAdmin sprawl is multiplexed onto a small fixed pool of backends, admin -tooling moves on-VM (lower latency, persistent workspace, smaller external -attack surface), and host port 5433 becomes optional/closeable in a follow-up. +budget via transaction-mode pooling. Desktop pgAdmin (or any other admin +client) connects through pgbouncer and is multiplexed onto a small fixed +pool of backends. **Frozen scope (unchanged this round):** - DWH bronze pipeline (`dwh/*.sql`, `tracksolid_dwh@31.97.44.246:5888`) @@ -152,96 +150,15 @@ the password from the env var so the placeholder is never live. --- -## Phase 2 — pgAdmin4 sidecar pointed at pgbouncer - -Coolify UI maps an HTTPS subdomain (e.g. `pgadmin.stage.rahamafresh.com`) to -internal port 80, mirroring the Grafana pattern at `docker-compose.yaml:78–80`. - -```yaml - pgadmin: - image: dpage/pgadmin4:8.14 - restart: always - depends_on: - pgbouncer: - condition: service_healthy - env_file: .env - environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL} - - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD} - - PGADMIN_CONFIG_SERVER_MODE=True - - PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=False - - PGADMIN_DISABLE_POSTFIX=True - volumes: - - pgadmin-data:/var/lib/pgadmin - - ./pgadmin/servers.json:/pgadmin4/servers.json:ro - # COOLIFY DOMAIN LOGIC: - # Set the actual URL in the Coolify UI; service exposes port 80 internally. -``` - -Add `pgadmin-data` to the `volumes:` block at the bottom of the compose file. - -### Pre-registered server (`pgadmin/servers.json`) - -```json -{ - "Servers": { - "1": { - "Name": "tracksolid_db (via pgbouncer)", - "Group": "Servers", - "Host": "pgbouncer", - "Port": 6432, - "MaintenanceDB": "tracksolid_db", - "Username": "postgres", - "SSLMode": "disable", - "ConnectionParameters": { - "sslmode": "disable", - "connect_timeout": 10 - } - } - } -} -``` - -### New env vars in `.env` - -- `PGADMIN_DEFAULT_EMAIL` -- `PGADMIN_DEFAULT_PASSWORD` - -### Phase 2 verification - -1. In the Coolify UI, point a subdomain at the `pgadmin` service, port 80. -2. Open the URL, log in with `PGADMIN_DEFAULT_EMAIL` / - `PGADMIN_DEFAULT_PASSWORD`. -3. The pre-registered "tracksolid_db (via pgbouncer)" server appears in the - left tree. Connect; provide the `postgres` password when prompted (pgAdmin - stores it in its own keyring after first use). -4. Open a Query Tool, run `SELECT now(), current_user;`. -5. From the `pgbouncer` container admin console: - ```sql - SHOW POOLS; - ``` - `cl_active` should reflect the open pgAdmin tab(s); `sv_active` / - `sv_idle` should sum to ≤ `default_pool_size` (15). -6. **Stress test:** open ~30 Query Tool tabs, run `SELECT pg_sleep(0.1);` in - each. Confirm `tracksolid_db` total connection count stays bounded: - ```sql - SELECT count(*) FROM pg_stat_activity; - ``` - Should be `default_pool_size + reserve_pool_size + (other clients)`, - not the number of pgAdmin tabs. - ---- - ## Files to modify / create | Path | Change | |---|---| | `260507_pgbouncer_deployment.md` | THIS FILE — runbook for the rollout | -| `docker-compose.yaml` | Add `pgbouncer` and `pgadmin` services; add `pgadmin-data` volume | +| `docker-compose.yaml` | Add `pgbouncer` service | | `10_pgbouncer_auth.sql` | NEW — creates `pgbouncer` role + `public.user_lookup` SECURITY DEFINER function | -| `pgadmin/servers.json` | NEW — pre-registers `pgbouncer:6432` as the default server | -| `.env` | Add `PGBOUNCER_AUTH_PASSWORD`, `PGADMIN_DEFAULT_EMAIL`, `PGADMIN_DEFAULT_PASSWORD` (do not commit values) | -| `docs/CONNECTIONS.md` | Add a "pgbouncer + pgAdmin" section: pool mode, exposure, who uses it, how to connect for ad-hoc admin | +| `.env` | Add `PGBOUNCER_AUTH_PASSWORD` (do not commit values) | +| `docs/CONNECTIONS.md` | Add a "pgbouncer" section: pool mode, exposure, who uses it, how to connect for ad-hoc admin | | `CLAUDE.md` §3 / §4 | Note that admin tooling now goes through `pgbouncer:6432`; ingest/grafana/backup remain direct; reference this runbook | ## Files NOT to modify (frozen scope) @@ -263,14 +180,10 @@ Add `pgadmin-data` to the `volumes:` block at the bottom of the compose file. - `env_file: .env` + `depends_on: condition: service_healthy` mirrors the existing pattern at `docker-compose.yaml:28–31, 39–42, 50–53, 67–70, 87–90`. -- Coolify domain-via-UI mirrors the Grafana comment at - `docker-compose.yaml:78–80` and the webhook_receiver comment at - `docker-compose.yaml:54–55`. - Container-name resolution rule from CLAUDE.md §3 still applies for any - `docker exec` against the new services: + `docker exec` against the new service: ```bash docker ps --filter name=pgbouncer --format "{{.Names}}" | head -1 - docker ps --filter name=pgadmin --format "{{.Names}}" | head -1 ``` --- @@ -282,21 +195,19 @@ Add `pgadmin-data` to the `volumes:` block at the bottom of the compose file. pool + SAVEPOINTs against transaction-mode pgbouncer (low risk per exploration — no LISTEN/NOTIFY, no advisory locks across statements, no prepared statements in the codebase). -2. **Close host port 5433** on `timescale_db` once pgAdmin web UI is the - established admin path. Removes the public-IP Postgres exposure entirely. -3. **Rotate `dwh_owner` / `grafana_ro` plaintext passwords** still in +2. **Rotate `dwh_owner` / `grafana_ro` plaintext passwords** still in `dwh/260423_dwh_ddl_v1.sql` (pre-existing item from CLAUDE.md §10). --- ## Rollback -If pgbouncer or pgAdmin misbehaves: +If pgbouncer misbehaves: -1. **Stop the new services without touching the rest of the stack:** +1. **Stop the service without touching the rest of the stack:** ```bash - docker compose stop pgbouncer pgadmin - docker compose rm -f pgbouncer pgadmin + docker compose stop pgbouncer + docker compose rm -f pgbouncer ``` Ingest, Grafana, webhook, backup are unaffected — they were never cut over. @@ -319,10 +230,6 @@ If pgbouncer or pgAdmin misbehaves: - [ ] `psql -h pgbouncer -p 6432 -U postgres -d tracksolid_db -c 'SELECT 1'` from inside the network - [ ] `SHOW POOLS;` in pgbouncer admin shows `transaction` mode -- [ ] `pgadmin` service healthy — Coolify domain reachable over HTTPS -- [ ] Login + query through pgAdmin succeeds -- [ ] `SELECT count(*) FROM pg_stat_activity;` stays bounded under - 30-tab stress test - [ ] Existing pipelines unaffected: `tracksolid.ingestion_log` continues growing at current rate; Grafana dashboards still render - [ ] pgcli no longer hits "too many connections" when used alongside pgAdmin diff --git a/docker-compose.yaml b/docker-compose.yaml index 260258e..e8cb20b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -115,26 +115,6 @@ services: timeout: 5s retries: 3 - pgadmin: - # Web pgAdmin4, connecting to tracksolid_db through pgbouncer. - # Runbook: 260507_pgbouncer_deployment.md (Phase 2) - # Coolify UI maps a subdomain to this service on internal port 80. - image: dpage/pgadmin4 - restart: always - depends_on: - pgbouncer: - condition: service_healthy - env_file: .env - environment: - - PGADMIN_DEFAULT_EMAIL=${PGADMIN_DEFAULT_EMAIL} - - PGADMIN_DEFAULT_PASSWORD=${PGADMIN_DEFAULT_PASSWORD} - - PGADMIN_CONFIG_SERVER_MODE=True - - PGADMIN_CONFIG_MASTER_PASSWORD_REQUIRED=False - - PGADMIN_DISABLE_POSTFIX=True - volumes: - - pgadmin-data:/var/lib/pgadmin - - ./pgadmin/servers.json:/pgadmin4/servers.json:ro - db_backup: build: context: ./backup @@ -161,5 +141,3 @@ volumes: name: timescale-data grafana-data: name: grafana-data - pgadmin-data: - name: pgadmin-data diff --git a/pgadmin/servers.json b/pgadmin/servers.json deleted file mode 100644 index 3ec4934..0000000 --- a/pgadmin/servers.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Servers": { - "1": { - "Name": "tracksolid_db (via pgbouncer)", - "Group": "Servers", - "Host": "pgbouncer", - "Port": 6432, - "MaintenanceDB": "tracksolid_db", - "Username": "postgres", - "SSLMode": "disable", - "ConnectionParameters": { - "sslmode": "disable", - "connect_timeout": 10 - } - } - } -}