Remove Trino: Superset reads fuel-db directly as fuel_ro (PD-14)
trino/ deleted; the role trino_ro becomes fuel_ro, its password read from a secret file at connect time; dashboards' SQL and column list for PostgreSQL; the Compose project is renamed petrolyst, with volumes pinned to their old names. AGENTS.md points to the architecture context. Not deployed: the next deploy needs README's one-time steps first. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
686318dfac
commit
8405706ed8
25 files changed
+453
-410
No files matched your search
@@ -1,9 +1,9 @@
|
||||
# Fuel-monitoring analytics: Superset on atoma.petrolyst.com, reading the
|
||||
# operational store (fuel-db) through Trino. Run only through deploy/deploy.sh
|
||||
# or deploy/rollback.sh: they export SUPERSET_IMAGE_TAG and TRINO_CONFIG_HASH
|
||||
# from the release's ARTIFACTS file, and the :? guards below stop Compose from
|
||||
# running with either unset.
|
||||
name: fuel-monitoring
|
||||
# operational store (fuel-db) directly as the read-only role fuel_ro (Trino was
|
||||
# removed on 29 Oct 2026). Run only through deploy/deploy.sh or
|
||||
# deploy/rollback.sh: they export SUPERSET_IMAGE_TAG from the release's
|
||||
# ARTIFACTS file, and the :? guard below stops Compose from running without it.
|
||||
name: petrolyst
|
||||
|
||||
x-superset-env: &superset-env
|
||||
KEYCLOAK_REALM_URL: ${KEYCLOAK_REALM_URL:-https://iam.thedigitalbridge.io/realms/thedigitalbridge}
|
||||
@@ -15,6 +15,8 @@ x-superset: &superset
|
||||
- superset_secret_key
|
||||
- superset_db_password
|
||||
- keycloak_petrolyst_client_secret
|
||||
# fuel_ro's password, read at connect time (superset_config.py)
|
||||
- fuel_db_ro_password
|
||||
|
||||
services:
|
||||
superset:
|
||||
@@ -100,25 +102,6 @@ services:
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
|
||||
trino:
|
||||
image: trinodb/trino:484
|
||||
env_file:
|
||||
- ${PETROLYST_ROOT:-/opt/petrolyst}/env/fuel-monitoring.env
|
||||
volumes:
|
||||
- ./trino/etc:/etc/trino:ro
|
||||
labels:
|
||||
# Compose cannot see content changes in a bind mount; this label changes
|
||||
# with trino/etc, so exactly a config change recreates the container.
|
||||
petrolyst.config-hash: ${TRINO_CONFIG_HASH:?TRINO_CONFIG_HASH unset - run through deploy/deploy.sh}
|
||||
depends_on:
|
||||
fuel-db:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- backend
|
||||
restart: unless-stopped
|
||||
mem_limit: 3g
|
||||
# The image's own healthcheck (/usr/lib/trino/bin/health-check) applies.
|
||||
|
||||
fuel-db:
|
||||
image: timescale/timescaledb:2.31.0-pg17
|
||||
environment:
|
||||
@@ -128,7 +111,7 @@ services:
|
||||
TS_TUNE_NUM_CPUS: "2"
|
||||
secrets:
|
||||
- fuel_db_password
|
||||
- fuel_db_trino_password
|
||||
- fuel_db_ro_password
|
||||
volumes:
|
||||
- fuel_db_data:/var/lib/postgresql/data
|
||||
# A single file, not the folder: the image's own TimescaleDB init scripts
|
||||
@@ -157,9 +140,14 @@ networks:
|
||||
# so fuel-db's loopback port works.
|
||||
fuel-db-tunnel: {}
|
||||
|
||||
# Created under the project's first name, fuel-monitoring; named explicitly so the
|
||||
# data stayed when the project became petrolyst (29 Oct 2026). Compose warns once
|
||||
# that they belong to another project; that is expected.
|
||||
volumes:
|
||||
superset_db_data:
|
||||
name: fuel-monitoring_superset_db_data
|
||||
fuel_db_data:
|
||||
name: fuel-monitoring_fuel_db_data
|
||||
|
||||
secrets:
|
||||
superset_secret_key:
|
||||
@@ -168,7 +156,7 @@ secrets:
|
||||
file: ${PETROLYST_ROOT:-/opt/petrolyst}/secrets/fuel-monitoring/superset_db_password
|
||||
fuel_db_password:
|
||||
file: ${PETROLYST_ROOT:-/opt/petrolyst}/secrets/fuel-monitoring/fuel_db_password
|
||||
fuel_db_trino_password:
|
||||
file: ${PETROLYST_ROOT:-/opt/petrolyst}/secrets/fuel-monitoring/fuel_db_trino_password
|
||||
fuel_db_ro_password:
|
||||
file: ${PETROLYST_ROOT:-/opt/petrolyst}/secrets/fuel-monitoring/fuel_db_ro_password
|
||||
keycloak_petrolyst_client_secret:
|
||||
file: ${PETROLYST_ROOT:-/opt/petrolyst}/secrets/shared/keycloak_petrolyst_client_secret
|
||||
@@ -5,21 +5,24 @@
|
||||
# Loaders log in as their own role but act as fuel_owner (README
|
||||
# "Data loaders"): the default privileges below cover only what
|
||||
# fuel_owner itself creates, not what its members create.
|
||||
# trino_ro what Trino connects as: read-only, also on tables created later.
|
||||
# fuel_ro what Superset and the Petrolyst server read as: SELECT on public,
|
||||
# also on tables created later. Read-only by default as well: a
|
||||
# second lock behind the grants.
|
||||
set -euo pipefail
|
||||
|
||||
trino_password=$(cat /run/secrets/fuel_db_trino_password)
|
||||
ro_password=$(cat /run/secrets/fuel_db_ro_password)
|
||||
|
||||
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" \
|
||||
-v trino_password="$trino_password" <<'SQL'
|
||||
-v ro_password="$ro_password" <<'SQL'
|
||||
CREATE ROLE fuel_owner NOLOGIN;
|
||||
ALTER DATABASE fuel OWNER TO fuel_owner;
|
||||
REVOKE CREATE ON SCHEMA public FROM PUBLIC;
|
||||
|
||||
CREATE ROLE trino_ro LOGIN PASSWORD :'trino_password';
|
||||
GRANT CONNECT ON DATABASE fuel TO trino_ro;
|
||||
GRANT USAGE ON SCHEMA public TO trino_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO trino_ro;
|
||||
CREATE ROLE fuel_ro LOGIN PASSWORD :'ro_password';
|
||||
ALTER ROLE fuel_ro SET default_transaction_read_only = on;
|
||||
GRANT CONNECT ON DATABASE fuel TO fuel_ro;
|
||||
GRANT USAGE ON SCHEMA public TO fuel_ro;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA public TO fuel_ro;
|
||||
ALTER DEFAULT PRIVILEGES FOR ROLE fuel_owner IN SCHEMA public
|
||||
GRANT SELECT ON TABLES TO trino_ro;
|
||||
GRANT SELECT ON TABLES TO fuel_ro;
|
||||
SQL
|
||||
@@ -1,4 +1,4 @@
|
||||
-- 0002_fm_derivations.sql: internal derivations in schema `fm` (not visible to Trino).
|
||||
-- 0002_fm_derivations.sql: internal derivations in schema `fm` (not readable by fuel_ro).
|
||||
-- DRAFT, not applied. Applied after 0001 by a loader acting as fuel_owner (README in this folder).
|
||||
-- Design: fuel-monitoring .context/plans/2026-10-24-fuel-db-baseline-dashboard.md, §4 and §5.1.
|
||||
BEGIN;
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
-- 0003_bi_views.sql: the views Superset reads through Trino (schema `public`), plus
|
||||
-- fm.v_site_baseline_status. Owned by fuel_owner, so trino_ro can read them through the existing
|
||||
-- default privileges; they run with the owner's rights, so Trino never reads `fm` (it lists fm's
|
||||
-- table names, as it lists every schema's, but any query on them is refused).
|
||||
-- 0003_bi_views.sql: the views Superset reads (schema `public`), plus
|
||||
-- fm.v_site_baseline_status. Owned by fuel_owner, so fuel_ro can read them through the existing
|
||||
-- default privileges; they run with the owner's rights, so fuel_ro never reads `fm` (the catalog
|
||||
-- shows fm's table names, as it shows every schema's, but any query on them is refused).
|
||||
-- DRAFT, not applied. Applied after 0002 by a loader acting as fuel_owner (README in this folder).
|
||||
-- Design: fuel-monitoring .context/plans/2026-10-24-fuel-db-baseline-dashboard.md, §5.2 and §6.
|
||||
BEGIN;
|
||||
@@ -97,7 +97,7 @@ LEFT JOIN pp USING (site_uid) LEFT JOIN fm.v_site_baseline_status bs USING (site
|
||||
LEFT JOIN cal USING (site_uid) LEFT JOIN latest USING (site_uid);
|
||||
|
||||
-- Every view below joins public.v_site for site_uid, site_code, site_name and site_label, so one
|
||||
-- Site filter drives every dataset, and casts to Trino-safe types (varchar(n), integer,
|
||||
-- Site filter drives every dataset, and casts to plain types (varchar(n), integer,
|
||||
-- numeric(p,s), date, timestamp(0), boolean). Live = asserted, reviewed or disputed.
|
||||
|
||||
-- S2, R3: one row per site and measure; the rows stay when a value is not established.
|
||||
|
||||
@@ -10,7 +10,7 @@ The design and its rationale live in the fuel-monitoring corpus:
|
||||
|
||||
## Files
|
||||
|
||||
| File | Creates | Visible to Trino |
|
||||
| File | Creates | Readable by `fuel_ro` (Superset, Petrolyst server) |
|
||||
|---|---|---|
|
||||
| `0001_fm_core.sql` | schema `fm`, `btree_gist` in it, 25 tables, constraints, trigger-backed rules 1–13 | no |
|
||||
| `0002_fm_derivations.sql` | internal views in `fm`: claims → visits → stock intervals → L/h and L/day intervals, dependency hashes, baseline validity | no |
|
||||
@@ -21,7 +21,7 @@ Each file is one transaction. Apply them in order.
|
||||
## Applying (on the server)
|
||||
|
||||
1. Create a loader role as in the repository README, "Data loaders". Its sessions act as
|
||||
`fuel_owner`, so every object belongs to `fuel_owner`. `trino_ro` can then read the `public`
|
||||
`fuel_owner`, so every object belongs to `fuel_owner`. `fuel_ro` can then read the `public`
|
||||
views through the existing default privileges, with no role or grant changes.
|
||||
2. Open the tunnel and apply the three files as the loader:
|
||||
|
||||
@@ -32,13 +32,14 @@ Each file is one transaction. Apply them in order.
|
||||
done
|
||||
```
|
||||
|
||||
3. Check through Trino (`docker exec -it fuel-monitoring-trino-1 trino`):
|
||||
- `SHOW TABLES FROM fuel.public` lists the 18 `v_*` views;
|
||||
- `SELECT count(*) FROM fuel.fm.site` fails with "permission denied for schema fm".
|
||||
3. Check as `fuel_ro`
|
||||
(`docker exec -it petrolyst-fuel-db-1 psql -U postgres -d fuel`, then `SET ROLE fuel_ro;`):
|
||||
- `\dv public.*` lists the 18 `v_*` views;
|
||||
- `SELECT count(*) FROM fm.site;` fails with "permission denied for schema fm".
|
||||
|
||||
The `public` views are owned by `fuel_owner` and run with its rights, so Trino never needs, and
|
||||
never gets, read access to `fm`. Trino's metadata still lists fm's table and column names, as it
|
||||
lists every schema in fuel-db (TimescaleDB's internal ones included); names only, no rows.
|
||||
The `public` views are owned by `fuel_owner` and run with its rights, so `fuel_ro` never needs,
|
||||
and never gets, read access to `fm`. PostgreSQL's catalog still shows every schema's table and
|
||||
column names to any role, so Superset's SQL Lab can list fm's names; names only, no rows.
|
||||
|
||||
## Data
|
||||
|
||||
@@ -91,6 +92,8 @@ it, except baseline versions and their approvals, which live only in fuel-db: ex
|
||||
|
||||
## Verified locally (2026-10-24, scratch only)
|
||||
|
||||
With Trino, before its removal on 29 Oct 2026; `trino_ro` is now `fuel_ro`.
|
||||
|
||||
Setup:
|
||||
- `timescale/timescaledb:2.31.0-pg17`, with this repository's `01-roles.sh`;
|
||||
- a loader created by the README recipe;
|
||||
|
||||
@@ -6,10 +6,9 @@
|
||||
FROM apache/superset:6.1.0@sha256:16b50bbef6648912a79e3293d418fefa743ce555d36c5af6b85d859119ed7f88
|
||||
|
||||
USER root
|
||||
# The 6.1.0 image ships neither an OAuth client nor the Postgres and Trino
|
||||
# drivers.
|
||||
# The 6.1.0 image ships neither an OAuth client nor the Postgres driver.
|
||||
RUN uv pip install --no-cache --python /app/.venv/bin/python \
|
||||
authlib==1.9.1 psycopg2-binary==2.9.13 "trino[sqlalchemy]==0.340.0"
|
||||
authlib==1.9.1 psycopg2-binary==2.9.13
|
||||
|
||||
COPY --from=insights --chown=superset:superset superset_config.py petrolyst_security.py /app/pythonpath/
|
||||
COPY --from=insights --chown=superset:superset roles/ /app/petrolyst-roles/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# superset-init: migrate the metadata database, sync roles and permissions, and
|
||||
# register the Trino connection. It is copied into the Superset image, so a
|
||||
# register the connection to fuel-db. It is copied into the Superset image, so a
|
||||
# change here changes the image tag; deploy/deploy.sh runs it on a fresh install
|
||||
# and whenever that tag changes. Every step is idempotent. Unlike the
|
||||
# upstream docker-init.sh it never creates a local admin: users come from
|
||||
@@ -9,6 +9,25 @@ set -euo pipefail
|
||||
|
||||
superset db upgrade
|
||||
superset init
|
||||
|
||||
# Trino was removed on 29 Oct 2026. An instance still holding its connection
|
||||
# "Fuel (Trino)" gets it renamed, so its datasets and dashboards keep their ids;
|
||||
# the next step points it at fuel-db. Drop this once no instance has it.
|
||||
/app/.venv/bin/python - <<'PY'
|
||||
from superset.app import create_app
|
||||
|
||||
with create_app().app_context():
|
||||
from superset import db
|
||||
from superset.models.core import Database
|
||||
|
||||
old = db.session.query(Database).filter_by(database_name="Fuel (Trino)").one_or_none()
|
||||
if old and not db.session.query(Database).filter_by(database_name="Fuel").count():
|
||||
old.database_name = "Fuel"
|
||||
db.session.commit()
|
||||
print('renamed the connection "Fuel (Trino)" to "Fuel"')
|
||||
PY
|
||||
|
||||
# No password here: superset_config.py reads it from /run/secrets at connect time.
|
||||
superset set-database-uri \
|
||||
--database_name "Fuel (Trino)" \
|
||||
--uri "trino://superset@trino:8080/fuel"
|
||||
--database_name "Fuel" \
|
||||
--uri "postgresql+psycopg2://fuel_ro@fuel-db:5432/fuel"
|
||||
@@ -1,3 +0,0 @@
|
||||
# Analytics never writes through Trino. fuel-db's trino_ro role is read-only as
|
||||
# well; this is the second lock.
|
||||
access-control.name=read-only
|
||||
@@ -1,5 +0,0 @@
|
||||
# The fuel-monitoring operational store (service fuel-db), read-only.
|
||||
connector.name=postgresql
|
||||
connection-url=jdbc:postgresql://fuel-db:5432/fuel
|
||||
connection-user=trino_ro
|
||||
connection-password=${ENV:FUEL_DB_TRINO_PASSWORD}
|
||||
@@ -1,14 +0,0 @@
|
||||
# Single node: coordinator and worker in one process. Reachable only on the
|
||||
# backend network (Superset); no port is published and Traefik has no route.
|
||||
coordinator=true
|
||||
node-scheduler.include-coordinator=true
|
||||
http-server.http.port=8080
|
||||
discovery.uri=http://localhost:8080
|
||||
catalog.management=static
|
||||
|
||||
# Memory bounds. The container is capped at 3g (compose.yaml) and jvm.config
|
||||
# gives the heap 75 % of it (~2.25 GB); 1 GB for queries plus 512 MB headroom
|
||||
# leaves room for the rest of the heap.
|
||||
query.max-memory=1GB
|
||||
query.max-memory-per-node=1GB
|
||||
memory.heap-headroom-per-node=512MB
|
||||
@@ -1,14 +0,0 @@
|
||||
-server
|
||||
-agentpath:/usr/lib/trino/bin/libjvmkill.so
|
||||
-XX:InitialRAMPercentage=75
|
||||
-XX:MaxRAMPercentage=75
|
||||
-XX:G1HeapRegionSize=32M
|
||||
-XX:+ExplicitGCInvokesConcurrent
|
||||
-XX:+HeapDumpOnOutOfMemoryError
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:-OmitStackTraceInFastThrow
|
||||
-XX:ReservedCodeCacheSize=256M
|
||||
-XX:PerMethodRecompilationCutoff=10000
|
||||
-XX:PerBytecodeRecompilationCutoff=10000
|
||||
-Djdk.attach.allowAttachSelf=true
|
||||
-Djdk.nio.maxCachedBufferSize=2000000
|
||||
@@ -1 +0,0 @@
|
||||
io.trino=INFO
|
||||
@@ -1,3 +0,0 @@
|
||||
node.environment=petrolyst
|
||||
node.data-dir=/data/trino
|
||||
node.id=${ENV:HOSTNAME}
|
||||
Reference in new issue
Block a user