From 1b8e57adb412b6efa558b3ebd4865c2375a855d7 Mon Sep 17 00:00:00 2001 From: Nasim Sadat Date: Thu, 29 Oct 2026 20:50:26 +0100 Subject: [PATCH] Trial state of 29 Oct 2026: ATOMA reader setup, app dashboards, embedding notes Baseline before the clean-up (PD-13): the PD-11 reader setup and the PD-12 app dashboards as they ran on the local stack, uncommitted until now. Not deployed, not independently reviewed. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 +- deploy/lib.sh | 3 + keycloak/petrolyst-partial-import.json | 1 + petrolyst-insights/README.md | 33 +- .../assets/branding/atoma_logo.svg | 39 + .../branding/petrolyst_wordmark_fullcolor.svg | 28 + .../branding/petrolyst_wordmark_white.svg | 18 + petrolyst-insights/assets/branding/reader.css | 154 +++ petrolyst-insights/dashboards/README.md | 50 +- petrolyst-insights/dashboards/atoma/SPEC.md | 542 ++++++-- .../dashboards/atoma/build_bundle.py | 1172 +++++++++++++++++ .../dashboards/atoma/columns.tsv | 309 +++++ .../dashboards/atoma/dashboard.css | 33 + .../dashboards/atoma/import_bundle.py | 151 +++ petrolyst-insights/patches/README.md | 5 +- .../patches/fix_redact_query_for_readers.py | 54 + petrolyst-insights/petrolyst_reader.py | 150 +++ petrolyst-insights/petrolyst_security.py | 6 +- petrolyst-insights/roles/atoma_guest.py | 23 + petrolyst-insights/roles/atoma_reader.py | 76 ++ petrolyst-insights/superset_config.py | 44 +- .../templates/head_custom_extra.html | 6 + stacks/fuel-monitoring/superset/Dockerfile | 4 +- stacks/fuel-monitoring/superset/init.sh | 2 + 24 files changed, 2777 insertions(+), 136 deletions(-) create mode 100644 petrolyst-insights/assets/branding/atoma_logo.svg create mode 100644 petrolyst-insights/assets/branding/petrolyst_wordmark_fullcolor.svg create mode 100644 petrolyst-insights/assets/branding/petrolyst_wordmark_white.svg create mode 100644 petrolyst-insights/assets/branding/reader.css create mode 100644 petrolyst-insights/dashboards/atoma/build_bundle.py create mode 100644 petrolyst-insights/dashboards/atoma/columns.tsv create mode 100644 petrolyst-insights/dashboards/atoma/dashboard.css create mode 100644 petrolyst-insights/dashboards/atoma/import_bundle.py create mode 100644 petrolyst-insights/patches/fix_redact_query_for_readers.py create mode 100644 petrolyst-insights/petrolyst_reader.py create mode 100644 petrolyst-insights/roles/atoma_guest.py create mode 100644 petrolyst-insights/roles/atoma_reader.py create mode 100644 petrolyst-insights/templates/head_custom_extra.html diff --git a/README.md b/README.md index eff10a9..91c350c 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,8 @@ deploy secrets. and "Skip" if a resource exists. This creates: - the confidential client `petrolyst`: standard flow only, PKCE S256, redirect URIs for both hosts; - - its client roles `superset-admin`, `superset-editor`, `superset-viewer` and `traefik-admin`; + - its client roles `superset-admin`, `superset-editor`, `superset-viewer`, `atoma-reader` + and `traefik-admin`; - a mapper that puts those roles into the claim `petrolyst_roles`. 2. Clients → petrolyst → Credentials: copy the client secret (step 3 stores it on the server). 3. **MFA**, which Exhibit B requires. A copy of the built-in `browser` flow is not enough, for two @@ -109,7 +110,11 @@ deploy secrets. enrols first. Other clients of the realm are unaffected. There is no silent SSO between Superset and the dashboard. 4. Give people roles: Users → (user) → Role mapping → Assign role → filter by clients → - `petrolyst`. Group mappings work too. + `petrolyst`. Group mappings work too. ATOMA's users get `atoma-reader` and nothing else. + They then see only the ATOMA dashboard: no menus, nothing to edit, and Settings offers only + Logout (Superset role "ATOMA Reader", `petrolyst-insights/roles/atoma_reader.py`). + A realm that already has the client gets the new role by hand (Clients → petrolyst → + Roles → Create role) or by importing again with "Skip". 5. Users of the Traefik dashboard need **Email verified** switched on, because oauth2-proxy refuses unverified addresses. @@ -160,6 +165,7 @@ becomes the first Superset admin; there is no local admin account. | fuel-db from the Mac | `ssh -L 5433:127.0.0.1:5433 petrolyst`, then connect to `localhost:5433`, database `fuel` | | Logs | `docker compose -p fuel-monitoring logs -f superset` (or `-p traefik`) | | Superset break-glass | `docker exec -it fuel-monitoring-superset-1 superset fab ...` | +| Reset the role "ATOMA Reader" (after an edit in the UI) | `docker exec fuel-monitoring-superset-1 /app/.venv/bin/python /app/petrolyst-roles/atoma_reader.py`; a deploy does it whenever the Superset image changes | **Manual deploy**, the only way to deploy (there is no CI runner). On the Mac, in this repository: diff --git a/deploy/lib.sh b/deploy/lib.sh index deaa1b1..c1b44a3 100644 --- a/deploy/lib.sh +++ b/deploy/lib.sh @@ -33,6 +33,9 @@ SUPERSET_INPUTS=( stacks/fuel-monitoring/superset/init.sh petrolyst-insights/superset_config.py petrolyst-insights/petrolyst_security.py + petrolyst-insights/petrolyst_reader.py + petrolyst-insights/templates + petrolyst-insights/roles petrolyst-insights/assets/branding petrolyst-insights/patches ) diff --git a/keycloak/petrolyst-partial-import.json b/keycloak/petrolyst-partial-import.json index 77e4295..ac31361 100644 --- a/keycloak/petrolyst-partial-import.json +++ b/keycloak/petrolyst-partial-import.json @@ -49,6 +49,7 @@ { "name": "superset-admin", "description": "Superset Admin at atoma.petrolyst.com" }, { "name": "superset-editor", "description": "Superset Alpha: build charts and dashboards" }, { "name": "superset-viewer", "description": "Superset Gamma: view dashboards shared with the role" }, + { "name": "atoma-reader", "description": "Superset ATOMA Reader: read the ATOMA dashboard only" }, { "name": "traefik-admin", "description": "Traefik dashboard at traefik.petrolyst.com" } ] } diff --git a/petrolyst-insights/README.md b/petrolyst-insights/README.md index 14a5228..478c077 100644 --- a/petrolyst-insights/README.md +++ b/petrolyst-insights/README.md @@ -14,9 +14,12 @@ and where it runs lives in `stacks/`, `traefik/`, `deploy/` and `host/`. | Path | What it is | In the image at | |---|---|---| | `superset_config.py` | Keycloak sign-in, role mapping, proxy and cookie settings, Redis caches, theme and chart palettes | `/app/pythonpath/` | -| `petrolyst_security.py` | `PetrolystSecurityManager`: user info and roles from the Keycloak ID token, `/login/` straight to Keycloak, `/logout/` through Keycloak | `/app/pythonpath/` | -| `assets/branding/` | logos (full colour and white), favicons, app icons | `/app/superset/static/assets/branding/`, served at `/static/assets/branding/…` | -| `patches/` | runtime patches, `fix_*.py` with `--verify` (see its README); empty at start | `/app/patches/petrolyst/`, applied during the build | +| `petrolyst_security.py` | `PetrolystSecurityManager`: user info and roles from the Keycloak ID token, `/login/` straight to Keycloak, `/logout/` through Keycloak; takes the reader rules | `/app/pythonpath/` | +| `petrolyst_reader.py` | readers ("ATOMA Reader"): reach only dashboards granted to the role, own nothing, get no SQL; land on the ATOMA dashboard; `register` is `FLASK_APP_MUTATOR` | `/app/pythonpath/` | +| `templates/` | `head_custom_extra.html`: loads `reader.css` for readers on every page | `/app/pythonpath/templates/` | +| `roles/` | `atoma_reader.py`: the role "ATOMA Reader" with exactly its permissions, run by `superset-init`; `atoma_guest.py`: the role "ATOMA Guest" for the Petrolyst app's embedded dashboards, run by hand (see "Embedding") | `/app/petrolyst-roles/` | +| `assets/branding/` | logos (full colour and white): the wordmark alone for the navbar, and the stacked logo; favicons, app icons; `reader.css`; ATOMA's logo for the fleet map | `/app/superset/static/assets/branding/`, served at `/static/assets/branding/…` | +| `patches/` | runtime patches, `fix_*.py` with `--verify` (see its README): `fix_redact_query_for_readers.py` | `/app/patches/petrolyst/`, applied during the build | | `dashboards/` | exported dashboards and import scripts (see its README) | not in the image | How the image consumes this folder: `stacks/fuel-monitoring/compose.yaml` passes it to the build @@ -51,10 +54,34 @@ On the server they come from `/opt/petrolyst/secrets/` (see the deployment READM | `superset-admin` | Admin | | `superset-editor` | Alpha | | `superset-viewer` | Gamma | + | `atoma-reader` | ATOMA Reader: the ATOMA dashboard only, read-only (SPEC.md §6) | | none | Public (no permissions) | - `DASHBOARD_RBAC` is on: a viewer sees a dashboard once an admin adds the Gamma role (or a custom one) to it. +- `FILTERBAR_CLOSED_BY_DEFAULT` is on: a dashboard's filter column opens collapsed, a narrow + strip with an expand button; `?expand_filters=true` opens it from a link. + +## Embedding (local trial only) + +The Petrolyst app (PD-12, petrolyst-platform `app/` and `services/app/`) shows dashboards inside +its Insights lens with Superset guest passes. This is switched on **only in the local scratch +stack** (`.local/fmdb-scratch`); `superset_config.py` here leaves it off, and loads without the +guest secret. + +- The scratch stack's `superset_config_local.py` turns on `EMBEDDED_SUPERSET`, names the guest + role `GUEST_ROLE_NAME = "ATOMA Guest"`, reads the signing secret with + `_read_secret("superset_guest_token_secret")` and expects the audience `petrolyst-app`. The + app service signs the guest passes itself with the same secret. +- `roles/atoma_guest.py` gives "ATOMA Guest" the same permissions as "ATOMA Reader". Which + dashboard a guest sees comes from the pass, not the role. It is not in `init.sh`; run it once, + and again whenever the permissions in `atoma_reader.py` change: + + docker exec /app/.venv/bin/python /app/petrolyst-roles/atoma_guest.py + +- The app's dashboards and their embedding ids: `dashboards/README.md`. + +Production embedding is an open point of PD-12 (`.context/design-decisions.md`). ## Look and feel diff --git a/petrolyst-insights/assets/branding/atoma_logo.svg b/petrolyst-insights/assets/branding/atoma_logo.svg new file mode 100644 index 0000000..dfb6e12 --- /dev/null +++ b/petrolyst-insights/assets/branding/atoma_logo.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/petrolyst-insights/assets/branding/petrolyst_wordmark_fullcolor.svg b/petrolyst-insights/assets/branding/petrolyst_wordmark_fullcolor.svg new file mode 100644 index 0000000..46d7ee0 --- /dev/null +++ b/petrolyst-insights/assets/branding/petrolyst_wordmark_fullcolor.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/petrolyst-insights/assets/branding/petrolyst_wordmark_white.svg b/petrolyst-insights/assets/branding/petrolyst_wordmark_white.svg new file mode 100644 index 0000000..8b53690 --- /dev/null +++ b/petrolyst-insights/assets/branding/petrolyst_wordmark_white.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/petrolyst-insights/assets/branding/reader.css b/petrolyst-insights/assets/branding/reader.css new file mode 100644 index 0000000..6ae6b24 --- /dev/null +++ b/petrolyst-insights/assets/branding/reader.css @@ -0,0 +1,154 @@ +/* Readers only (petrolyst_reader.py, templates/head_custom_extra.html). Hides what Superset 6.1 + shows whatever the permissions. Cosmetic: what protects is the role and the reader rules. */ + +/* The dashboard's title row: name, favourite, refresh, owner, last change, "…". */ +[data-test="dashboard-header-container"] { + display: none !important; +} + +/* ATOMA's Fleet tab: no filter column and its resize handle (no filter applies there); the + column is empty then, so the map takes the full width. Site details keeps it. Also hidden + until the dashboard's grid is there, so it does not flash up while the Fleet tab loads. + dashboards/atoma/build_bundle.py checks that this names the Fleet tab. */ +body:has(.ant-tabs-tab-active[data-node-key="TAB-fleet"]) [data-test="dashboard-filters-panel"], +body:has(.ant-tabs-tab-active[data-node-key="TAB-fleet"]) div:has(+ [data-test="dashboard-filters-panel"]), +body:not(:has([data-test="grid-container"])) [data-test="dashboard-filters-panel"], +body:not(:has([data-test="grid-container"])) div:has(+ [data-test="dashboard-filters-panel"]) { + display: none !important; +} + +/* Superset sizes blocks in pixels and measures again only about 0.3 s after loading, so the + map would start beside a column that is not there and then widen. Its block spans all 12 + columns (build_bundle.py checks it), so the full width is always right. Its column grows to + the row's width; the blocks inside it follow. + Height: down to the window's bottom edge instead of the layout's 800 px. Above the block are + 123 px (navbar, margin, tab row) and below it 16, so the map ends 32 px above the edge, as it + does from the sides. At least 592 px: the map's own minimum height, 560, and the padding. */ +.grid-row > .dragdroppable-column:has(#MARKDOWN-fleet-map) { + flex-grow: 1; +} +#MARKDOWN-fleet-map > .resizable-container { + width: 100% !important; + max-width: 100% !important; + height: max(592px, 100vh - 139px) !important; +} + +/* Navbar: "+" (new) and the theme switcher. */ +li.ant-menu-submenu:has(> .ant-menu-submenu-title[data-menu-id$="-new-dropdown"]), +li.ant-menu-submenu:has(> .ant-menu-submenu-title[data-menu-id$="-theme-sub-menu"]) { + display: none !important; +} + +/* Settings: Info and the About group (Superset's name and version); User → Logout remains. */ +li.ant-menu-item[data-menu-id$="-info"], +li.ant-menu-item-group:has([data-menu-id$="-about-info"]), +li.ant-menu-item-divider:has(+ li.ant-menu-item-group [data-menu-id$="-about-info"]) { + display: none !important; +} + +/* ATOMA's top-level tabs as a lens switch in the navbar, centred: Map and Insights (owner, + 29 Oct). Pinned to the window, a scroll-linked animation moves it up as fast as the navbar + scrolls away. Without that support, or narrower than 900 px, the tab row stays in the page. + With the switch, the map's block grows by the tab row's 46 px: 93 px are left above and + below it. */ +@supports (animation-timeline: scroll()) { + @media (min-width: 900px) { + #TABS-main > .ant-tabs-nav { + position: fixed; + top: 9px; + left: 50%; + transform: translateX(-50%); + z-index: 1000; + margin: 0; + padding: 4px; + border-radius: 12px; + background: rgba(11, 29, 51, 0.06); + animation: petrolyst-lens-scroll linear both; + animation-timeline: scroll(root block); + animation-range: 0 61px; + } + #TABS-main > .ant-tabs-nav::before, + #TABS-main > .ant-tabs-nav .ant-tabs-ink-bar, + #TABS-main > .ant-tabs-nav .anchor-link-container { + display: none; + } + #TABS-main > .ant-tabs-nav .ant-tabs-nav-list { + gap: 4px; + } + #TABS-main > .ant-tabs-nav .ant-tabs-tab { + margin: 0 !important; + padding: 7px 18px 7px 14px !important; + border-radius: 9px !important; + font-size: 15px; + font-weight: 600; + color: #3b4a5c; + } + #TABS-main > .ant-tabs-nav .ant-tabs-tab-active { + background: #0b1d33 !important; + } + #TABS-main > .ant-tabs-nav .ant-tabs-tab-active .ant-tabs-tab-btn { + color: #fff !important; + } + #TABS-main > .ant-tabs-nav [data-test="span-title"]::before { + content: ""; + display: inline-block; + width: 18px; + height: 18px; + margin-right: 8px; + vertical-align: -3px; + background: currentColor; + -webkit-mask: var(--lens-icon) center / contain no-repeat; + mask: var(--lens-icon) center / contain no-repeat; + } + #TABS-main .ant-tabs-tab[data-node-key="TAB-fleet"] { + --lens-icon: url("data:image/svg+xml,%3Csvg%20xmlns='http://www.w3.org/2000/svg'%20viewBox='0%200%2024%2024'%20fill='none'%20stroke='black'%20stroke-width='2'%20stroke-linecap='round'%20stroke-linejoin='round'%3E%3Cpath%20d='M9%204%203%206v14l6-2%206%202%206-2V4l-6%202-6-2z'/%3E%3Cpath%20d='M9%204v14M15%206v14'/%3E%3C/svg%3E"); + } + #TABS-main .ant-tabs-tab[data-node-key="TAB-site"] { + --lens-icon: url("data:image/svg+xml,%3Csvg%20xmlns='http://www.w3.org/2000/svg'%20viewBox='0%200%2024%2024'%20fill='none'%20stroke='black'%20stroke-width='2'%20stroke-linecap='round'%20stroke-linejoin='round'%3E%3Cpath%20d='M4%2020V10M10%2020V4M16%2020v-7M22%2020H2'/%3E%3C/svg%3E"); + } + #MARKDOWN-fleet-map > .resizable-container { + height: max(592px, 100vh - 93px) !important; + } + /* A third lens, AI, as a teaser: greyed, tagged PRO, not a tab and not clickable (owner, + 29 Oct; no AI lens is built until Botswerk exists). Drawn by the tab list itself, so the + dashboard's layout stays Map and Insights. From 1100 px only: narrower, the wider switch + would cover the logo. */ + @media (min-width: 1100px) { + #TABS-main > .ant-tabs-nav .ant-tabs-nav-list::before { + content: "AI" / "AI, available with Petrolyst Pro"; + order: 1; + display: flex; + align-items: center; + padding: 7px 58px 7px 40px; + border-radius: 9px; + font-size: 15px; + font-weight: 600; + line-height: 22px; + color: #8a96a3; + cursor: not-allowed; + background: url("data:image/svg+xml,%3Csvg%20xmlns='http://www.w3.org/2000/svg'%20viewBox='0%200%2024%2024'%20fill='none'%20stroke='%238a96a3'%20stroke-width='2'%20stroke-linecap='round'%20stroke-linejoin='round'%3E%3Cpath%20d='M9.937%2015.5A2%202%200%200%200%208.5%2014.063l-6.135-1.582a.5.5%200%200%201%200-.962L8.5%209.936A2%202%200%200%200%209.937%208.5l1.582-6.135a.5.5%200%200%201%20.963%200L14.063%208.5A2%202%200%200%200%2015.5%209.937l6.135%201.581a.5.5%200%200%201%200%20.964L15.5%2014.063a2%202%200%200%200-1.437%201.437l-1.582%206.135a.5.5%200%200%201-.963%200z'/%3E%3Cpath%20d='M20%203v4M22%205h-4'/%3E%3C/svg%3E") 14px center / 18px 18px no-repeat; + } + #TABS-main > .ant-tabs-nav .ant-tabs-nav-list::after { + content: "PRO" / ""; + position: absolute; + right: 14px; + top: 50%; + transform: translateY(-50%); + padding: 3px 7px; + border-radius: 999px; + background: #0f7b6c; + color: #fff; + font-size: 10px; + font-weight: 700; + letter-spacing: 0.06em; + line-height: 1; + pointer-events: none; + } + } + } +} +@keyframes petrolyst-lens-scroll { + to { + transform: translate(-50%, -61px); + } +} diff --git a/petrolyst-insights/dashboards/README.md b/petrolyst-insights/dashboards/README.md index 1472731..9487cdb 100644 --- a/petrolyst-insights/dashboards/README.md +++ b/petrolyst-insights/dashboards/README.md @@ -1,8 +1,52 @@ # dashboards/ -Exported Superset dashboards and the operator scripts that import them. Not baked into the -image, so changes here do not rebuild or restart Superset. Empty until the first fuel -dashboards exist. +Superset dashboards as code and the operator scripts that import them. Not baked into the +image, so changes here do not rebuild or restart Superset. + +- `atoma/`: the ATOMA dashboard, with a Map tab (the fleet map) and an Insights tab (one site's + details), shown to readers as a lens switch in the navbar. `SPEC.md` is + the design; `build_bundle.py` generates the import bundle and `import_bundle.py` imports it + (overwriting the objects the bundle defines). `dashboard.css` is the dashboard's own CSS, + which the bundle carries. + + The Fleet tab is an iframe of the fleet-map view from the petrolyst-platform repository. It + needs that view and its read service served on Superset's own origin under `/views`, and the + image's `superset_config.py` with the iframe setting (SPEC.md §6). Without them the tab stays + empty, and the Site details tab works as before. + + Build and import: + + docker cp atoma/import_bundle.py fuel-monitoring-superset-1:/tmp/ + docker exec fuel-monitoring-superset-1 /app/.venv/bin/python /tmp/import_bundle.py --show-database + python3 atoma/build_bundle.py --database-uuid --out atoma-dashboards.zip + docker cp atoma-dashboards.zip fuel-monitoring-superset-1:/tmp/ + docker exec fuel-monitoring-superset-1 /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip --user + + Exit status 2 means a dataset's columns differ from its view: regenerate `atoma/columns.tsv` + (command in its header), rebuild and import again. + + The import attaches the role "ATOMA Reader" to the dashboard (a bundle cannot carry roles), + so the role must exist first: `superset-init` creates it (`petrolyst-insights/roles/`). The + Fleet tab shows ATOMA's logo from `petrolyst-insights/assets/branding/atoma_logo.svg`, in the + image. + + An import never deletes. On an instance that still has the two dashboards from before + 28 Oct 2026 ("ATOMA - Sites" and "ATOMA - Site Detail") and their charts M1, M3, M4 and M5, + delete those by hand after importing. + + The bundle also carries the Petrolyst app's dashboards (PD-12, SPEC.md §10): `atoma-overview` + (fleet totals) and `atoma-site` (one site, its four sections as tabs; it replaced + `atoma-site-summary`, `-fuel`, `-runtime` and `-baseline` on 29 Oct: delete those by hand + where they exist). + They get no role; the app shows them with guest passes (see "Embedding" in the parent + README; local stack only). Add `--embedded-out FILE --embed-domain ` to the + import to make them embeddable from the app's origin; FILE then receives + `{slug: embedding id}` for the app service. The ids stay the same across imports. Local + stack: + + docker exec fmdb-scratch-superset /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip \ + --user admin --embedded-out /tmp/embedded-dashboards.json --embed-domain http://127.0.0.1:8098 + docker cp fmdb-scratch-superset:/tmp/embedded-dashboards.json /.local/fmdb-scratch/ Export from a running instance (inside the container, as the server's deploy user): diff --git a/petrolyst-insights/dashboards/atoma/SPEC.md b/petrolyst-insights/dashboards/atoma/SPEC.md index 30656a1..464772c 100644 --- a/petrolyst-insights/dashboards/atoma/SPEC.md +++ b/petrolyst-insights/dashboards/atoma/SPEC.md @@ -1,8 +1,13 @@ -# ATOMA dashboards: Sites Map and Site Detail +# ATOMA dashboard: Fleet map and Site details -**Status: proposed, not built.** -- `build_bundle.py` (phase P2) will generate both dashboards, every dataset and every chart from - this specification, as one Superset import bundle. +**Status: built and tested on a local Superset (P2, 24 Oct 2026); merged into one dashboard with +the fleet-map view on 28 Oct 2026 (§9); not imported live.** +- `build_bundle.py` generates the dashboard, every dataset and every chart from this + specification, as one Superset import bundle; `import_bundle.py` imports it (§6). +- The local tests changed parts of this design. Each change is marked *(P2)* and §8 lists them. +- The merge of 28 Oct (owner's decisions PD-5 and PD-6, `Petrolist/.context/design-decisions.md`) + replaced the Sites Map dashboard by a Fleet tab holding the fleet-map view of petrolyst-platform. + Changed parts are marked *(28 Oct)*; §9 lists the results. - Nothing is imported into the live Superset without the owner's go-ahead (phase P3). **Datasets.** The `public.v_*` views of @@ -15,72 +20,104 @@ - the site-specific facts left out here (positions, distances, pilot sites) are there too; - Q-numbers refer to its §7, and phases P0–P5 to its §8. -Two dashboards replace the earlier tab-per-site design; nothing in them is specific to a site. +One dashboard (two until 28 Oct) replaces the earlier tab-per-site design; nothing in it is +specific to a site. ## 1. Capabilities of the installed Superset 6.1.0 Source: read-only inspection of `config.py`, `superset_config.py` and the frontend bundle on -24 Oct 2026. Items marked *(local test)* are confirmed in P2 before the design relies on them. +24 Oct 2026. The former *(local test)* items were tested in P2 (§8); rows marked *(P2)* changed as a result. | Requirement | What 6.1.0 provides | Choice | |---|---|---| | Map chart | `deck_scatter` (deck.gl Scatterplot): a modern plugin with the InteractiveChart behaviour. Also: deck_multi, deck_screengrid / hex / grid, country_map, world_map, cartodiagram, legacy `mapbox` | **deck.gl Scatterplot** | | Basemap | `DECKGL_BASE_MAP` unset → OSM raster (`tile.openstreetmap.org`), OSM Swiss topo, Mapbox styles. `MAPBOX_API_KEY` is empty, so the Mapbox styles fail. CSP `connect-src` already allows the OSM host; Traefik sets no CSP | set `DECKGL_BASE_MAP` to **OSM only**; "© OpenStreetMap contributors" in a Markdown under the map | | OSM terms | no key, no cost; tile usage policy: light use, attribution, no bulk download, a valid Referer (ours sends the origin) | fine for an internal user base; viewers' browsers reveal the viewed area (tile coordinates) and their IP to OSMF, not our data. Self-hosted tiles later if needed | -| Tooltip | "Tooltip contents" (drag columns) plus "Customize tooltips template" (Handlebars). `js_tooltip` exists but `ENABLE_JAVASCRIPT_CONTROLS` is False | contents and template; **no JavaScript** | -| Marker colour | "Categorical color" by a dimension, from the colour scheme; a fixed colour per label through the dashboard's `label_colors` *(local test)* | colour by `baseline_status_label`. Fallback: `deck_multi` with one fixed-colour scatter layer per status | -| Marker size | fixed radius, with minimum = maximum pixel radius | uniform | +| Tooltip | "Tooltip contents" (drag columns) plus "Customize tooltips template" (Handlebars). `js_tooltip` exists but `ENABLE_JAVASCRIPT_CONTROLS` is False. The contents show column names, not labels, and leave out the colour column | **contents only** *(P2)*: short tooltip-only calculated columns (`code`, `name`, `l_per_h`, `method`, `status`, `validity`, `latest_record`); no template, no JavaScript | +| Marker colour | "Categorical color" by a dimension, from the colour scheme; a fixed colour per label through the dashboard's `label_colors` (confirmed in P2) | colour by `baseline_status_label` | +| Marker size | fixed radius, with minimum = maximum pixel radius. No outline: the layer is drawn with `stroked: false` | uniform, no outline *(P2)* | +| Text cards | The Handlebars chart compiles its template with `eval`; Superset's own CSP (`script-src 'self' 'strict-dynamic'` + nonce) blocks that, so the chart shows a CSP error instead *(P2)* | **no Handlebars**: one-row tables. The CSP stays as it is | +| Chart descriptions | "Show description" (`expanded_slices`) never renders in 6.1: the dashboard loads `description_markeddown`, the chart holder reads `description_markdown` *(P2)* | descriptions stay on the charts; the visible reading notes are a Markdown block at the top of each tab | | Viewport | `autozoom` fits the filtered points | on | | Clustering | none in deck.gl; only the legacy `mapbox` chart (`clustering_radius`, needs a Mapbox key) | **no clustering** (§3) | | Marker click | emits a **cross-filter** on the latitude and longitude columns: the exact point, or a box around the click that shrinks as you zoom in. It applies to other charts on the same dashboard | cross-filters the Sites list | | Navigation | **no chart-to-dashboard navigation.** A dashboard reads `?native_filters=` into preselected native filter values (also `native_filters_key`). Table cells render sanitized HTML (`allow_render_html`) | an "Open site detail" link column in the Sites list | | Native filters | Select (single or multi, required, searchable, "first value by default"), Time range; scoping per chart and tab; cross-filter scoping per chart | §4, §5 | +| Markdown iframe *(28 Oct)* | `HTML_SANITIZATION` is on and strips an iframe from a Markdown block; `HTML_SANITIZATION_SCHEMA_EXTENSIONS` lets named tags and attributes through. The CSP (`default-src 'self'`, no `frame-src`) allows frames from this origin only | allow `iframe` with `src`, `title`, `loading`, `style`; the fleet-map view is served on the same origin under `/views` (§3) | +| Deep link into a tab *(28 Oct)* | `#TAB-` after the URL opens that tab; it combines with `?native_filters=` (tested 28 Oct) | the "Site details" button opens the Site details tab with the site preset | + +*(28 Oct)* The map moved out of Superset into the fleet-map view (§3). The rows Map chart, +Basemap, Tooltip, Marker colour, Marker size, Viewport, Clustering and Marker click describe the +deck.gl option the dashboard used until then; the view's choices are in §3. The Navigation row +still holds: the view builds the same `?native_filters=` link. No custom frontend and no patch are needed. ## 2. Navigation flow -``` -ATOMA — Sites (landing, slug atoma-sites) - filters: Site search · Zone · Province · Pilot status (default: active) · Baseline status - M1 map: markers fitted to the filtered sites ──hover──► tooltip - └─click marker──► cross-filter: M3 Sites list shows that site (or the few under the click) - M3 Sites list ─────────────"Open site detail"──► ATOMA — Site Detail ?native_filters=(site preset) - M4/M5 Sites without an accepted location ──────"Open site detail"──► same -ATOMA — Site Detail (slug atoma-site-detail) - filters: Site (exactly one, searchable) · Viewing period - tabs: Summary · Fuel & Deliveries · Runtime & Electricity · Baseline & Evidence - header link "← Sites map" -``` - -## 3. Sites Map (landing dashboard) +*(28 Oct)* One dashboard, two top-level tabs. *(29 Oct)* They are named **Map** (`TAB-fleet`) and +**Insights** (`TAB-site`), the dashboard's lenses; earlier text in this document calls them Fleet and +Site details. Readers see them as a lens switch, centred in the navbar (§6). ``` -[Filter bar: Site search · Zone · Province · Pilot status · Baseline status] -[M1 Map (deck.gl Scatterplot) ................................ w8][M2 Status key + attribution w4] -[M3 Sites list (table with link column) ..................................................... w12] -[M4 Sites without accepted location: count w3][M5 Sites without accepted location (table) .. w9] +ATOMA (slug atoma) + filters, in Superset's left column: Site (exactly one, searchable) · Viewing period + — both out of scope on Map; readers have no column there (§6) + [Map] the fleet-map view (iframe, /views/fleet-map/), the page's full width + towers fitted to the placed sites ──hover or click──► popup + "Find a site" (code or name) ──► the map flies to it ──► its popup + └─"Site details"──► /superset/dashboard/atoma/?native_filters=(site preset)#TAB-site + sites without an accepted location: counted in the map's key ("Choose them in Insights", + the view's tab_label parameter); chosen in the Site filter + [Insights] D0 site header, then the tabs + Summary · Fuel & Deliveries · Runtime & Electricity · Baseline & Evidence + (a row of section buttons, not a second row of tabs: dashboard.css) ``` -| # | Question | Type | Dataset | Content | Missing or insufficient | -|---|---|---|---|---|---| -| M1 | Where are the sites, and what is each one's baseline status? | deck.gl Scatterplot; lat/lon from `latitude`, `longitude`; uniform size; colour = `baseline_status_label`; autozoom | v_site | tooltip: `site_code`, `site_name`, `lph_estimate` ("not available" when NULL), `lph_method`, `baseline_status_label`, `baseline_validity_note` (when set), `latest_record_on` | sites without an accepted location have NULL lat/lon and are not plotted; M4/M5 show them | -| M2 | What do the colours mean? | Markdown (static) | — | the status definitions below; "Colours show evidence status, not performance"; OSM attribution | — | -| M3 | Which sites match, and how do I open one? | Table; search box; server pagination | v_site | code, name, province, pilot status, baseline status, L/h (+ method), L/day, latest record, location status, `detail_link` | marker click narrows it; "not available" shown as text | -| M4 | How many sites lack an accepted location? | Big Number Total, COUNT(*) with saved filter `location_status <> 'accepted'` | v_site | count; follows the dashboard filters | 0 is shown as 0 | -| M5 | Which sites are they? | Table; search box | v_site, same saved filter | code, name, province, `location_status` (awaiting acceptance · missing), baseline status, `detail_link` | — | +The filters sit in Superset's left column (`filter_bar_orientation: VERTICAL`) *(owner, +29 Oct; `build_bundle.py` checks it)*. A filter shows in the column on the tabs where it is in +scope; the others are listed under "Filters out of scope" (the Viewing period on Summary, for +example). No filter applies to the Fleet tab, so readers have no column there and the map takes +the full width (`reader.css`, §6); admins keep the column on both tabs. The column opens +collapsed, a 32 px strip with an expand button (`FILTERBAR_CLOSED_BY_DEFAULT`, +`superset_config.py`; owner, 29 Oct); the selected site shows in the Site table above the +sections. So for a reader the tabs move sideways by 32 px between Fleet and Site details, or by +the column's width once it is expanded; the owner accepted that. -**Filters** (all on `v_site`, scoped to M1, M3, M4 and M5): -- Site search: Select on `site_label`, multi, searchable. The label holds code, name and uid, so - code or name both match. -- Zone, Province: Select. -- Pilot status: default `active`. -- Baseline status: Select on `baseline_status_label`. +From 28 to 29 Oct the filters sat in a row above the tabs (`HORIZONTAL`). On the Fleet tab that +row held only an empty "More filters", and hiding it there made the tabs jump down on Site +details. Superset 6.1 has no per-tab setting for the filter bar; it hides it only per link +(`?show_filters=0`). -The cross-filter from M1 is scoped **to M3 only**. M4 and M5 hold the sites without coordinates and -must not be emptied by it. +Until 28 Oct there were two dashboards: "ATOMA - Sites" (slug `atoma-sites`: a deck.gl map, five +filters, a sites list) and "ATOMA - Site Detail" (slug `atoma-site-detail`), linked by "Open site +detail" and "← Sites map". -**Baseline status and marker colours.** Colours are neutral categorical hues, with no red or green +## 3. Fleet tab: the fleet-map view *(28 Oct)* + +The Fleet tab holds one Markdown block with an iframe of the fleet-map view, a page of +petrolyst-platform (`views/src/fleet-map/`) served under `/views/` on Superset's own origin. The +view reads the read service's `site_status` operation (`/views/api/v1/site-status`, from +`public.v_site`). It is not a Superset chart: it has no dataset and no native filter, and both +filters are out of scope on this tab. + +``` +[Fleet tab: MARKDOWN-fleet-map, w12 h100] + ') + +# SPEC §3: neutral hues, no red or green, keyed by v_site.baseline_status_label. The +# fleet-map view uses the same colours (petrolyst-platform views/src/shared/status.ts). +STATUS_COLOURS = { + "Approved baseline": "#009688", + "Eligible - not yet approved": "#3987e5", + "Provisional": "#c98500", + "Insufficient evidence": "#64748B", +} + +# SPEC §4 F3: a stock interval that is not "ok" is greyed; its status names the reason. +EXCLUDED_GREY = "#A3A9B5" +STOCK_STATUS_COLOURS = {status: EXCLUDED_GREY for status in ( + "no_opening_or_closing", "endpoint_movement_order_unknown", "unquantified_movement", + "same_day", "negative")} + +# Datasets with a time axis: the column the Viewing period applies to. +MAIN_DTTM = { + "v_delivery": "delivery_date", + "v_stock_reading": "observed_on", + "v_interval": "start_on", + "v_runtime": "start_on", + "v_hour_meter_evidence": "reading_date", + "v_electricity_cost": "bill_month", +} + +# Calculated columns: (name, Trino SQL, type, label). The fleet-map view's read service +# (petrolyst-platform services/read, site_status) shows the same two texts. +CALCULATED = { + "v_site": [ + ("lph_text", "COALESCE(CAST(lph_estimate AS varchar), 'not available')", "VARCHAR", "L/h"), + # SPEC §4: "Eligible" and "Provisional" carry "not approved". + ("baseline_status_text", "CASE WHEN baseline_status = 'provisional' THEN baseline_status_label" + " || ' (not approved)' ELSE baseline_status_label END", "VARCHAR", "Baseline status"), + ], + # PD-12 Overview: money is grouped by currency; a missing one is its own row. + "v_delivery": [ + ("currency_text", "COALESCE(currency, 'currency not recorded')", "VARCHAR", "Currency"), + ], + "v_electricity_cost": [ + ("currency_text", "COALESCE(currency, 'currency not recorded')", "VARCHAR", "Currency"), + ("currency_basis_text", "COALESCE(currency_basis, 'basis not recorded')", "VARCHAR", + "Currency basis"), + ], + "v_stock_reading": [ + ("reading_series", "reading_type || ', ' || value_kind || CASE WHEN seq_in_day > 1" + " THEN ' (visit ' || CAST(seq_in_day AS varchar) || ')' ELSE '' END", "VARCHAR", "Reading"), + ], +} + +# Virtual datasets: name -> (view it extends, extra output columns, Trino SQL). +# coverage_detail feeds S3. The view's lane names start with a sort number ("1 Stock +# intervals"), which reads as a count on the chart, so it adds lane_name without the number +# (the chart still sorts by lane). Its amount column is the tooltip: the litres delivered in a +# stock interval (from its opening visit up to, not including, its closing visit; the opening +# delivery is what the interval then uses), or a bill's amount. +# +# v_coverage carries dates, not visits or bills, so an amount is withheld wherever dates cannot +# place it (Codex inspection round 3, ATOMA-08/09): a delivery on a boundary day that has more +# than one visit could belong to either neighbouring interval, and bills that start on the same +# day cannot be told apart. Everywhere else the date window is exact: on a single-visit day the +# opening visit's delivery is included and the closing visit's excluded. +VIRTUAL = { + "coverage_detail": ("v_coverage", [("lane_name", "varchar(60)"), ("amount", "varchar(120)")], """\ +SELECT c.site_uid, c.site_code, c.site_name, c.site_label, c.lane, c.status, + c.start_on, c.end_on, c.start_ts, c.end_ts, + regexp_replace(c.lane, '^[0-9]+ ', '') AS lane_name, + CASE WHEN c.lane = '1 Stock intervals' THEN + CASE WHEN f.ambiguous THEN 'not attributed: a delivery on a day with several visits (see Deliveries)' + WHEN f.filled_l IS NULL AND coalesce(f.n_unknown, 0) = 0 THEN 'no delivery' + WHEN f.filled_l IS NULL THEN 'filled, quantity not recorded' + ELSE format('%,.0f L filled', CAST(f.filled_l AS double)) END + || CASE WHEN f.n_unknown > 0 AND f.filled_l IS NOT NULL AND NOT f.ambiguous + THEN ', plus ' || CAST(f.n_unknown AS varchar) || ' of unknown quantity' ELSE '' END + WHEN c.lane = '4 Electricity bills' THEN + CASE WHEN b.n_bills > 1 THEN 'several bills start this day (see Electricity bills)' + WHEN b.disputed THEN 'bill disputed, amount withheld' + WHEN b.amount IS NULL THEN 'bill amount not recorded' + ELSE format('%,.0f AFN billed (inferred)', CAST(b.amount AS double)) END + END AS amount +FROM public.v_coverage c +LEFT JOIN (SELECT s.site_uid, s.start_on, s.end_on, + sum(CASE WHEN d.delivery_date < s.end_on THEN d.delivered_l END) AS filled_l, + count_if(d.delivery_date < s.end_on AND d.delivered_l IS NULL) AS n_unknown, + bool_or(d.delivery_date IN (s.start_on, s.end_on) AND m.site_uid IS NOT NULL) AS ambiguous + FROM (SELECT DISTINCT site_uid, start_on, end_on FROM public.v_coverage + WHERE lane = '1 Stock intervals') s + JOIN public.v_delivery d ON d.site_uid = s.site_uid + AND d.delivery_date >= s.start_on AND d.delivery_date <= s.end_on + LEFT JOIN (SELECT site_uid, day + FROM (SELECT site_uid, delivery_date AS day, seq_in_day FROM public.v_delivery + UNION ALL + SELECT site_uid, observed_on, seq_in_day FROM public.v_stock_reading) x + GROUP BY site_uid, day HAVING max(seq_in_day) > 1) m + ON m.site_uid = d.site_uid AND m.day = d.delivery_date + GROUP BY s.site_uid, s.start_on, s.end_on) f + ON c.lane = '1 Stock intervals' AND f.site_uid = c.site_uid + AND f.start_on = c.start_on AND f.end_on = c.end_on +LEFT JOIN (SELECT site_uid, coalesce(period_start, bill_month) AS start_on, sum(amount) AS amount, + bool_or(claim_status = 'disputed') AS disputed, count(*) AS n_bills + FROM public.v_electricity_cost GROUP BY 1, 2) b + ON c.lane = '4 Electricity bills' AND b.site_uid = c.site_uid AND b.start_on = c.start_on +"""), +} + +# Saved metrics: (name, SQL, label). Every dataset also gets count. +METRICS = { + "v_delivery": [("litres_delivered", "SUM(delivered_l)", "Litres delivered")], + "v_electricity_cost": [("amount", "SUM(amount)", "Amount (AFN, inferred)")], + # SPEC §5: one eligibility filter on numerator and denominator. Group or filter + # by metric and population_key wherever this is used. + "v_interval": [( + "pooled_l_per_h", + "SUM(CASE WHEN eligibility <> 'excluded' THEN fuel_used_l END)" + " / NULLIF(SUM(CASE WHEN eligibility <> 'excluded' THEN runtime_h END), 0)", + "Pooled L/h (usable intervals)", + )], +} + +LABELS = { + "site_code": "Code", "site_name": "Site name", "site_label": "Site", "site_uid": "Site uid", + "pilot_status": "Pilot status", "baseline_status_label": "Baseline status", + "baseline_status": "Baseline status code", "baseline_validity_note": "Validity note", + "lph_estimate": "L/h", "lph_method": "L/h method", "lph_window_start": "L/h window start", + "lph_window_end": "L/h window end", "baseline_decision_ref": "Decision", + "fuel_l_per_calendar_day": "L/day, whole record", "latest_record_on": "Latest record", + "location_status": "Location status", "location_precision_m": "Location precision (m)", + "delivered_l": "Delivered (L)", "fuel_l_per_day": "L per calendar day", + "l_per_h": "L/h", "l_per_h_low": "L/h low", "l_per_h_high": "L/h high", + "current_baseline_l_per_h": "Approved baseline L/h", "fuel_used_l": "Used (L)", + "fuel_used_unc_l": "Used, uncertainty (L)", "opening_l": "Opening (L)", "closing_l": "Closing (L)", + "deliveries_l": "Deliveries (L)", "withdrawals_l": "Withdrawals (L)", "runtime_h": "Runtime (h)", + "runtime_h_per_day": "Runtime (h/day)", "h_per_day": "h/day", "h_per_day_low": "h/day low", + "h_per_day_high": "h/day high", "l_per_day": "L/day", "modelled_l_per_h": "Modelled L/h", + "modelled_l_per_h_low": "Modelled L/h low", "modelled_l_per_h_high": "Modelled L/h high", + "stock_balance_l": "Stock depletion, site (L)", "engine_l": "Engine-measured, this generator (L)", + "engine_all_generators_l": "Engine-measured, all generators (L)", + "all_generators_metered": "All generators metered", + "covered_h": "Hours covered", "interval_h": "Interval hours", "difference_l": "Difference, site (L)", + "difference_pct": "Difference, site (%)", "fuel_l": "Fuel (L)", "kwh": "kWh", + "amount": "Amount", "value_kind": "Value kind", "claim_status": "Claim status", + "interval_label": "Interval", "exclusion_reasons": "Exclusion reasons", + "in_current_baseline": "In current baseline", "cached_value": "Workbook value", + "unit_as_labelled": "Unit (as labelled)", "source_label": "Workbook label", + "our_reading": "Our reading", "baseline_use": "Use", "make_model": "Make and model", + "load_pct": "Load (%)", "value_text": "Value (text)", "assumption_code": "Assumption", + "matched_days": "Matched days", "document_date": "Date", "used_for": "Used for", + "check_code": "Check code", "label": "Check", "measure": "Measure", "lane_name": "Evidence", + "seq_in_day": "Visit of the day", "event_key": "Event", "n_sources": "Sources", + "sources_disagree": "Sources disagree", "delivery_date": "Date", +} + + +def label_for(column: str) -> str: + return LABELS.get(column) or column.replace("_", " ").capitalize() + + +def uid(kind: str, key: str) -> str: + return str(uuid.uuid5(NAMESPACE, f"{kind}/{key}")) + + +# --- Datasets ------------------------------------------------------------------- + +def read_columns(path: Path) -> dict[str, list[tuple[str, str]]]: + views: dict[str, list[tuple[str, str]]] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip() or line.startswith("#"): + continue + view, column, data_type = line.split("\t") + views.setdefault(view, []).append((column, data_type)) + return views + + +def superset_type(trino_type: str) -> str: + return trino_type.upper().replace(",", ", ") + + +def dataset_yaml(view: str, columns: list[tuple[str, str]], database_uuid: str) -> dict[str, Any]: + cols = [] + for name, data_type in columns: + temporal = data_type.startswith(("date", "timestamp")) + cols.append(column_entry(name, superset_type(data_type), temporal, None)) + for name, expression, data_type, verbose in CALCULATED.get(view, []): + cols.append(column_entry(name, data_type, False, expression, verbose)) + metrics = [metric_entry("count", "COUNT(*)", "Count")] + metrics += [metric_entry(*m) for m in METRICS.get(view, [])] + return { + "table_name": view, + "main_dttm_col": MAIN_DTTM.get(view), + "currency_code_column": None, + "description": None, + "default_endpoint": None, + "offset": 0, + "cache_timeout": None, + "catalog": CATALOG, + "schema": SCHEMA, + "sql": VIRTUAL[view][2] if view in VIRTUAL else None, + "params": None, + "template_params": None, + "filter_select_enabled": True, + "fetch_values_predicate": None, + "extra": None, + "normalize_columns": False, + "always_filter_main_dttm": False, + "folders": None, + "uuid": uid("dataset", view), + "metrics": metrics, + "columns": cols, + "version": VERSION, + "database_uuid": database_uuid, + } + + +def column_entry(name: str, data_type: str, temporal: bool, expression: str | None, + verbose: str | None = None) -> dict[str, Any]: + return { + "column_name": name, + "verbose_name": verbose or label_for(name), + "is_dttm": temporal, + "is_active": True, + "type": data_type, + "advanced_data_type": None, + "groupby": True, + "filterable": True, + "expression": expression, + "description": None, + "python_date_format": None, + "datetime_format": None, + "extra": None, + } + + +def metric_entry(name: str, expression: str, verbose: str) -> dict[str, Any]: + return { + "metric_name": name, + "verbose_name": verbose, + "metric_type": None, + "expression": expression, + "description": None, + "d3format": None, + "currency": None, + "extra": None, + "warning_text": None, + } + + +# --- Chart building blocks ---------------------------------------------------------- + +def sql_metric(expression: str, label: str) -> dict[str, Any]: + return { + "expressionType": "SQL", + "sqlExpression": expression, + "label": label, + "hasCustomLabel": True, + "optionName": "metric_" + re.sub(r"\W+", "_", label.lower()).strip("_"), + "column": None, + "aggregate": None, + "datasourceWarning": False, + } + + +def sql_filter(expression: str) -> dict[str, Any]: + return { + "expressionType": "SQL", + "sqlExpression": expression, + "clause": "WHERE", + "subject": None, + "operator": None, + "comparator": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": "filter_" + uuid.uuid5(NAMESPACE, expression).hex[:12], + } + + +def time_filter(column: str) -> dict[str, Any]: + """An open TEMPORAL_RANGE filter: the Viewing period replaces its range.""" + return { + "expressionType": "SIMPLE", + "subject": column, + "operator": "TEMPORAL_RANGE", + "comparator": "No filter", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": f"filter_period_{column}", + } + + +def order(*columns: str, ascending: bool = True) -> list[str]: + return [json.dumps([c, ascending]) for c in columns] + + +class Chart: + """One chart: key (SPEC id), title, dataset, viz type, params, description.""" + + def __init__(self, key: str, title: str, view: str, viz: str, params: dict[str, Any], + description: str | None = None, columns_used: list[str] | None = None) -> None: + self.key, self.title, self.view, self.viz = key, title, view, viz + self.params = {"viz_type": viz, "datasource": "0__table", **params} + self.description = description + self.columns_used = columns_used or [] + + @property + def uuid(self) -> str: + return uid("chart", self.key) + + def yaml(self, dataset_uuid: str) -> dict[str, Any]: + return { + "slice_name": self.title, + "description": self.description, + "certified_by": None, + "certification_details": None, + "viz_type": self.viz, + "params": self.params, + "query_context": None, + "cache_timeout": None, + "uuid": self.uuid, + "version": VERSION, + "dataset_uuid": dataset_uuid, + } + + +def table(key: str, title: str, view: str, columns: list[str], *, order_by: list[str] | None = None, + filters: list[dict[str, Any]] | None = None, period: str | None = None, + server_pages: bool = False, search: bool = False, html: bool = False, + description: str | None = None) -> Chart: + adhoc = list(filters or []) + if period: + adhoc.append(time_filter(period)) + params = { + "query_mode": "raw", + "all_columns": columns, + "groupby": [], + "metrics": [], + "percent_metrics": [], + "order_by_cols": order_by or [], + "adhoc_filters": adhoc, + "row_limit": 10000, + "server_pagination": server_pages, + "server_page_length": 25, + "page_length": 25, + "include_search": search, + "show_cell_bars": False, + "color_pn": False, + "allow_render_html": html, + "table_timestamp_format": "%Y-%m-%d", + "column_config": {}, + "conditional_formatting": [], + } + used = columns + [json.loads(o)[0] for o in order_by or []] + ([period] if period else []) + return Chart(key, title, view, "table", params, description, used) + + +def bars(key: str, title: str, view: str, x: str, metrics: list[Any], *, groupby: list[str] | None = None, + grain: str | None = None, period: str | None = None, y_title: str = "", + filters: list[dict[str, Any]] | None = None, description: str | None = None) -> Chart: + adhoc = list(filters or []) + if period: + adhoc.append(time_filter(period)) + params = { + **timeseries_common(x, metrics, groupby or [], grain, adhoc, y_title), + "orientation": "vertical", + "stack": "Stack" if groupby else None, + "only_total": True, + "show_value": False, + } + used = [x] + list(groupby or []) + ([period] if period else []) + return Chart(key, title, view, "echarts_timeseries_bar", params, description, used) + + +def summary_table(key: str, title: str, view: str, groupby: list[str], metrics: list[Any], *, + period: str | None = None, description: str | None = None) -> Chart: + """An aggregated table: one row per group, one column per metric.""" + params = { + "query_mode": "aggregate", + "groupby": groupby, + "metrics": metrics, + "all_columns": [], + "percent_metrics": [], + "order_by_cols": [], + "order_desc": True, + "adhoc_filters": [time_filter(period)] if period else [], + "row_limit": 1000, + "server_pagination": False, + "include_search": False, + "show_cell_bars": False, + "color_pn": False, + "allow_render_html": False, + "table_timestamp_format": "%Y-%m-%d", + "column_config": {}, + "conditional_formatting": [], + } + return Chart(key, title, view, "table", params, description, groupby + ([period] if period else [])) + + +def big_number(key: str, title: str, view: str, metric: dict[str, Any], *, + period: str | None = None, subheader: str = "", + description: str | None = None) -> Chart: + params = { + "metric": metric, + "adhoc_filters": [time_filter(period)] if period else [], + "subheader": subheader, + "header_font_size": 0.4, + "subheader_font_size": 0.15, + "y_axis_format": "SMART_NUMBER", + "time_format": "smart_date", + "force_timestamp_formatting": False, + } + return Chart(key, title, view, "big_number_total", params, description, + [period] if period else []) + + +def timeseries_common(x: str, metrics: list[Any], groupby: list[str], grain: str | None, + adhoc: list[dict[str, Any]], y_title: str) -> dict[str, Any]: + return { + "x_axis": x, + "time_grain_sqla": grain, + "metrics": metrics, + "groupby": groupby, + "adhoc_filters": adhoc, + "row_limit": 10000, + "truncate_metric": True, + "show_empty_columns": True, + "comparison_type": "values", + "annotation_layers": [], + "color_scheme": "supersetColors", + "show_legend": bool(groupby), + "legendType": "scroll", + "legendOrientation": "top", + "x_axis_title": "", + "x_axis_title_margin": 15, + "y_axis_title": y_title, + "y_axis_title_margin": 40, + "y_axis_title_position": "Left", + "x_axis_time_format": "%Y-%m-%d", + "y_axis_format": "SMART_NUMBER", + "rich_tooltip": True, + "tooltipTimeFormat": "%Y-%m-%d", + "truncateXAxis": True, + "y_axis_bounds": [None, None], + "zoomable": False, + } + + +# --- Overview (PD-12, embedded in the Petrolyst app) --------------------------------------- + +OVERVIEW_SLUG = "atoma-overview" +SITE_SLUG = "atoma-site" # one embedded dashboard with the DETAIL_TABS as its tabs + +# SPEC §10: only sums and counts of recorded values, never an average of per-site rates; money +# only grouped by currency. The value says whether the metric is money. +OVERVIEW_METRICS = { + "COUNT(*)": False, + "SUM(delivered_l)": False, + "COUNT(unit_price)": False, + "SUM(delivered_l * unit_price)": True, + "SUM(CASE WHEN COALESCE(claim_status, '') <> 'disputed' THEN amount END)": True, + "SUM(CASE WHEN claim_status = 'disputed' THEN amount END)": True, +} + + +def overview_charts() -> list[Chart]: + def m(expression: str, label: str) -> dict[str, Any]: + assert expression in OVERVIEW_METRICS, expression + return sql_metric(expression, label) + + return [ + big_number("O3", "Litres delivered", "v_delivery", m("SUM(delivered_l)", "Litres delivered"), + period="delivery_date", subheader="all sites, in the viewing period", + description="Recorded deliveries. Deliveries are not consumption."), + summary_table("O1", "Sites by pilot status", "v_site", ["pilot_status"], + [m("COUNT(*)", "Sites")]), + summary_table("O2", "Sites by baseline status", "v_site", ["baseline_status_label"], + [m("COUNT(*)", "Sites")]), + summary_table("O4", "Fuel cost by currency", "v_delivery", ["currency_text"], + [m("SUM(delivered_l * unit_price)", "Cost (priced deliveries)"), + m("COUNT(unit_price)", "Priced deliveries"), + m("COUNT(*)", "Deliveries")], period="delivery_date", + description="Litres times the recorded unit price, only where a price was " + "recorded. Never added up across currencies."), + summary_table("O5", "Electricity billed by currency", "v_electricity_cost", + ["currency_text", "currency_basis_text"], + [m("SUM(CASE WHEN COALESCE(claim_status, '') <> 'disputed' THEN amount END)", + "Billed"), + m("SUM(CASE WHEN claim_status = 'disputed' THEN amount END)", "Disputed"), + m("COUNT(*)", "Bills")], period="bill_month", + description="Grid cost as billed; disputed bills apart. Never converted to " + "fuel or hours, never added up across currencies."), + bars("O6", "Litres delivered per month", "v_delivery", "delivery_date", + [m("SUM(delivered_l)", "Litres delivered")], grain="P1M", period="delivery_date", + y_title="Litres", description="All sites' recorded deliveries per calendar month."), + ] + + +OVERVIEW_NOTES = ("**Reading this page.** All sites together, from recorded values only: " + "deliveries are not consumption, and site rates (L/h, L/day) are never averaged " + "here; each site's own page has them. Money is shown per currency and never added " + "up across currencies. The viewing period narrows the deliveries, the costs and " + "the monthly bars; the site counts are always the current state.") +OVERVIEW_ROWS = [ + [("MARKDOWN:notes-overview", 12, 14)], + [("O3", 4, 40), ("O1", 4, 40), ("O2", 4, 40)], + [("O4", 6, 40), ("O5", 6, 40)], + [("O6", 12, 60)], +] +OVERVIEW_PERIOD_SCOPE = {"O3", "O4", "O5", "O6"} + + +# --- Site Detail ----------------------------------------------------------------------- + +def detail_charts() -> list[Chart]: + return [ + # Handlebars would render these as cards, but it compiles templates with + # eval, which Superset's own CSP blocks. One-row tables need no CSP change. + table("D0", "Site", "v_site", [ + "site_code", "site_name", "province", "pilot_status", "baseline_status_text", + "location_status"]), + table("S1", "Fuel per running hour", "v_site", [ + "lph_text", "lph_method", "lph_window_start", "lph_window_end", "baseline_status_text", + "baseline_validity_note", "baseline_decision_ref", "baseline_conditions", + "fuel_l_per_calendar_day"], + description="L/h comes only from an approved or proposed baseline version. " + "L/day is per calendar day over the whole record, not per running hour."), + table("S2", "Supporting measures", "v_kpi_measures", [ + "measure", "value", "unit", "value_kind", "window_start", "window_end", "note"], + order_by=order("measure_order")), + Chart("S3", "Evidence coverage", "coverage_detail", "gantt_chart", { + "start_time": "start_ts", + "end_time": "end_ts", + "y_axis": "lane_name", + "series": "status", + "subcategories": False, + "tooltip_metrics": [], + "order_by_cols": order("lane"), + "adhoc_filters": [], + "row_limit": 10000, + "color_scheme": "supersetColors", + "show_legend": True, + "legendType": "scroll", + "legendOrientation": "top", + "zoomable": True, + "x_axis_time_format": "%Y-%m-%d", + "tooltip_time_format": "%Y-%m-%d", + "tooltip_columns": ["amount"], + "y_axis_title": "Evidence", + "y_axis_title_margin": 150, + "y_axis_title_position": "Left", + }, "Periods covered by each kind of evidence. Gaps are not interpolated.", + ["start_ts", "end_ts", "lane", "lane_name", "status", "amount"]), + bars("F1", "Litres delivered per day (all deliveries that day)", "v_delivery", + "delivery_date", ["litres_delivered"], grain="P1D", period="delivery_date", + y_title="Litres", description="Deliveries are not consumption. F5 lists each delivery."), + table("F5", "Deliveries", "v_delivery", [ + "delivery_date", "seq_in_day", "event_key", "delivered_l", "unit_price", "currency", + "n_sources", "sources_disagree", "reviewed", "source_ref"], + order_by=order("delivery_date", "seq_in_day"), period="delivery_date", search=True, + description="Every delivery event, including one whose quantity is not known."), + Chart("F2", "Recorded stock", "v_stock_reading", "echarts_timeseries_scatter", { + # A second visit on the same day is its own series, so no reading is hidden + # behind the day's MAX. + **timeseries_common("observed_on", [sql_metric("MAX(litres)", "Litres")], + ["reading_series"], "P1D", [time_filter("observed_on")], "Litres"), + "markerSize": 6, + }, "Points only; nothing is drawn between readings.", + ["observed_on", "litres", "reading_series", "seq_in_day"]), + bars("F3", "Fuel per calendar day, by stock interval", "v_interval", "interval_label", + [sql_metric("MAX(fuel_l_per_day)", "L per calendar day")], groupby=["stock_status"], + period="start_on", y_title="L/day", + description="One bar per observed stock interval; nothing is spread over months. " + "F4 lists the intervals with their status."), + table("F4", "Stock intervals", "v_interval", [ + "interval_label", "start_on", "end_on", "days", "opening_l", "opening_basis", + "deliveries_l", "withdrawals_l", "closing_l", "closing_basis", "fuel_used_l", + "fuel_l_per_day", "stock_status"], + order_by=order("start_on"), period="start_on"), + bars("R1", "Runtime per day, by meter interval", "v_runtime", "interval_label", + [sql_metric("MAX(h_per_day)", "h/day")], groupby=["meter_label"], + period="start_on", y_title="h/day", + description="Empty until hour-meter readings exist. An invalid reading pair " + "(two epochs, a decreasing timer, over 24 h/day) shows no bar."), + table("R1T", "Runtime pairs and their validity", "v_runtime", [ + "meter_label", "mapped_set", "interval_label", "start_on", "end_on", "days", + "runtime_h", "h_per_day", "pair_valid", "same_epoch", "monotonic", + "over_24h_per_day", "reviewed"], + order_by=order("meter_label", "start_on"), period="start_on", + description="Every pair of hour-meter readings, valid or not. R1 shows no bar " + "for an invalid pair."), + table("R2", "Hour-meter evidence", "v_hour_meter_evidence", [ + "meter_label", "mapped_set", "reading_date", "date_basis", "hours", + "reading_state", "epoch_no", "claim_status"], + order_by=order("reading_date"), period="reading_date", + description="Empty until hour-meter readings exist."), + table("R3", "Grid energy (kWh)", "v_kpi_measures", [ + "measure", "value", "unit", "value_kind", "note"], + filters=[sql_filter("unit = 'kWh'")], + description="Empty value: no kWh records. Bills are cost only and are not " + "converted to fuel or hours."), + bars("R4", "Electricity cost per bill month (AFN, inferred)", "v_electricity_cost", + "bill_month", ["amount"], grain="P1M", period="bill_month", y_title="AFN", + description="Grid cost only; never converted to fuel or hours."), + table("R4T", "Electricity bills", "v_electricity_cost", [ + "bill_month", "period_start", "period_end", "amount", "currency", "currency_basis", + "kwh", "claim_status", "source_ref"], + order_by=order("bill_month"), period="bill_month", + description="A disputed bill is listed without amount or kWh."), + table("B1", "Baseline versions", "v_baseline_version", [ + "code", "metric", "status", "value", "range_low", "range_high", "unit", + "window_start", "window_end", "matched_days", "meets_policy", "policy_check", + "evidence_grade", "conditions", "approved_on", "approved_by_role", "decision_ref"], + order_by=order("window_start"), + description="Empty while no baseline version is defined."), + table("B2", "Inputs of each version", "v_baseline_input", [ + "baseline_code", "interval_label", "included", "exclusion_reason", "fuel_l", + "runtime_h", "days", "eligibility_at_computation", "dependencies_changed"], + order_by=order("baseline_code", "start_on"), + description="Empty until a baseline version exists."), + table("B3", "All intervals: eligibility and exclusions", "v_interval", [ + "interval_label", "eligibility", "exclusion_reasons", "evidence_grade", + "conditions", "metric", "population_key", "in_current_baseline"], + order_by=order("start_on")), + Chart("B4", "Interval L/h against the approved baseline", "v_interval", "mixed_timeseries", { + "x_axis": "interval_label", + "time_grain_sqla": None, + "metrics": [sql_metric("MAX(l_per_h)", "Interval L/h")], + "groupby": [], + "adhoc_filters": [sql_filter("eligibility <> 'excluded'")], + "row_limit": 10000, + "truncate_metric": True, + "comparison_type": "values", + "metrics_b": [sql_metric("MAX(current_baseline_l_per_h)", "Approved baseline L/h")], + "groupby_b": [], + "adhoc_filters_b": [sql_filter("baseline_comparable = true")], + "row_limit_b": 10000, + "truncate_metric_b": True, + "comparison_type_b": "values", + "seriesType": "bar", + "seriesTypeB": "line", + "markerEnabledB": True, + "markerSizeB": 6, + # One y-axis for both (no dual axis). + "yAxisIndex": 0, + "yAxisIndexB": 0, + "annotation_layers": [], + "color_scheme": "supersetColors", + "show_legend": True, + "legendType": "scroll", + "legendOrientation": "top", + "y_axis_format": "SMART_NUMBER", + "rich_tooltip": True, + "x_axis_time_format": "%Y-%m-%d", + "y_axis_title": "L/h", + "y_axis_title_margin": 40, + "y_axis_title_position": "Left", + }, "Bars only for usable intervals; the line only for an approved baseline of the same " + "metric and population. See S1 for the site's status.", + ["interval_label", "l_per_h", "eligibility", "current_baseline_l_per_h", + "baseline_comparable"]), + table("B5", "Assumptions", "v_assumption", [ + "code", "topic", "value", "value_low", "value_high", "unit", "applies_from", + "applies_to", "basis", "derived_from_fuel", "status", "decision_ref", "rationale"], + order_by=order("code")), + table("B6", "Missing data and quality checks", "v_data_quality", [ + "label", "status", "detail"], order_by=order("check_order")), + table("B7", "Evidence references", "v_evidence", [ + "title", "kind", "issuer", "document_date", "used_for", "document_key"], + order_by=order("document_date")), + table("B8", "Workbook calculations, assessed", "v_imported_calculation", [ + "source_label", "formula", "cached_value", "unit_as_labelled", "our_reading", + "baseline_use", "exclusion_reason", "claim_status", "cell_ref"], + order_by=order("cell_ref")), + table("B9", "Stock against engine-measured fuel", "v_reconciliation", [ + "interval_label", "asset_uid", "engine_l", "covered_h", "interval_h", + "placed_throughout", "stock_balance_l", "engine_all_generators_l", + "all_generators_metered", "difference_l", "difference_pct", "stock_status", + "endpoint_times_known"], + order_by=order("start_on"), + description="Empty until flow-meter data exist. The difference is shown only " + "with full coverage and known visit times."), + table("B10", "Equipment specifications: manufacturer and nameplate data, not site performance", + "v_spec_reference", [ + "slot", "make_model", "attribute", "load_pct", "value", "value_text", "unit", + "basis", "applicability", "applicability_note", "claim_status"], + order_by=order("slot", "attribute"), + description="Manufacturer and nameplate data, not site performance."), + table("B11", "Modelled L/h under an assumed runtime: not measured", "v_modelled_lph", [ + "assumption_code", "status", "basis", "h_per_day", "h_per_day_low", + "h_per_day_high", "window_start", "window_end", "matched_days", "l_per_day", + "modelled_l_per_h", "modelled_l_per_h_low", "modelled_l_per_h_high", "value_kind"], + order_by=order("assumption_code"), + description="Modelled from an assumed runtime, not measured. It never reaches " + "S1, the map or the sites list."), + ] + + +# Reading notes shown at the top of each tab (SPEC §4 "when data are missing"). +TAB_NOTES = { + "summary": "**Reading this tab.** L/h comes only from a baseline version; while no runtime " + "evidence exists it is *not available*. L/day is per calendar day over the whole " + "record, not per running hour. The coverage chart has up to four lanes: Stock " + "intervals, Runtime (one per meter), Baseline window, Electricity bills. A " + "lane missing from the chart means no coverage span could be formed: no evidence of " + "that kind, or too few readings (see Recorded stock and Hour-meter evidence); gaps " + "are not interpolated. Missing data and quality checks (Baseline & Evidence) list what " + "is missing.", + "fuel": "**Reading this tab.** Deliveries are not consumption; the bars add up each day's " + "deliveries, and the Deliveries table lists each one, including any of unknown quantity. Recorded stock is shown as points only; nothing is drawn between " + "readings. Fuel per calendar day is one bar per observed stock interval, never spread " + "over months; a grey bar is an interval whose stock balance is not usable, and its " + "status (legend, tooltip, Stock intervals table) names the reason. The Viewing period narrows every chart here (by delivery, reading or " + "interval start date).", + "runtime": "**Reading this tab.** Runtime and hour-meter evidence stay empty until hour-meter " + "readings exist; an invalid reading pair (two epochs, a decreasing timer, over " + "24 h/day) shows no bar and is listed with its flags in the pair table. Grid energy without a value means no kWh records. " + "Electricity cost is grid cost only (AFN, inferred) and is never converted to fuel " + "or hours; a disputed bill is listed without amount or kWh. The Viewing period " + "narrows every chart here except grid energy.", + "baseline": "**Reading this tab.** Baseline versions and their inputs stay empty until a " + "version is proposed. Interval L/h shows bars only for usable intervals, and the " + "line only for an approved baseline of the same metric and population. Stock " + "against engine-measured fuel stays empty until flow-meter data exist; the " + "difference is shown only with full coverage and known visit times. Modelled L/h " + "comes from an assumed runtime, is not measured, and never reaches the site's L/h, " + "the map or the sites list. The Viewing period does not apply to this tab.", +} + +DETAIL_HEADER = [("D0", 12, 24)] +DETAIL_TABS = [ + ("summary", "Summary", [ + [("MARKDOWN:notes-summary", 12, 17)], + [("S1", 12, 30)], + [("S2", 12, 60)], + [("S3", 12, 50)], + ]), + ("fuel", "Fuel & Deliveries", [ + [("MARKDOWN:notes-fuel", 12, 17)], + [("F1", 6, 60), ("F2", 6, 60)], + [("F5", 12, 50)], + [("F3", 12, 60)], + [("F4", 12, 60)], + ]), + ("runtime", "Runtime & Electricity", [ + [("MARKDOWN:notes-runtime", 12, 14)], + [("R1", 6, 55), ("R2", 6, 55)], + [("R1T", 12, 45)], + [("R3", 6, 55), ("COLUMN:R4", 6, [("R4", 50), ("R4T", 50)])], + ]), + ("baseline", "Baseline & Evidence", [ + [("MARKDOWN:notes-baseline", 12, 14)], + [("B1", 6, 50), ("B2", 6, 50)], + [("B3", 12, 50)], + [("B4", 12, 55)], + [("B5", 6, 50), ("B6", 6, 50)], + [("B7", 6, 50), ("B8", 6, 50)], + [("B9", 6, 50), ("B10", 6, 50)], + [("B11", 12, 45)], + ]), +] +# SPEC §4: the Viewing period applies to the time-series charts of these two tabs only. +PERIOD_SCOPE = {"F1", "F2", "F3", "F4", "F5", "R1", "R1T", "R2", "R4", "R4T"} + + +def detail_filters(ids: dict[str, int]) -> list[dict[str, Any]]: + site = select_filter(SITE_FILTER, "Site", "v_site", "site_label", multi=False, + required=True, excluded=[]) + # SPEC §4 fallback: "required" alone still lets the charts query every site while + # the filter is empty (local test, 24 Oct 2026), so the first site is preselected. + site["controlValues"]["defaultToFirstItem"] = True + # What the filter editor writes with that option: charts wait for this filter's + # value, and the first value is applied without "Apply filters". + site["requiredFirst"] = True + site["description"] = "Exactly one site. Search by code or name." + period = period_filter(ids, PERIOD_SCOPE, + "Narrows the fuel, runtime and electricity charts. " + "Summary and Baseline & Evidence keep their fixed windows.") + return [site, period] + + +def period_filter(ids: dict[str, int], scope: set[str], description: str) -> dict[str, Any]: + """The Viewing period, applied only to the charts named in scope.""" + return { + "id": PERIOD_FILTER, + "name": "Viewing period", + "filterType": "filter_time", + "type": "NATIVE_FILTER", + "targets": [{}], + "controlValues": {"enableEmptyFilter": False}, + "defaultDataMask": {"extraFormData": {}, "filterState": {}, "ownState": {}}, + "cascadeParentIds": [], + "scope": {"rootPath": ["ROOT_ID"], + "excluded": sorted(i for k, i in ids.items() if k not in scope)}, + "description": description, + } + + +def select_filter(fid: str, name: str, view: str, column: str, *, multi: bool, required: bool, + excluded: list[int]) -> dict[str, Any]: + return { + "id": fid, + "name": name, + "filterType": "filter_select", + "type": "NATIVE_FILTER", + "targets": [{"datasetUuid": uid("dataset", view), "column": {"name": column}}], + "controlValues": { + "enableEmptyFilter": required, + "defaultToFirstItem": False, + "multiSelect": multi, + "searchAllOptions": False, + "inverseSelection": False, + "creatable": False, + "sortAscending": True, + }, + "defaultDataMask": {"extraFormData": {}, "filterState": {}, "ownState": {}}, + "cascadeParentIds": [], + # The frontend derives chartsInScope from this; excluded must be a list. + "scope": {"rootPath": ["ROOT_ID"], "excluded": excluded}, + "description": "", + } + + +# --- Dashboards --------------------------------------------------------------------------- + +class Layout: + """Builds a dashboard position (v2) from rows of (key, width, height).""" + + def __init__(self, title: str, charts: dict[str, Chart]) -> None: + self.charts = charts + self.ids: dict[str, int] = {} + self.nodes: dict[str, Any] = { + "DASHBOARD_VERSION_KEY": "v2", + "ROOT_ID": {"children": [], "id": "ROOT_ID", "type": "ROOT"}, + "GRID_ID": {"children": [], "id": "GRID_ID", "parents": ["ROOT_ID"], "type": "GRID"}, + "HEADER_ID": {"id": "HEADER_ID", "meta": {"text": title}, "type": "HEADER"}, + } + self.nodes["ROOT_ID"]["children"].append("GRID_ID") + self.markdown: dict[str, str] = {} + + def add(self, node_id: str, node_type: str, parent: str, meta: dict[str, Any]) -> str: + parents = self.nodes[parent].get("parents", []) + [parent] + self.nodes[node_id] = {"children": [], "id": node_id, "meta": meta, "parents": parents, + "type": node_type} + self.nodes[parent]["children"].append(node_id) + return node_id + + def chart(self, key: str, width: int, height: int, parent: str) -> None: + chart = self.charts[key] + self.ids[key] = len(self.ids) + 1 # placeholder id; the importer maps it by uuid + self.add(f"CHART-{key}", "CHART", parent, { + "chartId": self.ids[key], "height": height, "sliceName": chart.title, + "uuid": chart.uuid, "width": width}) + + def rows(self, rows: list[list[tuple]], parent: str, prefix: str) -> None: + for n, row in enumerate(rows, 1): + if sum(item[1] for item in row) > 12: + raise SystemExit(f"layout: row {prefix}-{n} is wider than 12") + row_id = self.add(f"ROW-{prefix}-{n}", "ROW", parent, + {"background": "BACKGROUND_TRANSPARENT"}) + for key, width, rest in row: + if key.startswith("MARKDOWN:"): + name = key.split(":", 1)[1] + self.add(f"MARKDOWN-{name}", "MARKDOWN", row_id, + {"width": width, "height": rest, "code": self.markdown[name]}) + elif key.startswith("COLUMN:"): + col_id = self.add(f"COLUMN-{key.split(':', 1)[1]}", "COLUMN", row_id, + {"background": "BACKGROUND_TRANSPARENT", "width": width}) + for sub_key, height in rest: + self.chart(sub_key, width, height, col_id) + else: + self.chart(key, width, rest, row_id) + + def tab(self, tab_id: str, title: str, tabs_id: str) -> str: + return self.add(tab_id, "TAB", tabs_id, + {"text": title, "defaultText": "Tab title", "placeholder": "Tab title"}) + + def tabs(self, tabs: list[tuple[str, str, list]], parent: str, tabs_id: str) -> None: + tabs_id = self.add(tabs_id, "TABS", parent, {}) + for slug, title, rows in tabs: + self.rows(rows, self.tab(f"TAB-{slug}", title, tabs_id), slug) + + +def dashboard_yaml(key: str, title: str, slug: str, layout: Layout, metadata: dict[str, Any], + css: str = "") -> dict[str, Any]: + return { + "dashboard_title": title, + "description": None, + "css": css, + "slug": slug, + "certified_by": None, + "certification_details": None, + "published": True, + "uuid": uid("dashboard", key), + "position": layout.nodes, + "metadata": metadata, + "version": VERSION, + } + + +def base_metadata(layout: Layout, filters: list[dict[str, Any]], cross_filters: bool) -> dict[str, Any]: + return { + "color_scheme": "supersetColors", + "refresh_frequency": 0, + # Expanded descriptions never render in 6.1 (the dashboard loads description_markeddown, + # the chart holder reads description_markdown), so each tab has a notes block instead. + "expanded_slices": {}, + "label_colors": dict(STATUS_COLOURS), + "shared_label_colors": [], + "map_label_colors": {}, + "color_scheme_domain": [], + "cross_filters_enabled": cross_filters, + # Filters in Superset's left column (owner, 29 Oct). Readers don't see it on the Fleet + # tab (assets/branding/reader.css), so their map gets the full width. + "filter_bar_orientation": "VERTICAL", + "native_filter_configuration": filters, + "chart_configuration": {}, + "global_chart_configuration": {"scope": {"rootPath": ["ROOT_ID"], "excluded": []}}, + "timed_refresh_immune_slices": [], + "default_filters": "{}", + "filter_scopes": {}, + } + + +def build(database_uuid: str, columns_path: Path) -> dict[str, Any]: + """Return {path inside the bundle: yaml object}.""" + views = read_columns(columns_path) + files: dict[str, Any] = { + "metadata.yaml": {"version": VERSION, "type": "Dashboard", "timestamp": "2026-10-24T00:00:00+00:00"}, + "databases/Fuel_Trino.yaml": { + "database_name": DATABASE_NAME, + "sqlalchemy_uri": DATABASE_URI, + "cache_timeout": None, + "expose_in_sqllab": True, + "allow_run_async": False, + "allow_ctas": False, + "allow_cvas": False, + "allow_dml": False, + "allow_file_upload": False, + "extra": {"metadata_params": {}, "engine_params": {}, "metadata_cache_timeout": {}, + "schemas_allowed_for_file_upload": []}, + "impersonate_user": False, + "uuid": database_uuid, + "version": VERSION, + }, + } + for name, (base, extra, _) in VIRTUAL.items(): + views[name] = views[base] + extra + for view in sorted(views): + files[f"datasets/Fuel_Trino/{view}.yaml"] = dataset_yaml(view, views[view], database_uuid) + + charts = {c.key: c for c in detail_charts()} + + # SPEC §2: [Map] [Insights]; the site header and the four detail tabs live in + # Insights, so the Map tab shows the map only. + layout = Layout("ATOMA", charts) + layout.markdown["fleet-map"] = FLEET_MAP + layout.markdown.update({f"notes-{slug}": text for slug, text in TAB_NOTES.items()}) + main_tabs = layout.add("TABS-main", "TABS", "GRID_ID", {}) + layout.rows([[("MARKDOWN:fleet-map", 12, 100)]], layout.tab(FLEET_TAB, FLEET_TAB_TITLE, main_tabs), "fleet") + site_tab = layout.tab(SITE_TAB, SITE_TAB_TITLE, main_tabs) + layout.rows([DETAIL_HEADER], site_tab, "header") + layout.tabs(DETAIL_TABS, site_tab, "TABS-detail") + metadata = base_metadata(layout, detail_filters(layout.ids), cross_filters=False) + metadata["label_colors"].update(STOCK_STATUS_COLOURS) + + files["dashboards/ATOMA.yaml"] = dashboard_yaml("atoma", "ATOMA", SLUG, layout, metadata, + DASHBOARD_CSS.read_text()) + dashboards = [(layout, charts, metadata, (DASHBOARD_CSS, READER_CSS))] + + # PD-12: the Petrolyst app embeds the Overview and one site dashboard, whose four detail + # tabs are Superset's own (owner, 29 Oct). The app shows the site header and chooses site + # and period, so these have no D0 and no CSS. + overview = {c.key: c for c in overview_charts()} + all_charts = {**charts, **overview} + for key, title in [(OVERVIEW_SLUG, "Overview"), (SITE_SLUG, "Site")]: + page = Layout(f"ATOMA · {title}", all_charts) + page.markdown.update({f"notes-{k}": v for k, v in TAB_NOTES.items()}) + page.markdown["notes-overview"] = OVERVIEW_NOTES + if key == OVERVIEW_SLUG: + page.rows(OVERVIEW_ROWS, "GRID_ID", "overview") + filters = [period_filter(page.ids, OVERVIEW_PERIOD_SCOPE, + "Narrows the deliveries, the costs and the monthly bars.")] + else: + page.tabs(DETAIL_TABS, "GRID_ID", "TABS-detail") + filters = detail_filters(page.ids) + placed = {k: all_charts[k] for k in page.ids} + page_metadata = base_metadata(page, filters, cross_filters=False) + page_metadata["label_colors"].update(STOCK_STATUS_COLOURS) + files[f"dashboards/{key}.yaml"] = dashboard_yaml(key, f"ATOMA · {title}", key, page, + page_metadata) + dashboards.append((page, placed, page_metadata, ())) + + for chart in all_charts.values(): + name = re.sub(r"\W+", "_", f"{chart.key}_{chart.title}").strip("_") + files[f"charts/{name}.yaml"] = chart.yaml(uid("dataset", chart.view)) + + check(files, views, dashboards, overview) + return files + + +def check(files: dict[str, Any], views: dict[str, list[tuple[str, str]]], + dashboards: list[tuple[Layout, dict[str, Chart], dict[str, Any], tuple[Path, ...]]], + overview: dict[str, Chart]) -> None: + problems = [] + columns = {v: {c for c, _ in cols} | {c[0] for c in CALCULATED.get(v, [])} + for v, cols in views.items()} + for view in list(MAIN_DTTM) + list(CALCULATED) + list(METRICS): + if view not in columns: + problems.append(f"{view}: configured but not in the column snapshot") + for view, col in MAIN_DTTM.items(): + if col not in columns.get(view, set()): + problems.append(f"{view}: main time column {col} missing") + for layout, charts, metadata, stylesheets in dashboards: + for css in stylesheets: + for ref in sorted(set(re.findall(r"#([A-Z]+-[\w-]+)", css.read_text()))): + if ref not in layout.nodes: + problems.append(f"{css.name}: #{ref} is not in the layout") + placed = set(layout.ids) + if placed != set(charts): + problems.append(f"charts not placed or unknown: {sorted(placed ^ set(charts))}") + for chart in charts.values(): + missing = set(chart.columns_used) - columns.get(chart.view, set()) + if missing: + problems.append(f"{chart.key}: {sorted(missing)} not in {chart.view}") + for f in metadata["native_filter_configuration"]: + in_scope = [k for k, i in layout.ids.items() if i not in f["scope"]["excluded"]] + for target in f["targets"]: + col = target.get("column", {}).get("name") + if col is None: + continue + for key in in_scope: + if col not in columns.get(charts[key].view, set()): + problems.append(f"{f['id']}: {key} ({charts[key].view}) lacks {col}") + # The fleet map's "Site details" link is built from its iframe address. + fleet = layout.nodes.get("MARKDOWN-fleet-map") + if fleet is not None: + filter_ids = {f["id"] for f in metadata["native_filter_configuration"]} + for needle, what in [(f"detail=/superset/dashboard/{SLUG}/", "this dashboard"), + (f"filter={SITE_FILTER}", "the Site filter"), + (f"tab={SITE_TAB}", "the Insights tab"), + (f"tab_label={SITE_TAB_TITLE}", "the Insights tab's name")]: + if needle not in fleet["meta"]["code"]: + problems.append(f"fleet map: its address does not name {what}") + if SITE_FILTER not in filter_ids or SITE_TAB not in layout.nodes: + problems.append("fleet map: the Site filter or the Insights tab is missing") + if not (HERE.parent.parent / "assets" / "branding" / OPERATOR_LOGO).is_file(): + problems.append(f"fleet map: assets/branding/{OPERATOR_LOGO} is missing") + if metadata.get("filter_bar_orientation") != "VERTICAL": + problems.append("fleet map: the filters must sit in the left column") + if f'data-node-key="{FLEET_TAB}"' not in READER_CSS.read_text(): + problems.append(f"fleet map: reader.css does not hide the filters on {FLEET_TAB}") + if fleet["meta"]["width"] != 12: + problems.append("fleet map: reader.css stretches it, so it must span all 12 columns") + # SPEC §10: the Overview sums recorded values only, and money only per currency. + for chart in overview.values(): + metrics = chart.params.get("metrics") or [chart.params["metric"]] + for metric in metrics: + expression = metric["sqlExpression"] + if expression not in OVERVIEW_METRICS: + problems.append(f"{chart.key}: metric {expression!r} is not an allowed total") + elif (OVERVIEW_METRICS[expression] + and "currency_text" not in chart.params.get("groupby", [])): + problems.append(f"{chart.key}: money must be grouped by currency_text") + uuids = [obj["uuid"] for path, obj in files.items() if path != "metadata.yaml"] + if len(uuids) != len(set(uuids)): + problems.append("duplicate uuids") + if problems: + raise SystemExit("bundle check failed:\n " + "\n ".join(problems)) + + +def write_zip(files: dict[str, Any], out: Path) -> None: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as bundle: + for path in sorted(files): + text = yaml.safe_dump(files[path], sort_keys=False, allow_unicode=True, width=100) + info = zipfile.ZipInfo(f"atoma_dashboards/{path}", date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + bundle.writestr(info, text) + out.write_bytes(buffer.getvalue()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--database-uuid", required=True, type=uuid.UUID, + help='uuid of the "Fuel (Trino)" database in the target Superset') + parser.add_argument("--columns", type=Path, default=HERE / "columns.tsv") + parser.add_argument("--out", type=Path, default=Path("atoma-dashboards.zip")) + args = parser.parse_args(argv) + files = build(str(args.database_uuid), args.columns) + write_zip(files, args.out) + counts = {k: sum(p.startswith(k + "/") for p in files) for k in ("datasets", "charts", "dashboards")} + print(f"{args.out}: {counts['dashboards']} dashboards, {counts['charts']} charts, " + f"{counts['datasets']} datasets") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/petrolyst-insights/dashboards/atoma/columns.tsv b/petrolyst-insights/dashboards/atoma/columns.tsv new file mode 100644 index 0000000..5439fd2 --- /dev/null +++ b/petrolyst-insights/dashboards/atoma/columns.tsv @@ -0,0 +1,309 @@ +# Columns of the public.v_* views (migrations 0001-0003), as Trino reports them. +# build_bundle.py lists every one in its dataset; import_bundle.py reports drift. +# Regenerate after a migration that changes a view: +# trino --output-format TSV --execute "select table_name, column_name, data_type +# from fuel.information_schema.columns where table_schema = 'public' +# and table_name like 'v\_%' escape '\' order by table_name, ordinal_position" +v_assumption site_uid varchar(12) +v_assumption site_code varchar(20) +v_assumption site_name varchar(120) +v_assumption site_label varchar(170) +v_assumption code varchar(60) +v_assumption topic varchar(30) +v_assumption asset_uid varchar(40) +v_assumption value decimal(10,3) +v_assumption value_low decimal(10,3) +v_assumption value_high decimal(10,3) +v_assumption unit varchar(20) +v_assumption applies_from date +v_assumption applies_to date +v_assumption basis varchar(20) +v_assumption derived_from_fuel boolean +v_assumption status varchar(12) +v_assumption accepted_on date +v_assumption decision_ref varchar(12) +v_assumption rationale varchar(400) +v_assumption contradicted_by varchar(200) +v_assumption members varchar(400) +v_baseline_input site_uid varchar(12) +v_baseline_input site_code varchar(20) +v_baseline_input site_name varchar(120) +v_baseline_input site_label varchar(170) +v_baseline_input baseline_code varchar(60) +v_baseline_input baseline_status varchar(12) +v_baseline_input interval_label varchar(30) +v_baseline_input start_on date +v_baseline_input end_on date +v_baseline_input included boolean +v_baseline_input exclusion_reason varchar(300) +v_baseline_input fuel_l decimal(10,1) +v_baseline_input runtime_h decimal(10,1) +v_baseline_input days integer +v_baseline_input eligibility_at_computation varchar(12) +v_baseline_input evidence_grade_at_computation varchar(2) +v_baseline_input dependencies_changed boolean +v_baseline_version site_uid varchar(12) +v_baseline_version site_code varchar(20) +v_baseline_version site_name varchar(120) +v_baseline_version site_label varchar(170) +v_baseline_version code varchar(60) +v_baseline_version metric varchar(40) +v_baseline_version status varchar(12) +v_baseline_version population varchar(200) +v_baseline_version value decimal(8,3) +v_baseline_version range_low decimal(8,3) +v_baseline_version range_high decimal(8,3) +v_baseline_version unit varchar(8) +v_baseline_version window_start date +v_baseline_version window_end date +v_baseline_version fuel_l decimal(10,1) +v_baseline_version runtime_h decimal(10,1) +v_baseline_version matched_days integer +v_baseline_version policy_version varchar(10) +v_baseline_version meets_policy boolean +v_baseline_version policy_check varchar(300) +v_baseline_version evidence_grade varchar(2) +v_baseline_version conditions varchar(200) +v_baseline_version season_note varchar(200) +v_baseline_version approved_on date +v_baseline_version approved_by_role varchar(40) +v_baseline_version decision_ref varchar(12) +v_baseline_version inputs_unchanged boolean +v_baseline_version computed_at timestamp(0) +v_coverage site_uid varchar(12) +v_coverage site_code varchar(20) +v_coverage site_name varchar(120) +v_coverage site_label varchar(170) +v_coverage lane varchar(60) +v_coverage status varchar(40) +v_coverage start_on date +v_coverage end_on date +v_coverage start_ts timestamp(0) +v_coverage end_ts timestamp(0) +v_data_quality site_uid varchar(12) +v_data_quality site_code varchar(20) +v_data_quality site_name varchar(120) +v_data_quality site_label varchar(170) +v_data_quality check_order integer +v_data_quality check_code varchar(40) +v_data_quality label varchar(80) +v_data_quality status varchar(10) +v_data_quality detail varchar(300) +v_delivery site_uid varchar(12) +v_delivery site_code varchar(20) +v_delivery site_name varchar(120) +v_delivery site_label varchar(170) +v_delivery event_key varchar(80) +v_delivery delivery_date date +v_delivery seq_in_day integer +v_delivery delivered_l decimal(8,1) +v_delivery unit_price decimal(8,2) +v_delivery currency varchar(8) +v_delivery source_ref varchar(60) +v_delivery n_sources integer +v_delivery sources_disagree boolean +v_delivery reviewed boolean +v_electricity_cost site_uid varchar(12) +v_electricity_cost site_code varchar(20) +v_electricity_cost site_name varchar(120) +v_electricity_cost site_label varchar(170) +v_electricity_cost bill_month date +v_electricity_cost period_start date +v_electricity_cost period_end date +v_electricity_cost period_known boolean +v_electricity_cost amount decimal(12,2) +v_electricity_cost currency varchar(8) +v_electricity_cost currency_basis varchar(10) +v_electricity_cost kwh decimal(10,1) +v_electricity_cost source_ref varchar(60) +v_electricity_cost claim_status varchar(12) +v_evidence site_uid varchar(12) +v_evidence site_code varchar(20) +v_evidence site_name varchar(120) +v_evidence site_label varchar(170) +v_evidence document_key varchar(16) +v_evidence title varchar(200) +v_evidence kind varchar(20) +v_evidence issuer varchar(80) +v_evidence document_date date +v_evidence used_for varchar(200) +v_evidence contains_personal_data boolean +v_hour_meter_evidence site_uid varchar(12) +v_hour_meter_evidence site_code varchar(20) +v_hour_meter_evidence site_name varchar(120) +v_hour_meter_evidence site_label varchar(170) +v_hour_meter_evidence meter_label varchar(40) +v_hour_meter_evidence mapped_set varchar(60) +v_hour_meter_evidence reading_date date +v_hour_meter_evidence date_basis varchar(30) +v_hour_meter_evidence hours decimal(9,1) +v_hour_meter_evidence reading_state varchar(16) +v_hour_meter_evidence source_field varchar(16) +v_hour_meter_evidence epoch_no integer +v_hour_meter_evidence claim_status varchar(12) +v_imported_calculation site_uid varchar(12) +v_imported_calculation site_code varchar(20) +v_imported_calculation site_name varchar(120) +v_imported_calculation site_label varchar(170) +v_imported_calculation cell_ref varchar(40) +v_imported_calculation source_label varchar(80) +v_imported_calculation formula varchar(200) +v_imported_calculation cached_value decimal(14,6) +v_imported_calculation unit_as_labelled varchar(30) +v_imported_calculation our_reading varchar(300) +v_imported_calculation baseline_use varchar(12) +v_imported_calculation exclusion_reason varchar(300) +v_imported_calculation document_key varchar(16) +v_imported_calculation claim_status varchar(12) +v_interval site_uid varchar(12) +v_interval site_code varchar(20) +v_interval site_name varchar(120) +v_interval site_label varchar(170) +v_interval interval_label varchar(30) +v_interval start_on date +v_interval end_on date +v_interval days integer +v_interval opening_l decimal(9,1) +v_interval opening_basis varchar(8) +v_interval deliveries_l decimal(9,1) +v_interval withdrawals_l decimal(9,1) +v_interval closing_l decimal(9,1) +v_interval closing_basis varchar(8) +v_interval fuel_used_l decimal(9,1) +v_interval fuel_used_unc_l decimal(9,1) +v_interval fuel_l_per_day decimal(8,1) +v_interval stock_status varchar(40) +v_interval runtime_h decimal(9,1) +v_interval runtime_h_per_day decimal(6,2) +v_interval n_meters integer +v_interval metric varchar(40) +v_interval population_key varchar(200) +v_interval l_per_h decimal(8,3) +v_interval l_per_h_low decimal(8,3) +v_interval l_per_h_high decimal(8,3) +v_interval eligibility varchar(12) +v_interval exclusion_reasons varchar(600) +v_interval evidence_grade varchar(2) +v_interval conditions varchar(200) +v_interval in_current_baseline boolean +v_interval baseline_comparable boolean +v_interval current_baseline_l_per_h decimal(8,3) +v_kpi_measures site_uid varchar(12) +v_kpi_measures site_code varchar(20) +v_kpi_measures site_name varchar(120) +v_kpi_measures site_label varchar(170) +v_kpi_measures measure_order integer +v_kpi_measures measure varchar(60) +v_kpi_measures value decimal(12,2) +v_kpi_measures unit varchar(12) +v_kpi_measures value_kind varchar(16) +v_kpi_measures window_start date +v_kpi_measures window_end date +v_kpi_measures note varchar(200) +v_modelled_lph site_uid varchar(12) +v_modelled_lph site_code varchar(20) +v_modelled_lph site_name varchar(120) +v_modelled_lph site_label varchar(170) +v_modelled_lph assumption_code varchar(60) +v_modelled_lph status varchar(12) +v_modelled_lph basis varchar(20) +v_modelled_lph h_per_day decimal(5,2) +v_modelled_lph h_per_day_low decimal(5,2) +v_modelled_lph h_per_day_high decimal(5,2) +v_modelled_lph window_start date +v_modelled_lph window_end date +v_modelled_lph matched_days integer +v_modelled_lph l_per_day decimal(7,1) +v_modelled_lph modelled_l_per_h decimal(6,2) +v_modelled_lph modelled_l_per_h_low decimal(6,2) +v_modelled_lph modelled_l_per_h_high decimal(6,2) +v_modelled_lph value_kind varchar(70) +v_reconciliation site_uid varchar(12) +v_reconciliation site_code varchar(20) +v_reconciliation site_name varchar(120) +v_reconciliation site_label varchar(170) +v_reconciliation interval_label varchar(30) +v_reconciliation start_on date +v_reconciliation end_on date +v_reconciliation stock_status varchar(40) +v_reconciliation endpoint_times_known boolean +v_reconciliation asset_uid varchar(40) +v_reconciliation placed_throughout boolean +v_reconciliation stock_balance_l decimal(9,1) +v_reconciliation engine_l decimal(9,1) +v_reconciliation covered_h decimal(9,1) +v_reconciliation interval_h decimal(9,1) +v_reconciliation engine_all_generators_l decimal(9,1) +v_reconciliation all_generators_metered boolean +v_reconciliation difference_l decimal(9,1) +v_reconciliation difference_pct decimal(7,1) +v_runtime site_uid varchar(12) +v_runtime site_code varchar(20) +v_runtime site_name varchar(120) +v_runtime site_label varchar(170) +v_runtime meter_label varchar(40) +v_runtime mapped_set varchar(60) +v_runtime interval_label varchar(80) +v_runtime start_on date +v_runtime end_on date +v_runtime days integer +v_runtime runtime_h decimal(9,1) +v_runtime h_per_day decimal(6,2) +v_runtime pair_valid boolean +v_runtime same_epoch boolean +v_runtime monotonic boolean +v_runtime over_24h_per_day boolean +v_runtime reviewed boolean +v_site site_uid varchar(12) +v_site site_code varchar(20) +v_site site_name varchar(120) +v_site site_label varchar(170) +v_site zone varchar(40) +v_site province varchar(40) +v_site district varchar(60) +v_site pilot_status varchar(12) +v_site latitude decimal(9,6) +v_site longitude decimal(9,6) +v_site location_status varchar(20) +v_site location_basis varchar(20) +v_site location_precision_m integer +v_site baseline_status varchar(24) +v_site baseline_status_label varchar(40) +v_site baseline_validity_note varchar(60) +v_site lph_estimate decimal(6,2) +v_site lph_method varchar(60) +v_site lph_window_start date +v_site lph_window_end date +v_site baseline_conditions varchar(200) +v_site baseline_decision_ref varchar(12) +v_site fuel_l_per_calendar_day decimal(7,1) +v_site latest_record_on date +v_spec_reference site_uid varchar(12) +v_spec_reference site_code varchar(20) +v_spec_reference site_name varchar(120) +v_spec_reference site_label varchar(170) +v_spec_reference slot varchar(20) +v_spec_reference asset_uid varchar(40) +v_spec_reference asset_kind varchar(10) +v_spec_reference make_model varchar(120) +v_spec_reference attribute varchar(40) +v_spec_reference load_pct decimal(5,1) +v_spec_reference value decimal(12,3) +v_spec_reference value_text varchar(120) +v_spec_reference unit varchar(20) +v_spec_reference basis varchar(30) +v_spec_reference applicability varchar(12) +v_spec_reference applicability_note varchar(300) +v_spec_reference value_kind varchar(30) +v_spec_reference claim_status varchar(12) +v_stock_reading site_uid varchar(12) +v_stock_reading site_code varchar(20) +v_stock_reading site_name varchar(120) +v_stock_reading site_label varchar(170) +v_stock_reading observed_on date +v_stock_reading seq_in_day integer +v_stock_reading reading_type varchar(20) +v_stock_reading litres decimal(8,1) +v_stock_reading value_kind varchar(12) +v_stock_reading spread_l decimal(8,1) +v_stock_reading reviewed boolean diff --git a/petrolyst-insights/dashboards/atoma/dashboard.css b/petrolyst-insights/dashboards/atoma/dashboard.css new file mode 100644 index 0000000..25cf6a1 --- /dev/null +++ b/petrolyst-insights/dashboards/atoma/dashboard.css @@ -0,0 +1,33 @@ +/* The ATOMA dashboard's own CSS, for everyone who opens it (build_bundle.py puts it into the + bundle and checks that every #id here is in the layout). + + Site details' inner tabs (TABS-detail) as a row of section buttons, so they read as a level + below Fleet | Site details (owner, 29 Oct). Aligned with the content's left edge. + Translucent colours: right in the light and the dark theme. */ +#TABS-detail > .ant-tabs-nav { + margin: 0 16px 16px; +} +#TABS-detail > .ant-tabs-nav::before { + border-bottom: 0; +} +#TABS-detail > .ant-tabs-nav .ant-tabs-ink-bar { + display: none; +} +#TABS-detail > .ant-tabs-nav .ant-tabs-nav-list { + gap: 4px; + padding: 4px; + border-radius: 10px; + background: rgba(127, 127, 127, 0.1); +} +#TABS-detail > .ant-tabs-nav .ant-tabs-tab { + margin: 0 !important; + padding: 6px 14px !important; + border-radius: 8px !important; + font-size: 13px; +} +#TABS-detail > .ant-tabs-nav .ant-tabs-tab-active { + background: rgba(0, 150, 136, 0.14) !important; +} +#TABS-detail > .ant-tabs-nav .ant-tabs-tab-active .ant-tabs-tab-btn { + font-weight: 600; +} diff --git a/petrolyst-insights/dashboards/atoma/import_bundle.py b/petrolyst-insights/dashboards/atoma/import_bundle.py new file mode 100644 index 0000000..53d19d5 --- /dev/null +++ b/petrolyst-insights/dashboards/atoma/import_bundle.py @@ -0,0 +1,151 @@ +"""Import the ATOMA dashboard bundle into Superset, overwriting what it defines. + +Runs inside the Superset container, with the image's Python: + + docker cp import_bundle.py :/tmp/ && docker cp atoma-dashboards.zip :/tmp/ + docker exec /app/.venv/bin/python /tmp/import_bundle.py --show-database + docker exec /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip --user + +`superset import-dashboards` imports a dashboard's datasets and charts with +overwrite off, so an existing chart or dataset keeps its old definition. This +script runs the dataset, chart and dashboard importers in turn, each with +overwrite on (the bundle's metadata.yaml type is set to what each importer +expects). A dataset import replaces the column list, so columns missing from +the bundle are dropped; afterwards each dataset's columns are compared with the +live view and any drift is reported (exit status 2). + +The database is never created or changed: the bundle's database uuid must +already exist under the name "Fuel (Trino)", otherwise nothing is imported. + +A bundle cannot carry a dashboard's roles, so afterwards the role "ATOMA Reader" +is attached to the ATOMA dashboard: with DASHBOARD_RBAC that is how ATOMA's +readers see it. The role comes from roles/atoma_reader.py (run by superset-init); +without it nothing is imported. + +The Petrolyst app's dashboards (PD-12: atoma-overview, atoma-site) get no role: +the app shows them with guest passes. With --embedded-out FILE --embed-domain URL, +each of them is made embeddable from that origin (its embedding id stays the same +across imports), and FILE receives {slug: embedding id} for the app service: + + docker exec /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip \ + --user --embedded-out /tmp/embedded-dashboards.json --embed-domain http://127.0.0.1:8098 +""" + +from __future__ import annotations + +import argparse +import json +import sys +import zipfile + +import yaml + +DATABASE_NAME = "Fuel (Trino)" +READER_ROLE = "ATOMA Reader" +READER_SLUG = "atoma" # the readers' dashboard; the others are the app's (PD-12) +EMBEDDED_PREFIX = "atoma-" +STEPS = [("datasets", "SqlaTable"), ("charts", "Slice"), ("dashboards", "Dashboard")] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Import the ATOMA dashboard bundle.") + parser.add_argument("bundle", nargs="?") + parser.add_argument("--user", help="existing Superset user who will own new objects") + parser.add_argument("--show-database", action="store_true", + help=f'print the uuid of "{DATABASE_NAME}" and stop') + parser.add_argument("--embedded-out", help="write {slug: embedding id} of the app's dashboards") + parser.add_argument("--embed-domain", help="the app's origin, e.g. http://127.0.0.1:8098") + args = parser.parse_args() + if not args.show_database and not (args.bundle and args.user): + parser.error("give a bundle and --user, or --show-database") + if bool(args.embedded_out) != bool(args.embed_domain): + parser.error("--embedded-out and --embed-domain go together") + + from superset.app import create_app + + app = create_app() + with app.app_context(): + from superset import db, security_manager + from superset.commands.chart.importers.v1 import ImportChartsCommand + from superset.commands.dashboard.importers.v1 import ImportDashboardsCommand + from superset.commands.dataset.importers.v1 import ImportDatasetsCommand + from superset.commands.importers.v1.utils import get_contents_from_bundle + from superset.connectors.sqla.models import SqlaTable + from superset.models.core import Database + from superset.models.dashboard import Dashboard + from superset.utils.core import override_user + + if args.show_database: + found = db.session.query(Database).filter_by(database_name=DATABASE_NAME).all() + for database in found: + print(database.uuid) + return 0 if found else 1 + + user = security_manager.find_user(username=args.user) + if user is None: + print(f"no Superset user {args.user!r}", file=sys.stderr) + return 1 + reader_role = security_manager.find_role(READER_ROLE) + if reader_role is None: + print(f'no role "{READER_ROLE}": run roles/atoma_reader.py first', file=sys.stderr) + return 1 + with zipfile.ZipFile(args.bundle) as bundle: + contents = get_contents_from_bundle(bundle) + databases = [yaml.safe_load(t) for p, t in contents.items() if p.startswith("databases/")] + for config in databases: + database = db.session.query(Database).filter_by(uuid=config["uuid"]).one_or_none() + if database is None or database.database_name != DATABASE_NAME: + print(f'database {config["uuid"]} is not "{DATABASE_NAME}" here; ' + "rebuild the bundle with the uuid from --show-database", file=sys.stderr) + return 1 + + metadata = yaml.safe_load(contents["metadata.yaml"]) + commands = {"datasets": ImportDatasetsCommand, "charts": ImportChartsCommand, + "dashboards": ImportDashboardsCommand} + with override_user(user): + for step, model in STEPS: + step_contents = dict(contents) + step_contents["metadata.yaml"] = yaml.safe_dump({**metadata, "type": model}) + commands[step](step_contents, overwrite=True).run() + count = sum(p.startswith(step + "/") for p in contents) + print(f"{step}: {count} imported (overwrite)") + + embedded: dict[str, str] = {} + for path, text in contents.items(): + if path.startswith("dashboards/"): + dashboard = db.session.query(Dashboard).filter_by( + uuid=yaml.safe_load(text)["uuid"]).one() + if dashboard.slug == READER_SLUG: + if reader_role not in dashboard.roles: + dashboard.roles.append(reader_role) + elif dashboard.slug.startswith(EMBEDDED_PREFIX) and args.embedded_out: + from superset.daos.dashboard import EmbeddedDashboardDAO + + config = EmbeddedDashboardDAO.upsert(dashboard, [args.embed_domain]) + db.session.flush() + embedded[dashboard.slug] = str(config.uuid) + db.session.commit() + print(f'dashboards: "{READER_SLUG}" readable for "{READER_ROLE}"') + if args.embedded_out: + with open(args.embedded_out, "w") as out: + json.dump(dict(sorted(embedded.items())), out, indent=2) + print(f"embedded: {len(embedded)} dashboard(s) for {args.embed_domain}, " + f"ids in {args.embedded_out}") + + drift = 0 + dataset_uuids = [yaml.safe_load(t)["uuid"] for p, t in contents.items() + if p.startswith("datasets/")] + for dataset_uuid in dataset_uuids: + dataset = db.session.query(SqlaTable).filter_by(uuid=dataset_uuid).one() + physical = {c.column_name for c in dataset.columns if not c.expression} + live = {c["column_name"] for c in dataset.external_metadata()} + if physical != live: + drift += 1 + print(f"drift {dataset.table_name}: only in bundle {sorted(physical - live)}, " + f"only in view {sorted(live - physical)}") + print(f"column drift: {drift} dataset(s)") + return 2 if drift else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/petrolyst-insights/patches/README.md b/petrolyst-insights/patches/README.md index 668752d..981205c 100644 --- a/petrolyst-insights/patches/README.md +++ b/petrolyst-insights/patches/README.md @@ -1,7 +1,10 @@ # patches/ Runtime patches applied while the Superset image is built -(`stacks/fuel-monitoring/superset/Dockerfile`). Empty until Petrolyst needs one. +(`stacks/fuel-monitoring/superset/Dockerfile`): + +- `fix_redact_query_for_readers.py`: chart-data responses carry no SQL for readers, as Superset + already does for embedded guests (`petrolyst_reader.py`). Same convention as media-insights: diff --git a/petrolyst-insights/patches/fix_redact_query_for_readers.py b/petrolyst-insights/patches/fix_redact_query_for_readers.py new file mode 100644 index 0000000..e983c60 --- /dev/null +++ b/petrolyst-insights/patches/fix_redact_query_for_readers.py @@ -0,0 +1,54 @@ +"""Readers' chart-data responses carry no SQL. + +Superset 6.1.0 drops each query's SQL from chart-data JSON for embedded guest users only +(superset/charts/data/api.py, ChartDataRestApi._send_chart_response). This extends that +condition with the security manager's redacts_query(), when it has one: the Petrolyst managers +return True for readers (petrolyst_reader.ReaderAccessMixin). It runs when the response is +written, so cached results and result_type=query are covered too. + + python3 fix_redact_query_for_readers.py apply (idempotent) + python3 fix_redact_query_for_readers.py --verify exit 1 unless applied +""" + +import sys +from pathlib import Path + +TARGET = Path("/app/superset/charts/data/api.py") +BEFORE = """\ + if security_manager.is_guest_user(): + for query in queries: + query.pop("query", None) +""" +AFTER = """\ + # Petrolyst (patches/fix_redact_query_for_readers.py): readers too. + if security_manager.is_guest_user() or getattr( + security_manager, "redacts_query", lambda: False + )(): + for query in queries: + query.pop("query", None) +""" + + +def main(argv: list[str]) -> int: + text = TARGET.read_text() + if "--verify" in argv: + compile(text, str(TARGET), "exec") + if text.count(AFTER) == 1 and BEFORE not in text: + print(f"{TARGET}: readers' SQL redaction in place") + return 0 + print(f"{TARGET}: readers' SQL redaction missing", file=sys.stderr) + return 1 + if text.count(AFTER) == 1: + print(f"{TARGET}: already patched") + return 0 + if text.count(BEFORE) != 1: + print(f"{TARGET}: the guest-only redaction was not found once; Superset changed?", + file=sys.stderr) + return 1 + TARGET.write_text(text.replace(BEFORE, AFTER)) + print(f"{TARGET}: patched") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/petrolyst-insights/petrolyst_reader.py b/petrolyst-insights/petrolyst_reader.py new file mode 100644 index 0000000..6294684 --- /dev/null +++ b/petrolyst-insights/petrolyst_reader.py @@ -0,0 +1,150 @@ +"""Readers: ATOMA's users, who only read the ATOMA dashboard. + +A reader is a signed-in user whose roles are "ATOMA Reader" and nothing more ("Public" aside, +which carries no permissions here). A user who also has Admin, Alpha or Gamma is not a reader. +The role itself (roles/atoma_reader.py) grants no menus and nothing to write; this module adds +what a role cannot express. Checked against apache/superset:6.1.0. + +- Superset lets an owner past the role check, and anyone onto a dashboard that has neither + roles nor datasets. ReaderAccessMixin lets readers reach a dashboard only through its roles + (DASHBOARD_RBAC), and the dashboard list shows them only those. +- Chart-data responses carry each query's SQL for everyone but embedded guests. Readers get none: + ReaderAccessMixin.redacts_query, read by patches/fix_redact_query_for_readers.py. +- Readers land on the ATOMA dashboard instead of Superset's home page, and any other page they + open (a chart or dashboard list, SQL Lab, Explore, user info) leads there too. Every page + loads reader.css for them (templates/head_custom_extra.html). It hides what Superset shows + whatever the permissions: the dashboard's title row, the theme switcher, Info and About in + Settings; and the filter column on ATOMA's Fleet tab. Its address names its content, since + browsers keep Superset's static files for a year. + +superset_config.py sets FLASK_APP_MUTATOR = register; both security managers (Keycloak in +petrolyst_security.py, database login on the local stack) take ReaderAccessMixin. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Any + +from flask import Flask, g, has_request_context, redirect, request +from flask_login import current_user +from jinja2 import ChoiceLoader, FileSystemLoader +from werkzeug.wrappers import Response as WerkzeugResponse + +READER_ROLE = "ATOMA Reader" +# Roles a reader may have besides READER_ROLE without stopping being one. +NEUTRAL_ROLES = {"Public"} +LANDING = "/superset/dashboard/atoma/" +# The only pages a reader opens; any other page leads to LANDING. Requests that are not page +# loads (the API, static files) keep their own permission checks. +READER_PAGES = ("/superset/dashboard/", "/login/", "/logout/") +TEMPLATES = Path(__file__).with_name("templates") + + +def _signed_in_user() -> Any: + user = getattr(g, "user", None) + if user is None and has_request_context(): + user = current_user + return user + + +def is_reader(user: Any = None) -> bool: + """True for a signed-in user whose roles are READER_ROLE and nothing that grants more.""" + user = user if user is not None else _signed_in_user() + if user is None or not getattr(user, "is_authenticated", False): + return False + names = {role.name for role in getattr(user, "roles", None) or []} + return READER_ROLE in names and names <= {READER_ROLE, *NEUTRAL_ROLES} + + +class ReaderAccessMixin: + """Reader rules for a SupersetSecurityManager subclass; list it before that base.""" + + def raise_for_ownership(self, resource: Any) -> None: + # is_owner() goes through here: a reader owns nothing, so no owner path to a + # dashboard, chart or dataset remains (also for a user who owned things before). + if is_reader(): + from superset.errors import ErrorLevel, SupersetError, SupersetErrorType + from superset.exceptions import SupersetSecurityException + + raise SupersetSecurityException( + SupersetError( + error_type=SupersetErrorType.MISSING_OWNERSHIP_ERROR, + message=f"A reader does not own {resource}", + level=ErrorLevel.ERROR, + ) + ) + super().raise_for_ownership(resource) # type: ignore[misc] + + def raise_for_access(self, *args: Any, **kwargs: Any) -> None: + # The dashboard page, its API, permalinks and chart data (via can_access_dashboard) + # all check a dashboard here. + dashboard = kwargs.get("dashboard", args[0] if args else None) + if dashboard is not None and is_reader() and not self._reader_may_open(dashboard): + from superset.exceptions import SupersetSecurityException + + raise SupersetSecurityException( + self.get_dashboard_access_error_object(dashboard) # type: ignore[attr-defined] + ) + super().raise_for_access(*args, **kwargs) # type: ignore[misc] + + def _reader_may_open(self, dashboard: Any) -> bool: + """Only DASHBOARD_RBAC: published, and one of its roles is the reader's.""" + mine = {role.id for role in self.get_user_roles()} # type: ignore[attr-defined] + return bool(dashboard.published) and bool(mine & {role.id for role in dashboard.roles}) + + def redacts_query(self) -> bool: + """Whether chart-data responses leave out the SQL (embedded guests aside).""" + return is_reader() + + +def _land_readers() -> WerkzeugResponse | None: + if request.method != "GET" or request.path.startswith(READER_PAGES) or not is_reader(): + return None + # A page load in a browser, or Superset's home page from anything. + if request.headers.get("Sec-Fetch-Dest") == "document" or request.path in ( + "/", + "/superset/welcome/", + ): + return redirect(LANDING) + return None + + +def _limit_dashboard_list() -> None: + """The dashboard list shows readers only published dashboards that carry their roles.""" + from superset import security_manager + from superset.dashboards.filters import DashboardAccessFilter + from superset.models.dashboard import Dashboard + + original = DashboardAccessFilter.apply + if getattr(original, "petrolyst_readers", False): + return + + def apply(self: DashboardAccessFilter, query: Any, value: Any) -> Any: + query = original(self, query, value) + if is_reader(): + role_ids = [role.id for role in security_manager.get_user_roles()] + query = query.filter( + Dashboard.published.is_(True), + Dashboard.roles.any(security_manager.role_model.id.in_(role_ids)), + ) + return query + + apply.petrolyst_readers = True # type: ignore[attr-defined] + DashboardAccessFilter.apply = apply # type: ignore[method-assign] + + +def register(app: Flask) -> None: + """FLASK_APP_MUTATOR: runs once Superset is initialized.""" + # Our templates first: head_custom_extra.html, which Superset's spa.html includes on every + # page, is the only file there. + app.jinja_env.loader = ChoiceLoader([FileSystemLoader(str(TEMPLATES)), app.jinja_env.loader]) + app.jinja_env.globals["petrolyst_is_reader"] = is_reader + # Static files are cached for a year (Cache-Control max-age), so the address carries the + # file's hash: a changed reader.css reaches readers on their next page load. + css = Path(app.static_folder or "") / "assets" / "branding" / "reader.css" + version = hashlib.sha256(css.read_bytes()).hexdigest()[:12] + app.jinja_env.globals["petrolyst_reader_css_version"] = version + app.before_request(_land_readers) + _limit_dashboard_list() diff --git a/petrolyst-insights/petrolyst_security.py b/petrolyst-insights/petrolyst_security.py index f5e6bb0..b1d504f 100644 --- a/petrolyst-insights/petrolyst_security.py +++ b/petrolyst-insights/petrolyst_security.py @@ -11,6 +11,9 @@ the stock behaviour: login page on /login/, which only shows a "Sign in with keycloak" button. - /logout/ also ends the Keycloak session, so the next visit asks for the password and OTP again. + +It also takes the reader rules (petrolyst_reader.ReaderAccessMixin), as does the +local stack's database-login manager. """ import logging @@ -24,6 +27,7 @@ from flask_appbuilder.utils.base import get_safe_redirect from flask_login import logout_user from werkzeug.wrappers import Response as WerkzeugResponse +from petrolyst_reader import ReaderAccessMixin from superset.security import SupersetSecurityManager logger = logging.getLogger(__name__) @@ -64,7 +68,7 @@ class PetrolystAuthOAuthView(AuthOAuthView): return redirect(f"{end_session}?{urlencode(params)}") -class PetrolystSecurityManager(SupersetSecurityManager): +class PetrolystSecurityManager(ReaderAccessMixin, SupersetSecurityManager): authoauthview = PetrolystAuthOAuthView # Without this, Superset registers its React login page on /login/ before # FAB's OAuth view, and that page wins the route. diff --git a/petrolyst-insights/roles/atoma_guest.py b/petrolyst-insights/roles/atoma_guest.py new file mode 100644 index 0000000..484341e --- /dev/null +++ b/petrolyst-insights/roles/atoma_guest.py @@ -0,0 +1,23 @@ +"""The role "ATOMA Guest": what the Petrolyst app's embedded dashboards may do (PD-12 trial). + +Superset gives every embedded viewer (a guest pass signed for one dashboard) the role named by +GUEST_ROLE_NAME. That role gets the same permissions as "ATOMA Reader": reading a dashboard +and its charts' data, nothing else. Which dashboard a guest sees comes from the pass, not from +the role. Superset leaves the SQL out of chart-data responses for guests. + +Embedding is switched on in the local stack only (.local/fmdb-scratch, superset_config_local.py), +so this script is not part of init.sh; run it by hand inside the Superset container: + + docker exec /app/.venv/bin/python /app/petrolyst-roles/atoma_guest.py +""" + +from __future__ import annotations + +import sys + +from atoma_reader import PERMISSIONS, sync + +ROLE = "ATOMA Guest" + +if __name__ == "__main__": + sys.exit(sync(ROLE, PERMISSIONS)) diff --git a/petrolyst-insights/roles/atoma_reader.py b/petrolyst-insights/roles/atoma_reader.py new file mode 100644 index 0000000..f08bc84 --- /dev/null +++ b/petrolyst-insights/roles/atoma_reader.py @@ -0,0 +1,76 @@ +"""The role "ATOMA Reader": read the dashboards granted to it, nothing else. + +Runs inside the Superset container, after `superset init` (stacks/fuel-monitoring/superset/init.sh): + + docker exec /app/.venv/bin/python /app/petrolyst-roles/atoma_reader.py + +Creates the role if needed and gives it exactly PERMISSIONS: missing ones are added, any other +is removed (also one added by hand in the UI). A second run changes nothing. Readers reach a +dashboard only when it lists this role (DASHBOARD_RBAC; dashboards/atoma/import_bundle.py +attaches it), and petrolyst_reader.py adds the rules a role cannot express. + +The list was established on the local stack by reading the ATOMA dashboard as a reader: +everything the dashboard asks the server for, and nothing more. No menu access, nothing to +write, no Explore, CSV, sharing, drill or query view. +""" + +from __future__ import annotations + +import sys + +ROLE = "ATOMA Reader" +PERMISSIONS = [ + ("can_read", "Dashboard"), # the dashboard and its charts' definitions + ("can_dashboard", "Superset"), # the dashboard page + ("can_dashboard_permalink", "Superset"), # a dashboard permalink + ("can_read", "Chart"), # chart data + ("can_time_range", "Api"), # the Viewing period filter + ("can_read", "DashboardFilterStateRestApi"), # native filter state + ("can_write", "DashboardFilterStateRestApi"), + ("can_read", "DashboardPermalinkRestApi"), + ("can_read", "CurrentUserRestApi"), # who is signed in + ("can_read", "SecurityRestApi"), # the CSRF token + ("can_get", "MenuApi"), + ("can_log", "Superset"), # Superset's event log +] + + +def sync(name: str, permissions: list[tuple[str, str]]) -> int: + """Give the role `name` exactly `permissions`, creating it if needed (atoma_guest.py too).""" + from superset.app import create_app + + app = create_app() + with app.app_context(): + from superset import db, security_manager as sm + + wanted = [] + for permission, view in permissions: + pvm = sm.find_permission_view_menu(permission, view) + if pvm is None: + print(f"no permission {permission} on {view}: run `superset init` first?", + file=sys.stderr) + return 1 + wanted.append(pvm) + + role = sm.find_role(name) or sm.add_role(name) + added = [pvm for pvm in wanted if pvm not in role.permissions] + removed = [pvm for pvm in role.permissions if pvm not in wanted] + for pvm in added: + sm.add_permission_role(role, pvm) + for pvm in removed: + sm.del_permission_role(role, pvm) + db.session.commit() + + print(f'role "{name}": {len(wanted)} permissions, {len(added)} added, ' + f"{len(removed)} removed") + for pvm in removed: + print(f" removed {pvm.permission.name} on {pvm.view_menu.name}") + return 0 + + +def main() -> int: + return sync(ROLE, PERMISSIONS) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/petrolyst-insights/superset_config.py b/petrolyst-insights/superset_config.py index 3ba6d4c..e7a1119 100644 --- a/petrolyst-insights/superset_config.py +++ b/petrolyst-insights/superset_config.py @@ -13,6 +13,7 @@ from copy import deepcopy from flask_appbuilder.security.manager import AUTH_OAUTH from sqlalchemy.engine.url import URL +import petrolyst_reader from petrolyst_security import PetrolystSecurityManager from superset import config as superset_defaults @@ -79,6 +80,8 @@ AUTH_ROLES_MAPPING = { "superset-admin": ["Admin"], "superset-editor": ["Alpha"], "superset-viewer": ["Gamma"], + # ATOMA's users: read the ATOMA dashboard, nothing else (petrolyst_reader.py). + "atoma-reader": ["ATOMA Reader"], } AUTH_ROLES_SYNC_AT_LOGIN = True AUTH_USER_REGISTRATION = True @@ -87,7 +90,31 @@ AUTH_USER_REGISTRATION = True AUTH_USER_REGISTRATION_ROLE = "Public" # Viewers (Gamma) see a dashboard once it lists their role. -FEATURE_FLAGS = {"DASHBOARD_RBAC": True} +# FILTERBAR_CLOSED_BY_DEFAULT: a dashboard's filter column opens collapsed, a +# narrow strip with an expand button (owner, 29 Oct); ?expand_filters=true +# opens it from a link. +FEATURE_FLAGS = {"DASHBOARD_RBAC": True, "FILTERBAR_CLOSED_BY_DEFAULT": True} + +# Readers ("ATOMA Reader"): land on the ATOMA dashboard, see it without the +# title row and the menus Superset shows to everyone, and reach no other +# dashboard (petrolyst_reader.py). +FLASK_APP_MUTATOR = petrolyst_reader.register + +# deck.gl maps: OpenStreetMap raster tiles only. The default list adds Mapbox +# styles, which fail without MAPBOX_API_KEY. The default CSP already allows the +# tile host. The ATOMA map credits OSM in a Markdown block beside it. +DECKGL_BASE_MAP = [ + ["tile://https://tile.openstreetmap.org/{z}/{x}/{y}.png", "OpenStreetMap"], +] + +# Markdown blocks may hold an iframe: the ATOMA Fleet tab embeds the fleet-map view +# from petrolyst-platform (/views/fleet-map/). Only these attributes survive. The CSP +# (default-src 'self', no frame-src) keeps frames to this origin, so a dashboard +# editor cannot embed another site. +HTML_SANITIZATION_SCHEMA_EXTENSIONS = { + "tagNames": ["iframe"], + "attributes": {"iframe": ["src", "title", "loading", "style"]}, +} # --- Behind Traefik ----------------------------------------------------------- ENABLE_PROXY_FIX = True @@ -133,7 +160,7 @@ RATELIMIT_STORAGE_URI = f"{REDIS_URL}/1" BRANDING = "/static/assets/branding" APP_NAME = "Petrolyst" -APP_ICON = f"{BRANDING}/petrolyst_logo_standard_fullcolor_no_tagline.svg" +APP_ICON = f"{BRANDING}/petrolyst_wordmark_fullcolor.svg" FAVICONS = [ {"href": f"{BRANDING}/favicon.svg", "type": "image/svg+xml"}, {"href": f"{BRANDING}/favicon.ico", "type": "image/x-icon"}, @@ -147,10 +174,13 @@ THEME_DEFAULT["token"].update( "brandLogoAlt": "Petrolyst", "brandLogoUrl": APP_ICON, "brandLogoHref": "/", - # The standard logo is stacked (mark over wordmark); 40px keeps the Y - # legible and, with the margin, the 60px navbar height. - "brandLogoHeight": "40px", - "brandLogoMargin": "10px 0", + # The wordmark alone (owner, 29 Oct): in the stacked logo, at 40px, its + # letters were about 5px high. Its own Y carries the brand accent. 24px + # (letters about 20px) and the margin fill the 60px navbar height. On the + # left, 15px more than the navbar's 16px: the P (1.4px inside the image) + # lines up with the dashboard's content and a reader's fleet map at 32px. + "brandLogoHeight": "24px", + "brandLogoMargin": "18px 0 18px 15px", # Action teal, not the raw brand teal #009688: the brand guide keeps # raw teal off small text and controls. "colorPrimary": "#00695C", @@ -171,7 +201,7 @@ THEME_DARK = deepcopy(THEME_DEFAULT) THEME_DARK["algorithm"] = "dark" THEME_DARK["token"].update( { - "brandLogoUrl": f"{BRANDING}/petrolyst_logo_standard_white_no_tagline.svg", + "brandLogoUrl": f"{BRANDING}/petrolyst_wordmark_white.svg", "colorPrimary": "#35C5B6", "colorLink": "#35C5B6", "colorSuccess": "#35C5B6", diff --git a/petrolyst-insights/templates/head_custom_extra.html b/petrolyst-insights/templates/head_custom_extra.html new file mode 100644 index 0000000..0c8fdc6 --- /dev/null +++ b/petrolyst-insights/templates/head_custom_extra.html @@ -0,0 +1,6 @@ +{#- Petrolyst: replaces Superset's empty head_custom_extra.html, which spa.html includes on every + page (petrolyst_reader.register puts this folder first). Readers get reader.css; its address + names its content, as browsers keep static files for a year. -#} +{% if petrolyst_is_reader() %} + +{% endif %} diff --git a/stacks/fuel-monitoring/superset/Dockerfile b/stacks/fuel-monitoring/superset/Dockerfile index a40a08b..dc96845 100644 --- a/stacks/fuel-monitoring/superset/Dockerfile +++ b/stacks/fuel-monitoring/superset/Dockerfile @@ -11,7 +11,9 @@ USER root RUN uv pip install --no-cache --python /app/.venv/bin/python \ authlib==1.9.1 psycopg2-binary==2.9.13 "trino[sqlalchemy]==0.340.0" -COPY --from=insights --chown=superset:superset superset_config.py petrolyst_security.py /app/pythonpath/ +COPY --from=insights --chown=superset:superset superset_config.py petrolyst_security.py petrolyst_reader.py /app/pythonpath/ +COPY --from=insights --chown=superset:superset templates/ /app/pythonpath/templates/ +COPY --from=insights --chown=superset:superset roles/ /app/petrolyst-roles/ COPY --from=insights --chown=superset:superset assets/branding/ /app/superset/static/assets/branding/ COPY --from=insights patches/ /app/patches/petrolyst/ # Stop at the first patch or verify that fails, so a later success cannot hide diff --git a/stacks/fuel-monitoring/superset/init.sh b/stacks/fuel-monitoring/superset/init.sh index d7bd43e..feba166 100755 --- a/stacks/fuel-monitoring/superset/init.sh +++ b/stacks/fuel-monitoring/superset/init.sh @@ -12,3 +12,5 @@ superset init superset set-database-uri \ --database_name "Fuel (Trino)" \ --uri "trino://superset@trino:8080/fuel" +# The role "ATOMA Reader", set to exactly its permissions (petrolyst-insights/roles/). +/app/.venv/bin/python /app/petrolyst-roles/atoma_reader.py