#!/usr/bin/env python3 """Generate the ATOMA dashboard as one Superset 6.1 import bundle. Implements SPEC.md in this folder: the "ATOMA" dashboard with a Map tab (the fleet-map view of petrolyst-platform, embedded as an iframe) and an Insights tab, one dataset per public.v_* view and every chart. Nothing in the bundle names a site. build_bundle.py --database-uuid [--out atoma-dashboards.zip] The database uuid differs per Superset instance; `import_bundle.py --show-database` prints it. Output is deterministic: the same inputs give a byte-identical zip, and stable uuid5 ids make a re-import update the objects instead of copying them. Import with import_bundle.py, which overwrites datasets, charts and dashboards (the stock dashboard import leaves existing datasets and charts unchanged). Before writing, the bundle is checked: every column a chart or filter uses exists in its dataset, every chart in a Site filter's scope has site_label, the fleet map's address names this dashboard, its Site filter and the Insights tab, every layout id that dashboard.css or reader.css names exists, and no row is wider than 12. Requires PyYAML. """ from __future__ import annotations import argparse import io import json import re import sys import uuid import zipfile from pathlib import Path from typing import Any import yaml HERE = Path(__file__).resolve().parent VERSION = "1.0.0" NAMESPACE = uuid.uuid5(uuid.NAMESPACE_URL, "https://petrolyst.com/superset/atoma-dashboards") DATABASE_NAME = "Fuel (Trino)" DATABASE_URI = "trino://superset@trino:8080/fuel" # as superset-init registers it CATALOG, SCHEMA = "fuel", "public" SLUG = "atoma" SITE_FILTER = "NATIVE_FILTER-site" PERIOD_FILTER = "NATIVE_FILTER-period" FLEET_TAB = "TAB-fleet" SITE_TAB = "TAB-site" # The tabs' names: the lenses Map and Insights (owner, 29 Oct); readers see them as a switch # in the navbar (assets/branding/reader.css). FLEET_TAB_TITLE = "Map" SITE_TAB_TITLE = "Insights" # SPEC §3: the Map tab is the fleet-map view of petrolyst-platform, served on the same # origin under /views (superset_config.py lets the iframe through the Markdown sanitiser). # Its parameters name where "Site details" leads: this dashboard, its Site filter and the # Insights tab, with that tab's name for the map's key. The view builds the native_filters # link from them. # The operator's logo in the map's bottom right corner: a file in assets/branding/, served by # Superset under /static/assets/branding/ (same origin, so the view's CSP allows it). OPERATOR_LOGO = "atoma_logo.svg" # The dashboard's own CSS (the Insights tab's inner tabs as section buttons), and the # readers' stylesheet, which names this layout's ids too (the lens switch, the map's size). DASHBOARD_CSS = HERE / "dashboard.css" READER_CSS = HERE.parent.parent / "assets" / "branding" / "reader.css" FLEET_MAP_SRC = (f"/views/fleet-map/?detail=/superset/dashboard/{SLUG}/" f"&filter={SITE_FILTER}&tab={SITE_TAB}&tab_label={SITE_TAB_TITLE}" f"&logo=/static/assets/branding/{OPERATOR_LOGO}&logo_alt=ATOMA") FLEET_MAP = (f'') # SPEC §3: neutral hues, no red or green, keyed by v_site.baseline_status_label. The # fleet-map view uses the same colours (petrolyst-platform views/src/shared/status.ts). STATUS_COLOURS = { "Approved baseline": "#009688", "Eligible - not yet approved": "#3987e5", "Provisional": "#c98500", "Insufficient evidence": "#64748B", } # SPEC §4 F3: a stock interval that is not "ok" is greyed; its status names the reason. EXCLUDED_GREY = "#A3A9B5" STOCK_STATUS_COLOURS = {status: EXCLUDED_GREY for status in ( "no_opening_or_closing", "endpoint_movement_order_unknown", "unquantified_movement", "same_day", "negative")} # Datasets with a time axis: the column the Viewing period applies to. MAIN_DTTM = { "v_delivery": "delivery_date", "v_stock_reading": "observed_on", "v_interval": "start_on", "v_runtime": "start_on", "v_hour_meter_evidence": "reading_date", "v_electricity_cost": "bill_month", } # Calculated columns: (name, Trino SQL, type, label). The fleet-map view's read service # (petrolyst-platform services/read, site_status) shows the same two texts. CALCULATED = { "v_site": [ ("lph_text", "COALESCE(CAST(lph_estimate AS varchar), 'not available')", "VARCHAR", "L/h"), # SPEC §4: "Eligible" and "Provisional" carry "not approved". ("baseline_status_text", "CASE WHEN baseline_status = 'provisional' THEN baseline_status_label" " || ' (not approved)' ELSE baseline_status_label END", "VARCHAR", "Baseline status"), ], # PD-12 Overview: money is grouped by currency; a missing one is its own row. "v_delivery": [ ("currency_text", "COALESCE(currency, 'currency not recorded')", "VARCHAR", "Currency"), ], "v_electricity_cost": [ ("currency_text", "COALESCE(currency, 'currency not recorded')", "VARCHAR", "Currency"), ("currency_basis_text", "COALESCE(currency_basis, 'basis not recorded')", "VARCHAR", "Currency basis"), ], "v_stock_reading": [ ("reading_series", "reading_type || ', ' || value_kind || CASE WHEN seq_in_day > 1" " THEN ' (visit ' || CAST(seq_in_day AS varchar) || ')' ELSE '' END", "VARCHAR", "Reading"), ], } # Virtual datasets: name -> (view it extends, extra output columns, Trino SQL). # coverage_detail feeds S3. The view's lane names start with a sort number ("1 Stock # intervals"), which reads as a count on the chart, so it adds lane_name without the number # (the chart still sorts by lane). Its amount column is the tooltip: the litres delivered in a # stock interval (from its opening visit up to, not including, its closing visit; the opening # delivery is what the interval then uses), or a bill's amount. # # v_coverage carries dates, not visits or bills, so an amount is withheld wherever dates cannot # place it (Codex inspection round 3, ATOMA-08/09): a delivery on a boundary day that has more # than one visit could belong to either neighbouring interval, and bills that start on the same # day cannot be told apart. Everywhere else the date window is exact: on a single-visit day the # opening visit's delivery is included and the closing visit's excluded. VIRTUAL = { "coverage_detail": ("v_coverage", [("lane_name", "varchar(60)"), ("amount", "varchar(120)")], """\ SELECT c.site_uid, c.site_code, c.site_name, c.site_label, c.lane, c.status, c.start_on, c.end_on, c.start_ts, c.end_ts, regexp_replace(c.lane, '^[0-9]+ ', '') AS lane_name, CASE WHEN c.lane = '1 Stock intervals' THEN CASE WHEN f.ambiguous THEN 'not attributed: a delivery on a day with several visits (see Deliveries)' WHEN f.filled_l IS NULL AND coalesce(f.n_unknown, 0) = 0 THEN 'no delivery' WHEN f.filled_l IS NULL THEN 'filled, quantity not recorded' ELSE format('%,.0f L filled', CAST(f.filled_l AS double)) END || CASE WHEN f.n_unknown > 0 AND f.filled_l IS NOT NULL AND NOT f.ambiguous THEN ', plus ' || CAST(f.n_unknown AS varchar) || ' of unknown quantity' ELSE '' END WHEN c.lane = '4 Electricity bills' THEN CASE WHEN b.n_bills > 1 THEN 'several bills start this day (see Electricity bills)' WHEN b.disputed THEN 'bill disputed, amount withheld' WHEN b.amount IS NULL THEN 'bill amount not recorded' ELSE format('%,.0f AFN billed (inferred)', CAST(b.amount AS double)) END END AS amount FROM public.v_coverage c LEFT JOIN (SELECT s.site_uid, s.start_on, s.end_on, sum(CASE WHEN d.delivery_date < s.end_on THEN d.delivered_l END) AS filled_l, count_if(d.delivery_date < s.end_on AND d.delivered_l IS NULL) AS n_unknown, bool_or(d.delivery_date IN (s.start_on, s.end_on) AND m.site_uid IS NOT NULL) AS ambiguous FROM (SELECT DISTINCT site_uid, start_on, end_on FROM public.v_coverage WHERE lane = '1 Stock intervals') s JOIN public.v_delivery d ON d.site_uid = s.site_uid AND d.delivery_date >= s.start_on AND d.delivery_date <= s.end_on LEFT JOIN (SELECT site_uid, day FROM (SELECT site_uid, delivery_date AS day, seq_in_day FROM public.v_delivery UNION ALL SELECT site_uid, observed_on, seq_in_day FROM public.v_stock_reading) x GROUP BY site_uid, day HAVING max(seq_in_day) > 1) m ON m.site_uid = d.site_uid AND m.day = d.delivery_date GROUP BY s.site_uid, s.start_on, s.end_on) f ON c.lane = '1 Stock intervals' AND f.site_uid = c.site_uid AND f.start_on = c.start_on AND f.end_on = c.end_on LEFT JOIN (SELECT site_uid, coalesce(period_start, bill_month) AS start_on, sum(amount) AS amount, bool_or(claim_status = 'disputed') AS disputed, count(*) AS n_bills FROM public.v_electricity_cost GROUP BY 1, 2) b ON c.lane = '4 Electricity bills' AND b.site_uid = c.site_uid AND b.start_on = c.start_on """), } # Saved metrics: (name, SQL, label). Every dataset also gets count. METRICS = { "v_delivery": [("litres_delivered", "SUM(delivered_l)", "Litres delivered")], "v_electricity_cost": [("amount", "SUM(amount)", "Amount (AFN, inferred)")], # SPEC §5: one eligibility filter on numerator and denominator. Group or filter # by metric and population_key wherever this is used. "v_interval": [( "pooled_l_per_h", "SUM(CASE WHEN eligibility <> 'excluded' THEN fuel_used_l END)" " / NULLIF(SUM(CASE WHEN eligibility <> 'excluded' THEN runtime_h END), 0)", "Pooled L/h (usable intervals)", )], } LABELS = { "site_code": "Code", "site_name": "Site name", "site_label": "Site", "site_uid": "Site uid", "pilot_status": "Pilot status", "baseline_status_label": "Baseline status", "baseline_status": "Baseline status code", "baseline_validity_note": "Validity note", "lph_estimate": "L/h", "lph_method": "L/h method", "lph_window_start": "L/h window start", "lph_window_end": "L/h window end", "baseline_decision_ref": "Decision", "fuel_l_per_calendar_day": "L/day, whole record", "latest_record_on": "Latest record", "location_status": "Location status", "location_precision_m": "Location precision (m)", "delivered_l": "Delivered (L)", "fuel_l_per_day": "L per calendar day", "l_per_h": "L/h", "l_per_h_low": "L/h low", "l_per_h_high": "L/h high", "current_baseline_l_per_h": "Approved baseline L/h", "fuel_used_l": "Used (L)", "fuel_used_unc_l": "Used, uncertainty (L)", "opening_l": "Opening (L)", "closing_l": "Closing (L)", "deliveries_l": "Deliveries (L)", "withdrawals_l": "Withdrawals (L)", "runtime_h": "Runtime (h)", "runtime_h_per_day": "Runtime (h/day)", "h_per_day": "h/day", "h_per_day_low": "h/day low", "h_per_day_high": "h/day high", "l_per_day": "L/day", "modelled_l_per_h": "Modelled L/h", "modelled_l_per_h_low": "Modelled L/h low", "modelled_l_per_h_high": "Modelled L/h high", "stock_balance_l": "Stock depletion, site (L)", "engine_l": "Engine-measured, this generator (L)", "engine_all_generators_l": "Engine-measured, all generators (L)", "all_generators_metered": "All generators metered", "covered_h": "Hours covered", "interval_h": "Interval hours", "difference_l": "Difference, site (L)", "difference_pct": "Difference, site (%)", "fuel_l": "Fuel (L)", "kwh": "kWh", "amount": "Amount", "value_kind": "Value kind", "claim_status": "Claim status", "interval_label": "Interval", "exclusion_reasons": "Exclusion reasons", "in_current_baseline": "In current baseline", "cached_value": "Workbook value", "unit_as_labelled": "Unit (as labelled)", "source_label": "Workbook label", "our_reading": "Our reading", "baseline_use": "Use", "make_model": "Make and model", "load_pct": "Load (%)", "value_text": "Value (text)", "assumption_code": "Assumption", "matched_days": "Matched days", "document_date": "Date", "used_for": "Used for", "check_code": "Check code", "label": "Check", "measure": "Measure", "lane_name": "Evidence", "seq_in_day": "Visit of the day", "event_key": "Event", "n_sources": "Sources", "sources_disagree": "Sources disagree", "delivery_date": "Date", } def label_for(column: str) -> str: return LABELS.get(column) or column.replace("_", " ").capitalize() def uid(kind: str, key: str) -> str: return str(uuid.uuid5(NAMESPACE, f"{kind}/{key}")) # --- Datasets ------------------------------------------------------------------- def read_columns(path: Path) -> dict[str, list[tuple[str, str]]]: views: dict[str, list[tuple[str, str]]] = {} for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip() or line.startswith("#"): continue view, column, data_type = line.split("\t") views.setdefault(view, []).append((column, data_type)) return views def superset_type(trino_type: str) -> str: return trino_type.upper().replace(",", ", ") def dataset_yaml(view: str, columns: list[tuple[str, str]], database_uuid: str) -> dict[str, Any]: cols = [] for name, data_type in columns: temporal = data_type.startswith(("date", "timestamp")) cols.append(column_entry(name, superset_type(data_type), temporal, None)) for name, expression, data_type, verbose in CALCULATED.get(view, []): cols.append(column_entry(name, data_type, False, expression, verbose)) metrics = [metric_entry("count", "COUNT(*)", "Count")] metrics += [metric_entry(*m) for m in METRICS.get(view, [])] return { "table_name": view, "main_dttm_col": MAIN_DTTM.get(view), "currency_code_column": None, "description": None, "default_endpoint": None, "offset": 0, "cache_timeout": None, "catalog": CATALOG, "schema": SCHEMA, "sql": VIRTUAL[view][2] if view in VIRTUAL else None, "params": None, "template_params": None, "filter_select_enabled": True, "fetch_values_predicate": None, "extra": None, "normalize_columns": False, "always_filter_main_dttm": False, "folders": None, "uuid": uid("dataset", view), "metrics": metrics, "columns": cols, "version": VERSION, "database_uuid": database_uuid, } def column_entry(name: str, data_type: str, temporal: bool, expression: str | None, verbose: str | None = None) -> dict[str, Any]: return { "column_name": name, "verbose_name": verbose or label_for(name), "is_dttm": temporal, "is_active": True, "type": data_type, "advanced_data_type": None, "groupby": True, "filterable": True, "expression": expression, "description": None, "python_date_format": None, "datetime_format": None, "extra": None, } def metric_entry(name: str, expression: str, verbose: str) -> dict[str, Any]: return { "metric_name": name, "verbose_name": verbose, "metric_type": None, "expression": expression, "description": None, "d3format": None, "currency": None, "extra": None, "warning_text": None, } # --- Chart building blocks ---------------------------------------------------------- def sql_metric(expression: str, label: str) -> dict[str, Any]: return { "expressionType": "SQL", "sqlExpression": expression, "label": label, "hasCustomLabel": True, "optionName": "metric_" + re.sub(r"\W+", "_", label.lower()).strip("_"), "column": None, "aggregate": None, "datasourceWarning": False, } def sql_filter(expression: str) -> dict[str, Any]: return { "expressionType": "SQL", "sqlExpression": expression, "clause": "WHERE", "subject": None, "operator": None, "comparator": None, "isExtra": False, "isNew": False, "datasourceWarning": False, "filterOptionName": "filter_" + uuid.uuid5(NAMESPACE, expression).hex[:12], } def time_filter(column: str) -> dict[str, Any]: """An open TEMPORAL_RANGE filter: the Viewing period replaces its range.""" return { "expressionType": "SIMPLE", "subject": column, "operator": "TEMPORAL_RANGE", "comparator": "No filter", "clause": "WHERE", "sqlExpression": None, "isExtra": False, "isNew": False, "datasourceWarning": False, "filterOptionName": f"filter_period_{column}", } def order(*columns: str, ascending: bool = True) -> list[str]: return [json.dumps([c, ascending]) for c in columns] class Chart: """One chart: key (SPEC id), title, dataset, viz type, params, description.""" def __init__(self, key: str, title: str, view: str, viz: str, params: dict[str, Any], description: str | None = None, columns_used: list[str] | None = None) -> None: self.key, self.title, self.view, self.viz = key, title, view, viz self.params = {"viz_type": viz, "datasource": "0__table", **params} self.description = description self.columns_used = columns_used or [] @property def uuid(self) -> str: return uid("chart", self.key) def yaml(self, dataset_uuid: str) -> dict[str, Any]: return { "slice_name": self.title, "description": self.description, "certified_by": None, "certification_details": None, "viz_type": self.viz, "params": self.params, "query_context": None, "cache_timeout": None, "uuid": self.uuid, "version": VERSION, "dataset_uuid": dataset_uuid, } def table(key: str, title: str, view: str, columns: list[str], *, order_by: list[str] | None = None, filters: list[dict[str, Any]] | None = None, period: str | None = None, server_pages: bool = False, search: bool = False, html: bool = False, description: str | None = None) -> Chart: adhoc = list(filters or []) if period: adhoc.append(time_filter(period)) params = { "query_mode": "raw", "all_columns": columns, "groupby": [], "metrics": [], "percent_metrics": [], "order_by_cols": order_by or [], "adhoc_filters": adhoc, "row_limit": 10000, "server_pagination": server_pages, "server_page_length": 25, "page_length": 25, "include_search": search, "show_cell_bars": False, "color_pn": False, "allow_render_html": html, "table_timestamp_format": "%Y-%m-%d", "column_config": {}, "conditional_formatting": [], } used = columns + [json.loads(o)[0] for o in order_by or []] + ([period] if period else []) return Chart(key, title, view, "table", params, description, used) def bars(key: str, title: str, view: str, x: str, metrics: list[Any], *, groupby: list[str] | None = None, grain: str | None = None, period: str | None = None, y_title: str = "", filters: list[dict[str, Any]] | None = None, description: str | None = None) -> Chart: adhoc = list(filters or []) if period: adhoc.append(time_filter(period)) params = { **timeseries_common(x, metrics, groupby or [], grain, adhoc, y_title), "orientation": "vertical", "stack": "Stack" if groupby else None, "only_total": True, "show_value": False, } used = [x] + list(groupby or []) + ([period] if period else []) return Chart(key, title, view, "echarts_timeseries_bar", params, description, used) def summary_table(key: str, title: str, view: str, groupby: list[str], metrics: list[Any], *, period: str | None = None, description: str | None = None) -> Chart: """An aggregated table: one row per group, one column per metric.""" params = { "query_mode": "aggregate", "groupby": groupby, "metrics": metrics, "all_columns": [], "percent_metrics": [], "order_by_cols": [], "order_desc": True, "adhoc_filters": [time_filter(period)] if period else [], "row_limit": 1000, "server_pagination": False, "include_search": False, "show_cell_bars": False, "color_pn": False, "allow_render_html": False, "table_timestamp_format": "%Y-%m-%d", "column_config": {}, "conditional_formatting": [], } return Chart(key, title, view, "table", params, description, groupby + ([period] if period else [])) def big_number(key: str, title: str, view: str, metric: dict[str, Any], *, period: str | None = None, subheader: str = "", description: str | None = None) -> Chart: params = { "metric": metric, "adhoc_filters": [time_filter(period)] if period else [], "subheader": subheader, "header_font_size": 0.4, "subheader_font_size": 0.15, "y_axis_format": "SMART_NUMBER", "time_format": "smart_date", "force_timestamp_formatting": False, } return Chart(key, title, view, "big_number_total", params, description, [period] if period else []) def timeseries_common(x: str, metrics: list[Any], groupby: list[str], grain: str | None, adhoc: list[dict[str, Any]], y_title: str) -> dict[str, Any]: return { "x_axis": x, "time_grain_sqla": grain, "metrics": metrics, "groupby": groupby, "adhoc_filters": adhoc, "row_limit": 10000, "truncate_metric": True, "show_empty_columns": True, "comparison_type": "values", "annotation_layers": [], "color_scheme": "supersetColors", "show_legend": bool(groupby), "legendType": "scroll", "legendOrientation": "top", "x_axis_title": "", "x_axis_title_margin": 15, "y_axis_title": y_title, "y_axis_title_margin": 40, "y_axis_title_position": "Left", "x_axis_time_format": "%Y-%m-%d", "y_axis_format": "SMART_NUMBER", "rich_tooltip": True, "tooltipTimeFormat": "%Y-%m-%d", "truncateXAxis": True, "y_axis_bounds": [None, None], "zoomable": False, } # --- Overview (PD-12, embedded in the Petrolyst app) --------------------------------------- OVERVIEW_SLUG = "atoma-overview" SITE_SLUG = "atoma-site" # one embedded dashboard with the DETAIL_TABS as its tabs # SPEC §10: only sums and counts of recorded values, never an average of per-site rates; money # only grouped by currency. The value says whether the metric is money. OVERVIEW_METRICS = { "COUNT(*)": False, "SUM(delivered_l)": False, "COUNT(unit_price)": False, "SUM(delivered_l * unit_price)": True, "SUM(CASE WHEN COALESCE(claim_status, '') <> 'disputed' THEN amount END)": True, "SUM(CASE WHEN claim_status = 'disputed' THEN amount END)": True, } def overview_charts() -> list[Chart]: def m(expression: str, label: str) -> dict[str, Any]: assert expression in OVERVIEW_METRICS, expression return sql_metric(expression, label) return [ big_number("O3", "Litres delivered", "v_delivery", m("SUM(delivered_l)", "Litres delivered"), period="delivery_date", subheader="all sites, in the viewing period", description="Recorded deliveries. Deliveries are not consumption."), summary_table("O1", "Sites by pilot status", "v_site", ["pilot_status"], [m("COUNT(*)", "Sites")]), summary_table("O2", "Sites by baseline status", "v_site", ["baseline_status_label"], [m("COUNT(*)", "Sites")]), summary_table("O4", "Fuel cost by currency", "v_delivery", ["currency_text"], [m("SUM(delivered_l * unit_price)", "Cost (priced deliveries)"), m("COUNT(unit_price)", "Priced deliveries"), m("COUNT(*)", "Deliveries")], period="delivery_date", description="Litres times the recorded unit price, only where a price was " "recorded. Never added up across currencies."), summary_table("O5", "Electricity billed by currency", "v_electricity_cost", ["currency_text", "currency_basis_text"], [m("SUM(CASE WHEN COALESCE(claim_status, '') <> 'disputed' THEN amount END)", "Billed"), m("SUM(CASE WHEN claim_status = 'disputed' THEN amount END)", "Disputed"), m("COUNT(*)", "Bills")], period="bill_month", description="Grid cost as billed; disputed bills apart. Never converted to " "fuel or hours, never added up across currencies."), bars("O6", "Litres delivered per month", "v_delivery", "delivery_date", [m("SUM(delivered_l)", "Litres delivered")], grain="P1M", period="delivery_date", y_title="Litres", description="All sites' recorded deliveries per calendar month."), ] OVERVIEW_NOTES = ("**Reading this page.** All sites together, from recorded values only: " "deliveries are not consumption, and site rates (L/h, L/day) are never averaged " "here; each site's own page has them. Money is shown per currency and never added " "up across currencies. The viewing period narrows the deliveries, the costs and " "the monthly bars; the site counts are always the current state.") OVERVIEW_ROWS = [ [("MARKDOWN:notes-overview", 12, 14)], [("O3", 4, 40), ("O1", 4, 40), ("O2", 4, 40)], [("O4", 6, 40), ("O5", 6, 40)], [("O6", 12, 60)], ] OVERVIEW_PERIOD_SCOPE = {"O3", "O4", "O5", "O6"} # --- Site Detail ----------------------------------------------------------------------- def detail_charts() -> list[Chart]: return [ # Handlebars would render these as cards, but it compiles templates with # eval, which Superset's own CSP blocks. One-row tables need no CSP change. table("D0", "Site", "v_site", [ "site_code", "site_name", "province", "pilot_status", "baseline_status_text", "location_status"]), table("S1", "Fuel per running hour", "v_site", [ "lph_text", "lph_method", "lph_window_start", "lph_window_end", "baseline_status_text", "baseline_validity_note", "baseline_decision_ref", "baseline_conditions", "fuel_l_per_calendar_day"], description="L/h comes only from an approved or proposed baseline version. " "L/day is per calendar day over the whole record, not per running hour."), table("S2", "Supporting measures", "v_kpi_measures", [ "measure", "value", "unit", "value_kind", "window_start", "window_end", "note"], order_by=order("measure_order")), Chart("S3", "Evidence coverage", "coverage_detail", "gantt_chart", { "start_time": "start_ts", "end_time": "end_ts", "y_axis": "lane_name", "series": "status", "subcategories": False, "tooltip_metrics": [], "order_by_cols": order("lane"), "adhoc_filters": [], "row_limit": 10000, "color_scheme": "supersetColors", "show_legend": True, "legendType": "scroll", "legendOrientation": "top", "zoomable": True, "x_axis_time_format": "%Y-%m-%d", "tooltip_time_format": "%Y-%m-%d", "tooltip_columns": ["amount"], "y_axis_title": "Evidence", "y_axis_title_margin": 150, "y_axis_title_position": "Left", }, "Periods covered by each kind of evidence. Gaps are not interpolated.", ["start_ts", "end_ts", "lane", "lane_name", "status", "amount"]), bars("F1", "Litres delivered per day (all deliveries that day)", "v_delivery", "delivery_date", ["litres_delivered"], grain="P1D", period="delivery_date", y_title="Litres", description="Deliveries are not consumption. F5 lists each delivery."), table("F5", "Deliveries", "v_delivery", [ "delivery_date", "seq_in_day", "event_key", "delivered_l", "unit_price", "currency", "n_sources", "sources_disagree", "reviewed", "source_ref"], order_by=order("delivery_date", "seq_in_day"), period="delivery_date", search=True, description="Every delivery event, including one whose quantity is not known."), Chart("F2", "Recorded stock", "v_stock_reading", "echarts_timeseries_scatter", { # A second visit on the same day is its own series, so no reading is hidden # behind the day's MAX. **timeseries_common("observed_on", [sql_metric("MAX(litres)", "Litres")], ["reading_series"], "P1D", [time_filter("observed_on")], "Litres"), "markerSize": 6, }, "Points only; nothing is drawn between readings.", ["observed_on", "litres", "reading_series", "seq_in_day"]), bars("F3", "Fuel per calendar day, by stock interval", "v_interval", "interval_label", [sql_metric("MAX(fuel_l_per_day)", "L per calendar day")], groupby=["stock_status"], period="start_on", y_title="L/day", description="One bar per observed stock interval; nothing is spread over months. " "F4 lists the intervals with their status."), table("F4", "Stock intervals", "v_interval", [ "interval_label", "start_on", "end_on", "days", "opening_l", "opening_basis", "deliveries_l", "withdrawals_l", "closing_l", "closing_basis", "fuel_used_l", "fuel_l_per_day", "stock_status"], order_by=order("start_on"), period="start_on"), bars("R1", "Runtime per day, by meter interval", "v_runtime", "interval_label", [sql_metric("MAX(h_per_day)", "h/day")], groupby=["meter_label"], period="start_on", y_title="h/day", description="Empty until hour-meter readings exist. An invalid reading pair " "(two epochs, a decreasing timer, over 24 h/day) shows no bar."), table("R1T", "Runtime pairs and their validity", "v_runtime", [ "meter_label", "mapped_set", "interval_label", "start_on", "end_on", "days", "runtime_h", "h_per_day", "pair_valid", "same_epoch", "monotonic", "over_24h_per_day", "reviewed"], order_by=order("meter_label", "start_on"), period="start_on", description="Every pair of hour-meter readings, valid or not. R1 shows no bar " "for an invalid pair."), table("R2", "Hour-meter evidence", "v_hour_meter_evidence", [ "meter_label", "mapped_set", "reading_date", "date_basis", "hours", "reading_state", "epoch_no", "claim_status"], order_by=order("reading_date"), period="reading_date", description="Empty until hour-meter readings exist."), table("R3", "Grid energy (kWh)", "v_kpi_measures", [ "measure", "value", "unit", "value_kind", "note"], filters=[sql_filter("unit = 'kWh'")], description="Empty value: no kWh records. Bills are cost only and are not " "converted to fuel or hours."), bars("R4", "Electricity cost per bill month (AFN, inferred)", "v_electricity_cost", "bill_month", ["amount"], grain="P1M", period="bill_month", y_title="AFN", description="Grid cost only; never converted to fuel or hours."), table("R4T", "Electricity bills", "v_electricity_cost", [ "bill_month", "period_start", "period_end", "amount", "currency", "currency_basis", "kwh", "claim_status", "source_ref"], order_by=order("bill_month"), period="bill_month", description="A disputed bill is listed without amount or kWh."), table("B1", "Baseline versions", "v_baseline_version", [ "code", "metric", "status", "value", "range_low", "range_high", "unit", "window_start", "window_end", "matched_days", "meets_policy", "policy_check", "evidence_grade", "conditions", "approved_on", "approved_by_role", "decision_ref"], order_by=order("window_start"), description="Empty while no baseline version is defined."), table("B2", "Inputs of each version", "v_baseline_input", [ "baseline_code", "interval_label", "included", "exclusion_reason", "fuel_l", "runtime_h", "days", "eligibility_at_computation", "dependencies_changed"], order_by=order("baseline_code", "start_on"), description="Empty until a baseline version exists."), table("B3", "All intervals: eligibility and exclusions", "v_interval", [ "interval_label", "eligibility", "exclusion_reasons", "evidence_grade", "conditions", "metric", "population_key", "in_current_baseline"], order_by=order("start_on")), Chart("B4", "Interval L/h against the approved baseline", "v_interval", "mixed_timeseries", { "x_axis": "interval_label", "time_grain_sqla": None, "metrics": [sql_metric("MAX(l_per_h)", "Interval L/h")], "groupby": [], "adhoc_filters": [sql_filter("eligibility <> 'excluded'")], "row_limit": 10000, "truncate_metric": True, "comparison_type": "values", "metrics_b": [sql_metric("MAX(current_baseline_l_per_h)", "Approved baseline L/h")], "groupby_b": [], "adhoc_filters_b": [sql_filter("baseline_comparable = true")], "row_limit_b": 10000, "truncate_metric_b": True, "comparison_type_b": "values", "seriesType": "bar", "seriesTypeB": "line", "markerEnabledB": True, "markerSizeB": 6, # One y-axis for both (no dual axis). "yAxisIndex": 0, "yAxisIndexB": 0, "annotation_layers": [], "color_scheme": "supersetColors", "show_legend": True, "legendType": "scroll", "legendOrientation": "top", "y_axis_format": "SMART_NUMBER", "rich_tooltip": True, "x_axis_time_format": "%Y-%m-%d", "y_axis_title": "L/h", "y_axis_title_margin": 40, "y_axis_title_position": "Left", }, "Bars only for usable intervals; the line only for an approved baseline of the same " "metric and population. See S1 for the site's status.", ["interval_label", "l_per_h", "eligibility", "current_baseline_l_per_h", "baseline_comparable"]), table("B5", "Assumptions", "v_assumption", [ "code", "topic", "value", "value_low", "value_high", "unit", "applies_from", "applies_to", "basis", "derived_from_fuel", "status", "decision_ref", "rationale"], order_by=order("code")), table("B6", "Missing data and quality checks", "v_data_quality", [ "label", "status", "detail"], order_by=order("check_order")), table("B7", "Evidence references", "v_evidence", [ "title", "kind", "issuer", "document_date", "used_for", "document_key"], order_by=order("document_date")), table("B8", "Workbook calculations, assessed", "v_imported_calculation", [ "source_label", "formula", "cached_value", "unit_as_labelled", "our_reading", "baseline_use", "exclusion_reason", "claim_status", "cell_ref"], order_by=order("cell_ref")), table("B9", "Stock against engine-measured fuel", "v_reconciliation", [ "interval_label", "asset_uid", "engine_l", "covered_h", "interval_h", "placed_throughout", "stock_balance_l", "engine_all_generators_l", "all_generators_metered", "difference_l", "difference_pct", "stock_status", "endpoint_times_known"], order_by=order("start_on"), description="Empty until flow-meter data exist. The difference is shown only " "with full coverage and known visit times."), table("B10", "Equipment specifications: manufacturer and nameplate data, not site performance", "v_spec_reference", [ "slot", "make_model", "attribute", "load_pct", "value", "value_text", "unit", "basis", "applicability", "applicability_note", "claim_status"], order_by=order("slot", "attribute"), description="Manufacturer and nameplate data, not site performance."), table("B11", "Modelled L/h under an assumed runtime: not measured", "v_modelled_lph", [ "assumption_code", "status", "basis", "h_per_day", "h_per_day_low", "h_per_day_high", "window_start", "window_end", "matched_days", "l_per_day", "modelled_l_per_h", "modelled_l_per_h_low", "modelled_l_per_h_high", "value_kind"], order_by=order("assumption_code"), description="Modelled from an assumed runtime, not measured. It never reaches " "S1, the map or the sites list."), ] # Reading notes shown at the top of each tab (SPEC §4 "when data are missing"). TAB_NOTES = { "summary": "**Reading this tab.** L/h comes only from a baseline version; while no runtime " "evidence exists it is *not available*. L/day is per calendar day over the whole " "record, not per running hour. The coverage chart has up to four lanes: Stock " "intervals, Runtime (one per meter), Baseline window, Electricity bills. A " "lane missing from the chart means no coverage span could be formed: no evidence of " "that kind, or too few readings (see Recorded stock and Hour-meter evidence); gaps " "are not interpolated. Missing data and quality checks (Baseline & Evidence) list what " "is missing.", "fuel": "**Reading this tab.** Deliveries are not consumption; the bars add up each day's " "deliveries, and the Deliveries table lists each one, including any of unknown quantity. Recorded stock is shown as points only; nothing is drawn between " "readings. Fuel per calendar day is one bar per observed stock interval, never spread " "over months; a grey bar is an interval whose stock balance is not usable, and its " "status (legend, tooltip, Stock intervals table) names the reason. The Viewing period narrows every chart here (by delivery, reading or " "interval start date).", "runtime": "**Reading this tab.** Runtime and hour-meter evidence stay empty until hour-meter " "readings exist; an invalid reading pair (two epochs, a decreasing timer, over " "24 h/day) shows no bar and is listed with its flags in the pair table. Grid energy without a value means no kWh records. " "Electricity cost is grid cost only (AFN, inferred) and is never converted to fuel " "or hours; a disputed bill is listed without amount or kWh. The Viewing period " "narrows every chart here except grid energy.", "baseline": "**Reading this tab.** Baseline versions and their inputs stay empty until a " "version is proposed. Interval L/h shows bars only for usable intervals, and the " "line only for an approved baseline of the same metric and population. Stock " "against engine-measured fuel stays empty until flow-meter data exist; the " "difference is shown only with full coverage and known visit times. Modelled L/h " "comes from an assumed runtime, is not measured, and never reaches the site's L/h, " "the map or the sites list. The Viewing period does not apply to this tab.", } DETAIL_HEADER = [("D0", 12, 24)] DETAIL_TABS = [ ("summary", "Summary", [ [("MARKDOWN:notes-summary", 12, 17)], [("S1", 12, 30)], [("S2", 12, 60)], [("S3", 12, 50)], ]), ("fuel", "Fuel & Deliveries", [ [("MARKDOWN:notes-fuel", 12, 17)], [("F1", 6, 60), ("F2", 6, 60)], [("F5", 12, 50)], [("F3", 12, 60)], [("F4", 12, 60)], ]), ("runtime", "Runtime & Electricity", [ [("MARKDOWN:notes-runtime", 12, 14)], [("R1", 6, 55), ("R2", 6, 55)], [("R1T", 12, 45)], [("R3", 6, 55), ("COLUMN:R4", 6, [("R4", 50), ("R4T", 50)])], ]), ("baseline", "Baseline & Evidence", [ [("MARKDOWN:notes-baseline", 12, 14)], [("B1", 6, 50), ("B2", 6, 50)], [("B3", 12, 50)], [("B4", 12, 55)], [("B5", 6, 50), ("B6", 6, 50)], [("B7", 6, 50), ("B8", 6, 50)], [("B9", 6, 50), ("B10", 6, 50)], [("B11", 12, 45)], ]), ] # SPEC §4: the Viewing period applies to the time-series charts of these two tabs only. PERIOD_SCOPE = {"F1", "F2", "F3", "F4", "F5", "R1", "R1T", "R2", "R4", "R4T"} def detail_filters(ids: dict[str, int]) -> list[dict[str, Any]]: site = select_filter(SITE_FILTER, "Site", "v_site", "site_label", multi=False, required=True, excluded=[]) # SPEC §4 fallback: "required" alone still lets the charts query every site while # the filter is empty (local test, 24 Oct 2026), so the first site is preselected. site["controlValues"]["defaultToFirstItem"] = True # What the filter editor writes with that option: charts wait for this filter's # value, and the first value is applied without "Apply filters". site["requiredFirst"] = True site["description"] = "Exactly one site. Search by code or name." period = period_filter(ids, PERIOD_SCOPE, "Narrows the fuel, runtime and electricity charts. " "Summary and Baseline & Evidence keep their fixed windows.") return [site, period] def period_filter(ids: dict[str, int], scope: set[str], description: str) -> dict[str, Any]: """The Viewing period, applied only to the charts named in scope.""" return { "id": PERIOD_FILTER, "name": "Viewing period", "filterType": "filter_time", "type": "NATIVE_FILTER", "targets": [{}], "controlValues": {"enableEmptyFilter": False}, "defaultDataMask": {"extraFormData": {}, "filterState": {}, "ownState": {}}, "cascadeParentIds": [], "scope": {"rootPath": ["ROOT_ID"], "excluded": sorted(i for k, i in ids.items() if k not in scope)}, "description": description, } def select_filter(fid: str, name: str, view: str, column: str, *, multi: bool, required: bool, excluded: list[int]) -> dict[str, Any]: return { "id": fid, "name": name, "filterType": "filter_select", "type": "NATIVE_FILTER", "targets": [{"datasetUuid": uid("dataset", view), "column": {"name": column}}], "controlValues": { "enableEmptyFilter": required, "defaultToFirstItem": False, "multiSelect": multi, "searchAllOptions": False, "inverseSelection": False, "creatable": False, "sortAscending": True, }, "defaultDataMask": {"extraFormData": {}, "filterState": {}, "ownState": {}}, "cascadeParentIds": [], # The frontend derives chartsInScope from this; excluded must be a list. "scope": {"rootPath": ["ROOT_ID"], "excluded": excluded}, "description": "", } # --- Dashboards --------------------------------------------------------------------------- class Layout: """Builds a dashboard position (v2) from rows of (key, width, height).""" def __init__(self, title: str, charts: dict[str, Chart]) -> None: self.charts = charts self.ids: dict[str, int] = {} self.nodes: dict[str, Any] = { "DASHBOARD_VERSION_KEY": "v2", "ROOT_ID": {"children": [], "id": "ROOT_ID", "type": "ROOT"}, "GRID_ID": {"children": [], "id": "GRID_ID", "parents": ["ROOT_ID"], "type": "GRID"}, "HEADER_ID": {"id": "HEADER_ID", "meta": {"text": title}, "type": "HEADER"}, } self.nodes["ROOT_ID"]["children"].append("GRID_ID") self.markdown: dict[str, str] = {} def add(self, node_id: str, node_type: str, parent: str, meta: dict[str, Any]) -> str: parents = self.nodes[parent].get("parents", []) + [parent] self.nodes[node_id] = {"children": [], "id": node_id, "meta": meta, "parents": parents, "type": node_type} self.nodes[parent]["children"].append(node_id) return node_id def chart(self, key: str, width: int, height: int, parent: str) -> None: chart = self.charts[key] self.ids[key] = len(self.ids) + 1 # placeholder id; the importer maps it by uuid self.add(f"CHART-{key}", "CHART", parent, { "chartId": self.ids[key], "height": height, "sliceName": chart.title, "uuid": chart.uuid, "width": width}) def rows(self, rows: list[list[tuple]], parent: str, prefix: str) -> None: for n, row in enumerate(rows, 1): if sum(item[1] for item in row) > 12: raise SystemExit(f"layout: row {prefix}-{n} is wider than 12") row_id = self.add(f"ROW-{prefix}-{n}", "ROW", parent, {"background": "BACKGROUND_TRANSPARENT"}) for key, width, rest in row: if key.startswith("MARKDOWN:"): name = key.split(":", 1)[1] self.add(f"MARKDOWN-{name}", "MARKDOWN", row_id, {"width": width, "height": rest, "code": self.markdown[name]}) elif key.startswith("COLUMN:"): col_id = self.add(f"COLUMN-{key.split(':', 1)[1]}", "COLUMN", row_id, {"background": "BACKGROUND_TRANSPARENT", "width": width}) for sub_key, height in rest: self.chart(sub_key, width, height, col_id) else: self.chart(key, width, rest, row_id) def tab(self, tab_id: str, title: str, tabs_id: str) -> str: return self.add(tab_id, "TAB", tabs_id, {"text": title, "defaultText": "Tab title", "placeholder": "Tab title"}) def tabs(self, tabs: list[tuple[str, str, list]], parent: str, tabs_id: str) -> None: tabs_id = self.add(tabs_id, "TABS", parent, {}) for slug, title, rows in tabs: self.rows(rows, self.tab(f"TAB-{slug}", title, tabs_id), slug) def dashboard_yaml(key: str, title: str, slug: str, layout: Layout, metadata: dict[str, Any], css: str = "") -> dict[str, Any]: return { "dashboard_title": title, "description": None, "css": css, "slug": slug, "certified_by": None, "certification_details": None, "published": True, "uuid": uid("dashboard", key), "position": layout.nodes, "metadata": metadata, "version": VERSION, } def base_metadata(layout: Layout, filters: list[dict[str, Any]], cross_filters: bool) -> dict[str, Any]: return { "color_scheme": "supersetColors", "refresh_frequency": 0, # Expanded descriptions never render in 6.1 (the dashboard loads description_markeddown, # the chart holder reads description_markdown), so each tab has a notes block instead. "expanded_slices": {}, "label_colors": dict(STATUS_COLOURS), "shared_label_colors": [], "map_label_colors": {}, "color_scheme_domain": [], "cross_filters_enabled": cross_filters, # Filters in Superset's left column (owner, 29 Oct). Readers don't see it on the Fleet # tab (assets/branding/reader.css), so their map gets the full width. "filter_bar_orientation": "VERTICAL", "native_filter_configuration": filters, "chart_configuration": {}, "global_chart_configuration": {"scope": {"rootPath": ["ROOT_ID"], "excluded": []}}, "timed_refresh_immune_slices": [], "default_filters": "{}", "filter_scopes": {}, } def build(database_uuid: str, columns_path: Path) -> dict[str, Any]: """Return {path inside the bundle: yaml object}.""" views = read_columns(columns_path) files: dict[str, Any] = { "metadata.yaml": {"version": VERSION, "type": "Dashboard", "timestamp": "2026-10-24T00:00:00+00:00"}, "databases/Fuel_Trino.yaml": { "database_name": DATABASE_NAME, "sqlalchemy_uri": DATABASE_URI, "cache_timeout": None, "expose_in_sqllab": True, "allow_run_async": False, "allow_ctas": False, "allow_cvas": False, "allow_dml": False, "allow_file_upload": False, "extra": {"metadata_params": {}, "engine_params": {}, "metadata_cache_timeout": {}, "schemas_allowed_for_file_upload": []}, "impersonate_user": False, "uuid": database_uuid, "version": VERSION, }, } for name, (base, extra, _) in VIRTUAL.items(): views[name] = views[base] + extra for view in sorted(views): files[f"datasets/Fuel_Trino/{view}.yaml"] = dataset_yaml(view, views[view], database_uuid) charts = {c.key: c for c in detail_charts()} # SPEC §2: [Map] [Insights]; the site header and the four detail tabs live in # Insights, so the Map tab shows the map only. layout = Layout("ATOMA", charts) layout.markdown["fleet-map"] = FLEET_MAP layout.markdown.update({f"notes-{slug}": text for slug, text in TAB_NOTES.items()}) main_tabs = layout.add("TABS-main", "TABS", "GRID_ID", {}) layout.rows([[("MARKDOWN:fleet-map", 12, 100)]], layout.tab(FLEET_TAB, FLEET_TAB_TITLE, main_tabs), "fleet") site_tab = layout.tab(SITE_TAB, SITE_TAB_TITLE, main_tabs) layout.rows([DETAIL_HEADER], site_tab, "header") layout.tabs(DETAIL_TABS, site_tab, "TABS-detail") metadata = base_metadata(layout, detail_filters(layout.ids), cross_filters=False) metadata["label_colors"].update(STOCK_STATUS_COLOURS) files["dashboards/ATOMA.yaml"] = dashboard_yaml("atoma", "ATOMA", SLUG, layout, metadata, DASHBOARD_CSS.read_text()) dashboards = [(layout, charts, metadata, (DASHBOARD_CSS, READER_CSS))] # PD-12: the Petrolyst app embeds the Overview and one site dashboard, whose four detail # tabs are Superset's own (owner, 29 Oct). The app shows the site header and chooses site # and period, so these have no D0 and no CSS. overview = {c.key: c for c in overview_charts()} all_charts = {**charts, **overview} for key, title in [(OVERVIEW_SLUG, "Overview"), (SITE_SLUG, "Site")]: page = Layout(f"ATOMA · {title}", all_charts) page.markdown.update({f"notes-{k}": v for k, v in TAB_NOTES.items()}) page.markdown["notes-overview"] = OVERVIEW_NOTES if key == OVERVIEW_SLUG: page.rows(OVERVIEW_ROWS, "GRID_ID", "overview") filters = [period_filter(page.ids, OVERVIEW_PERIOD_SCOPE, "Narrows the deliveries, the costs and the monthly bars.")] else: page.tabs(DETAIL_TABS, "GRID_ID", "TABS-detail") filters = detail_filters(page.ids) placed = {k: all_charts[k] for k in page.ids} page_metadata = base_metadata(page, filters, cross_filters=False) page_metadata["label_colors"].update(STOCK_STATUS_COLOURS) files[f"dashboards/{key}.yaml"] = dashboard_yaml(key, f"ATOMA · {title}", key, page, page_metadata) dashboards.append((page, placed, page_metadata, ())) for chart in all_charts.values(): name = re.sub(r"\W+", "_", f"{chart.key}_{chart.title}").strip("_") files[f"charts/{name}.yaml"] = chart.yaml(uid("dataset", chart.view)) check(files, views, dashboards, overview) return files def check(files: dict[str, Any], views: dict[str, list[tuple[str, str]]], dashboards: list[tuple[Layout, dict[str, Chart], dict[str, Any], tuple[Path, ...]]], overview: dict[str, Chart]) -> None: problems = [] columns = {v: {c for c, _ in cols} | {c[0] for c in CALCULATED.get(v, [])} for v, cols in views.items()} for view in list(MAIN_DTTM) + list(CALCULATED) + list(METRICS): if view not in columns: problems.append(f"{view}: configured but not in the column snapshot") for view, col in MAIN_DTTM.items(): if col not in columns.get(view, set()): problems.append(f"{view}: main time column {col} missing") for layout, charts, metadata, stylesheets in dashboards: for css in stylesheets: for ref in sorted(set(re.findall(r"#([A-Z]+-[\w-]+)", css.read_text()))): if ref not in layout.nodes: problems.append(f"{css.name}: #{ref} is not in the layout") placed = set(layout.ids) if placed != set(charts): problems.append(f"charts not placed or unknown: {sorted(placed ^ set(charts))}") for chart in charts.values(): missing = set(chart.columns_used) - columns.get(chart.view, set()) if missing: problems.append(f"{chart.key}: {sorted(missing)} not in {chart.view}") for f in metadata["native_filter_configuration"]: in_scope = [k for k, i in layout.ids.items() if i not in f["scope"]["excluded"]] for target in f["targets"]: col = target.get("column", {}).get("name") if col is None: continue for key in in_scope: if col not in columns.get(charts[key].view, set()): problems.append(f"{f['id']}: {key} ({charts[key].view}) lacks {col}") # The fleet map's "Site details" link is built from its iframe address. fleet = layout.nodes.get("MARKDOWN-fleet-map") if fleet is not None: filter_ids = {f["id"] for f in metadata["native_filter_configuration"]} for needle, what in [(f"detail=/superset/dashboard/{SLUG}/", "this dashboard"), (f"filter={SITE_FILTER}", "the Site filter"), (f"tab={SITE_TAB}", "the Insights tab"), (f"tab_label={SITE_TAB_TITLE}", "the Insights tab's name")]: if needle not in fleet["meta"]["code"]: problems.append(f"fleet map: its address does not name {what}") if SITE_FILTER not in filter_ids or SITE_TAB not in layout.nodes: problems.append("fleet map: the Site filter or the Insights tab is missing") if not (HERE.parent.parent / "assets" / "branding" / OPERATOR_LOGO).is_file(): problems.append(f"fleet map: assets/branding/{OPERATOR_LOGO} is missing") if metadata.get("filter_bar_orientation") != "VERTICAL": problems.append("fleet map: the filters must sit in the left column") if f'data-node-key="{FLEET_TAB}"' not in READER_CSS.read_text(): problems.append(f"fleet map: reader.css does not hide the filters on {FLEET_TAB}") if fleet["meta"]["width"] != 12: problems.append("fleet map: reader.css stretches it, so it must span all 12 columns") # SPEC §10: the Overview sums recorded values only, and money only per currency. for chart in overview.values(): metrics = chart.params.get("metrics") or [chart.params["metric"]] for metric in metrics: expression = metric["sqlExpression"] if expression not in OVERVIEW_METRICS: problems.append(f"{chart.key}: metric {expression!r} is not an allowed total") elif (OVERVIEW_METRICS[expression] and "currency_text" not in chart.params.get("groupby", [])): problems.append(f"{chart.key}: money must be grouped by currency_text") uuids = [obj["uuid"] for path, obj in files.items() if path != "metadata.yaml"] if len(uuids) != len(set(uuids)): problems.append("duplicate uuids") if problems: raise SystemExit("bundle check failed:\n " + "\n ".join(problems)) def write_zip(files: dict[str, Any], out: Path) -> None: buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as bundle: for path in sorted(files): text = yaml.safe_dump(files[path], sort_keys=False, allow_unicode=True, width=100) info = zipfile.ZipInfo(f"atoma_dashboards/{path}", date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED info.external_attr = 0o644 << 16 bundle.writestr(info, text) out.write_bytes(buffer.getvalue()) def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--database-uuid", required=True, type=uuid.UUID, help='uuid of the "Fuel (Trino)" database in the target Superset') parser.add_argument("--columns", type=Path, default=HERE / "columns.tsv") parser.add_argument("--out", type=Path, default=Path("atoma-dashboards.zip")) args = parser.parse_args(argv) files = build(str(args.database_uuid), args.columns) write_zip(files, args.out) counts = {k: sum(p.startswith(k + "/") for p in files) for k in ("datasets", "charts", "dashboards")} print(f"{args.out}: {counts['dashboards']} dashboards, {counts['charts']} charts, " f"{counts['datasets']} datasets") return 0 if __name__ == "__main__": sys.exit(main())