"""Import the ATOMA dashboard bundle into Superset, overwriting what it defines. Runs inside the Superset container, with the image's Python: docker cp import_bundle.py :/tmp/ && docker cp atoma-dashboards.zip :/tmp/ docker exec /app/.venv/bin/python /tmp/import_bundle.py --show-database docker exec /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip --user `superset import-dashboards` imports a dashboard's datasets and charts with overwrite off, so an existing chart or dataset keeps its old definition. This script runs the dataset, chart and dashboard importers in turn, each with overwrite on (the bundle's metadata.yaml type is set to what each importer expects). A dataset import replaces the column list, so columns missing from the bundle are dropped; afterwards each dataset's columns are compared with the live view and any drift is reported (exit status 2). The database is never created or changed: the bundle's database uuid must already exist under the name "Fuel (Trino)", otherwise nothing is imported. A bundle cannot carry a dashboard's roles, so afterwards the role "ATOMA Reader" is attached to the ATOMA dashboard: with DASHBOARD_RBAC that is how ATOMA's readers see it. The role comes from roles/atoma_reader.py (run by superset-init); without it nothing is imported. The Petrolyst app's dashboards (PD-12: atoma-overview, atoma-site) get no role: the app shows them with guest passes. With --embedded-out FILE --embed-domain URL, each of them is made embeddable from that origin (its embedding id stays the same across imports), and FILE receives {slug: embedding id} for the app service: docker exec /app/.venv/bin/python /tmp/import_bundle.py /tmp/atoma-dashboards.zip \ --user --embedded-out /tmp/embedded-dashboards.json --embed-domain http://127.0.0.1:8098 """ from __future__ import annotations import argparse import json import sys import zipfile import yaml DATABASE_NAME = "Fuel (Trino)" READER_ROLE = "ATOMA Reader" READER_SLUG = "atoma" # the readers' dashboard; the others are the app's (PD-12) EMBEDDED_PREFIX = "atoma-" STEPS = [("datasets", "SqlaTable"), ("charts", "Slice"), ("dashboards", "Dashboard")] def main() -> int: parser = argparse.ArgumentParser(description="Import the ATOMA dashboard bundle.") parser.add_argument("bundle", nargs="?") parser.add_argument("--user", help="existing Superset user who will own new objects") parser.add_argument("--show-database", action="store_true", help=f'print the uuid of "{DATABASE_NAME}" and stop') parser.add_argument("--embedded-out", help="write {slug: embedding id} of the app's dashboards") parser.add_argument("--embed-domain", help="the app's origin, e.g. http://127.0.0.1:8098") args = parser.parse_args() if not args.show_database and not (args.bundle and args.user): parser.error("give a bundle and --user, or --show-database") if bool(args.embedded_out) != bool(args.embed_domain): parser.error("--embedded-out and --embed-domain go together") from superset.app import create_app app = create_app() with app.app_context(): from superset import db, security_manager from superset.commands.chart.importers.v1 import ImportChartsCommand from superset.commands.dashboard.importers.v1 import ImportDashboardsCommand from superset.commands.dataset.importers.v1 import ImportDatasetsCommand from superset.commands.importers.v1.utils import get_contents_from_bundle from superset.connectors.sqla.models import SqlaTable from superset.models.core import Database from superset.models.dashboard import Dashboard from superset.utils.core import override_user if args.show_database: found = db.session.query(Database).filter_by(database_name=DATABASE_NAME).all() for database in found: print(database.uuid) return 0 if found else 1 user = security_manager.find_user(username=args.user) if user is None: print(f"no Superset user {args.user!r}", file=sys.stderr) return 1 reader_role = security_manager.find_role(READER_ROLE) if reader_role is None: print(f'no role "{READER_ROLE}": run roles/atoma_reader.py first', file=sys.stderr) return 1 with zipfile.ZipFile(args.bundle) as bundle: contents = get_contents_from_bundle(bundle) databases = [yaml.safe_load(t) for p, t in contents.items() if p.startswith("databases/")] for config in databases: database = db.session.query(Database).filter_by(uuid=config["uuid"]).one_or_none() if database is None or database.database_name != DATABASE_NAME: print(f'database {config["uuid"]} is not "{DATABASE_NAME}" here; ' "rebuild the bundle with the uuid from --show-database", file=sys.stderr) return 1 metadata = yaml.safe_load(contents["metadata.yaml"]) commands = {"datasets": ImportDatasetsCommand, "charts": ImportChartsCommand, "dashboards": ImportDashboardsCommand} with override_user(user): for step, model in STEPS: step_contents = dict(contents) step_contents["metadata.yaml"] = yaml.safe_dump({**metadata, "type": model}) commands[step](step_contents, overwrite=True).run() count = sum(p.startswith(step + "/") for p in contents) print(f"{step}: {count} imported (overwrite)") embedded: dict[str, str] = {} for path, text in contents.items(): if path.startswith("dashboards/"): dashboard = db.session.query(Dashboard).filter_by( uuid=yaml.safe_load(text)["uuid"]).one() if dashboard.slug == READER_SLUG: if reader_role not in dashboard.roles: dashboard.roles.append(reader_role) elif dashboard.slug.startswith(EMBEDDED_PREFIX) and args.embedded_out: from superset.daos.dashboard import EmbeddedDashboardDAO config = EmbeddedDashboardDAO.upsert(dashboard, [args.embed_domain]) db.session.flush() embedded[dashboard.slug] = str(config.uuid) db.session.commit() print(f'dashboards: "{READER_SLUG}" readable for "{READER_ROLE}"') if args.embedded_out: with open(args.embedded_out, "w") as out: json.dump(dict(sorted(embedded.items())), out, indent=2) print(f"embedded: {len(embedded)} dashboard(s) for {args.embed_domain}, " f"ids in {args.embedded_out}") drift = 0 dataset_uuids = [yaml.safe_load(t)["uuid"] for p, t in contents.items() if p.startswith("datasets/")] for dataset_uuid in dataset_uuids: dataset = db.session.query(SqlaTable).filter_by(uuid=dataset_uuid).one() physical = {c.column_name for c in dataset.columns if not c.expression} live = {c["column_name"] for c in dataset.external_metadata()} if physical != live: drift += 1 print(f"drift {dataset.table_name}: only in bundle {sorted(physical - live)}, " f"only in view {sorted(live - physical)}") print(f"column drift: {drift} dataset(s)") return 2 if drift else 0 if __name__ == "__main__": sys.exit(main())