feat(demo): privy - Fabric Livy without the throttling (POC, do NOT merge)

.devcontainer/overlay/install-python-tooling.sh

@@ -5,7 +5,7 @@
 # changes:
 #
 
-VERSION="1.13.0" # "latest"
+VERSION="1.13.1" # "latest"
 INSTALL_PATH="/usr/local/bin"
 RUNNER_OS="Linux"
 RUNNER_ARCH="X64"

.gitignore

@@ -337,5 +337,8 @@ node_modules/
 
 # nx
 .nx/*
-# Livy session handle written by the adapter when no session_id_file is configured
-livy-session-id.txt
+# Livy session handle written by the adapter when no session_id_file is configured
+livy-session-id.txt
+# Privy notebook job cache written by the adapter (privysession.py) to reuse
+# a triggered notebook run across separate dbt invocations
+privy-notebook-job.json

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.13.1
+
+- POC hack, do NOT merge
+
 ## v1.13.0
 
 ### Features

demo/README.md

@@ -0,0 +1,63 @@
+# Privy demo (dbt-fabricspark)
+
+Proves `method: privy` can run `SELECT 1` against a Fabric notebook over
+Azure Relay, using this branch's adapter wheel.
+
+## 1. Start the notebook manually
+
+Auto-start is flaky right now (Fabric session errors), so start it by hand:
+
+1. Open the notebook: value of `PRIVAY_NOTEBOOK_URL` in `test.env`.
+2. Run all cells. Wait until the `RelayServer(...).serve_forever()` cell
+   shows a running spinner (it never finishes — that's expected).
+
+`privy_auto_start_notebook: false` is already set in `demo/profiles.yml` so
+dbt won't try to trigger a run itself.
+
+## 2. Build & install the wheel
+
+```bash
+cd /workspaces/dbt-fabricspark
+uv build
+python3 -m venv demo/.venv
+demo/.venv/bin/pip install "$(ls dist/dbt_fabricspark-*-py3-none-any.whl)"
+```
+
+## 3. Run dbt
+
+```bash
+cd /workspaces/dbt-fabricspark
+set -a; source test.env; set +a
+cd demo
+../demo/.venv/bin/dbt debug --profiles-dir .
+../demo/.venv/bin/dbt run --profiles-dir .
+../demo/.venv/bin/dbt show --inline "select 1 as one" --profiles-dir .
+```
+
+`dbt debug` should show a successful Privy relay connection (no notebook
+trigger, since auto-start is off). `dbt run` builds `models/hello_privy.sql`
+(`select 1 as id`) as a view.
+
+**Verified output:**
+```
+$ dbt show --inline "select 1 as one"
+| one |
+| --- |
+|   1 |
+
+$ dbt run
+1 of 1 OK created sql view model dbo.hello_privy ... [OK in 2.45s]
+```
+
+`schema: dbo` must be a schema that already exists in whatever lakehouse the
+notebook is attached to — it's not related to the `privy_*` settings. If you
+get `SCHEMA_NOT_FOUND`, run `SHOW SCHEMAS` in the notebook to find a valid one.
+
+## Known issues
+
+- Notebook auto-start (`privy_auto_start_notebook: true`) currently fails
+  server-side after ~15s (`System_Cancelled_Session_Statements_Failed`),
+  suspected cause: the notebook's `%pip install --force-reinstall` step
+  clobbering packages the Fabric kernel relies on. Manual start avoids it.
+- SPN auth may not be able to trigger notebook runs (Fabric API limitation) —
+  use CLI auth (`az login`) for now.

demo/dbt_project.yml

@@ -0,0 +1,16 @@
+name: "privy_demo"
+version: "1.0.0"
+config-version: 2
+
+profile: "privy_demo"
+
+model-paths: ["models"]
+
+target-path: "target"
+clean-targets:
+  - "target"
+  - "logs"
+
+models:
+  privy_demo:
+    materialized: view

demo/models/hello_privy.sql

@@ -0,0 +1 @@
+select 1 as id

demo/profiles.yml

@@ -0,0 +1,19 @@
+privy_demo:
+  target: privy
+  outputs:
+    privy:
+      type: fabricspark
+      method: privy
+      privy_relay_namespace: "{{ env_var('PRIVY_RELAY_NAMESPACE') }}"
+      privy_relay_path: "{{ env_var('PRIVY_RELAY_PATH') }}"
+      privy_relay_keyrule: "{{ env_var('PRIVY_RELAY_KEYRULE') }}"
+      privy_relay_key: "{{ env_var('PRIVY_RELAY_KEY') }}"
+      privy_notebook_url: "{{ env_var('PRIVY_NOTEBOOK_URL') }}"
+      endpoint: "{{ env_var('FABRIC_ENDPOINT', 'https://api.fabric.microsoft.com/v1') }}"
+      authentication: "{{ env_var('FABRIC_AUTH_METHOD', 'CLI') }}"
+      schema: dbo
+      threads: 1
+      privy_auto_start_notebook: true
+      privy_ready_timeout: 900
+      spark_config:
+        name: "dbt-privy-demo"

pyproject.toml

@@ -39,6 +39,7 @@ dependencies = [
     "azure-identity>=1.21.0",
     "azure-core>=1.33.0",
     "requests>=2.32.0",
+    "privy @ https://rakirahman.blob.core.windows.net/public/whls/privy-0.1.0-py3-none-any.whl",
 ]
 
 [project.optional-dependencies]
@@ -68,6 +69,9 @@ dev = [
 default-groups = "all"
 prerelease = "allow"
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [project.urls]
 homepage = "https://github.com/microsoft/dbt-fabricspark"
 "Setup & configuration" = "https://docs.getdbt.com/reference/warehouse-profiles/fabricspark-profile"

src/dbt/adapters/fabricspark/__version__.py

@@ -1 +1 @@
-version = "1.13.0"
+version = "1.13.1"

src/dbt/adapters/fabricspark/connections.py

@@ -37,6 +37,7 @@
     LivySessionManager,
     get_lakehouse_properties,
 )
+from dbt.adapters.fabricspark.privysession import PrivyConnectionManager, PrivyConnectionWrapper
 from dbt.adapters.fabricspark.relation import FabricSparkRelation
 from dbt.adapters.sql import SQLConnectionManager
 
@@ -108,6 +109,7 @@ def render_spark_type(type_code: Any) -> str:
 
 class FabricSparkConnectionMethod(StrEnum):
     LIVY = "livy"
+    PRIVY = "privy"
 
 
 class FabricSparkConnectionWrapper(ABC):
@@ -228,7 +230,7 @@ def open(cls, connection: Connection) -> Connection:
         handle: FabricSparkConnectionWrapper = None
 
         # Fetch lakehouse properties and detect schema support (Fabric mode only).
-        if not creds.is_local_mode:
+        if not creds.is_local_mode and not creds.is_privy_mode:
             lakehouse_props = get_lakehouse_properties(creds)
             creds.apply_lakehouse_properties(lakehouse_props)
 
@@ -258,6 +260,11 @@ def open(cls, connection: Connection) -> Connection:
                     )
                     connection.state = ConnectionState.OPEN
 
+                elif creds.method == FabricSparkConnectionMethod.PRIVY:
+                    raw_handle = PrivyConnectionManager.connect(creds)
+                    handle = PrivyConnectionWrapper(raw_handle, creds)
+                    connection.state = ConnectionState.OPEN
+
                 else:
                     raise DbtConfigError(f"invalid credential method: {creds.method}")
                 break

src/dbt/adapters/fabricspark/credentials.py

@@ -124,6 +124,31 @@ class FabricSparkCredentials(Credentials):
     # ``DBT_FABRICSPARK_SKIP_OPTIMIZE`` environment variable disables it outright.
     auto_optimize: bool = True
 
+    # --- Privy connection method (experimental) ---------------------------
+    # ``method: privy`` sends statements to a Fabric notebook (running
+    # ``privy.RelayServer``) over an Azure Relay Hybrid Connection instead of
+    # the Livy REST API. ``privy`` is bundled as a core dependency of this
+    # package. Exempt from the workspaceid/lakehouseid/lakehouse requirements
+    # below (like local mode) — only these fields are needed.
+    privy_relay_namespace: Optional[str] = None
+    privy_relay_path: Optional[str] = None
+    privy_relay_keyrule: Optional[str] = None
+    privy_relay_key: Optional[str] = None
+    # Browser URL of the Fabric notebook hosting the RelayServer, e.g.
+    # https://<host>/groups/<workspaceId>/synapsenotebooks/<notebookId>. The
+    # workspace and notebook GUIDs are parsed out of this URL to trigger the
+    # notebook via the Fabric Job Scheduler API when the relay is unreachable.
+    privy_notebook_url: Optional[str] = None
+    # When True (default), the adapter tries to start the notebook (via the
+    # Fabric REST API) if the relay doesn't respond to a health check. Set to
+    # False to manage the run yourself and have the adapter only act as a
+    # Privy client.
+    privy_auto_start_notebook: bool = True
+    # Max seconds to keep pinging the relay for readiness (after an auto-start
+    # trigger, or while waiting for a manually-started run) before giving up.
+    # Independent of session_start_timeout (which is Livy-session-specific).
+    privy_ready_timeout: int = 900
+
     def __repr__(self) -> str:
         """Mask sensitive fields in repr to prevent credential leakage in logs/tracebacks."""
         return (
@@ -137,6 +162,11 @@ def __repr__(self) -> str:
             f"credential_class={self.credential_class!r}, "
             f"credential_kwargs_keys={sorted(map(str, self.credential_kwargs.keys()))!r}, "
             f"workspace_name={self.workspace_name!r}, "
+            f"privy_relay_namespace={self.privy_relay_namespace!r}, "
+            f"privy_relay_path={self.privy_relay_path!r}, "
+            f"privy_notebook_url={self.privy_notebook_url!r}, "
+            f"privy_ready_timeout={self.privy_ready_timeout!r}, "
+            f"privy_relay_key='***', "
             f"accessToken='***')"
         )
 
@@ -157,6 +187,10 @@ def __pre_deserialize__(cls, data: Any) -> Any:
     def is_local_mode(self) -> bool:
         return self.livy_mode == "local"
 
+    @property
+    def is_privy_mode(self) -> bool:
+        return self.method == "privy"
+
     @property
     def resolved_session_id_file(self) -> str:
         if self.session_id_file:
@@ -173,8 +207,10 @@ def __post_init__(self) -> None:
         if self.method is None:
             raise DbtRuntimeError("Must specify `method` in profile")
 
-        # Fabric-specific validations
-        if not self.is_local_mode:
+        # Fabric-specific validations. Skipped for local mode and for privy
+        # mode — privy talks to whatever lakehouse the notebook is already
+        # attached to, so it has no use for workspaceid/lakehouseid/lakehouse.
+        if not self.is_local_mode and not self.is_privy_mode:
             if self.endpoint is None:
                 raise DbtRuntimeError("Must specify `endpoint` in profile for Fabric mode")
             if self.workspaceid is None:
@@ -184,25 +220,48 @@ def __post_init__(self) -> None:
             if self.lakehouse is None:
                 raise DbtRuntimeError("Must specify `lakehouse` in profile for Fabric mode")
 
+        if self.is_privy_mode:
+            if not self.privy_relay_namespace:
+                raise DbtRuntimeError(
+                    "Must specify `privy_relay_namespace` in profile for method=privy"
+                )
+            if not self.privy_relay_path:
+                raise DbtRuntimeError(
+                    "Must specify `privy_relay_path` in profile for method=privy"
+                )
+            if not self.privy_relay_keyrule:
+                raise DbtRuntimeError(
+                    "Must specify `privy_relay_keyrule` in profile for method=privy"
+                )
+            if not self.privy_relay_key:
+                raise DbtRuntimeError("Must specify `privy_relay_key` in profile for method=privy")
+            if not self.privy_notebook_url:
+                raise DbtRuntimeError(
+                    "Must specify `privy_notebook_url` in profile for method=privy"
+                )
+
         # schema defaults to lakehouse name if not provided by user.
         # For schema-enabled lakehouses, user can override this in profiles.yml.
-        # For local mode without lakehouse, defaults to "default" (Spark's default database).
+        # For local/privy mode without lakehouse, defaults to "default" (Spark's default database).
         if self.schema is None:
             if self.lakehouse is not None:
                 self.schema = self.lakehouse
-            elif self.is_local_mode:
+            elif self.is_local_mode or self.is_privy_mode:
                 self.schema = "default"
 
         # database is always set to lakehouse name for relation rendering.
         # In non-schema mode, include_policy.database=False excludes it from SQL.
         # In schema-enabled mode, include_policy.database=True renders three-part names.
-        # For local mode without lakehouse, defaults to "default".
+        # For local/privy mode without lakehouse, defaults to "default".
         if self.lakehouse is not None:
             self.database = self.lakehouse
-        elif self.is_local_mode:
+        elif self.is_local_mode or self.is_privy_mode:
             self.database = "default"
 
-        # Security validations (Fabric mode only)
+        # Security validations (Fabric mode only). Privy still makes one real
+        # Fabric API call (triggering the notebook run), so endpoint/UUID
+        # validation stays active for it; workspaceid/lakehouseid are simply
+        # None for privy and _validate_uuid tolerates that.
         if not self.is_local_mode:
             self._validate_uuid(self.workspaceid, "workspaceid")
             self._validate_uuid(self.lakehouseid, "lakehouseid")
@@ -285,6 +344,8 @@ def type(self) -> str:
     def unique_field(self) -> str:
         if self.is_local_mode:
             return self.livy_url
+        if self.is_privy_mode:
+            return f"{self.privy_relay_namespace}/{self.privy_relay_path}"
         return self.lakehouseid
 
     def _validate_endpoint(self) -> None:
@@ -315,7 +376,8 @@ def _validate_uuid(self, value: Optional[str], field_name: str) -> None:
             )
 
     def _connection_keys(self) -> Tuple[str, ...]:
-        # Intentionally excludes client_secret, accessToken, tenant_id
+        # Intentionally excludes client_secret, accessToken, tenant_id,
+        # privy_relay_key, privy_relay_keyrule
         return (
             "workspaceid",
             "lakehouseid",
@@ -327,4 +389,10 @@ def _connection_keys(self) -> Tuple[str, ...]:
             "auto_optimize",
             "high_concurrency",
             "spark_config",
+            "method",
+            "privy_relay_namespace",
+            "privy_relay_path",
+            "privy_notebook_url",
+            "privy_auto_start_notebook",
+            "privy_ready_timeout",
         )