feat(kernel): thread Azure Entra auth (U2M + SP M2M) through the kernel path

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Release History
 
+## Unreleased
+
+- Kernel backend (`useKernel: true`): **Azure Entra (Azure AD) auth is now threaded through the kernel path.** On `authType: 'databricks-oauth'`: **U2M** (no secret) always routes to `OAuthU2m` — the kernel runs one cloud-blind in-house workspace-federated browser flow (it uses the workspace's OIDC-discovered authorize endpoint verbatim), which works against Azure workspaces, so Azure U2M forwards the in-house app (`databricks-sql-connector`) + `sql offline_access` scopes exactly like AWS/GCP, regardless of `useDatabricksOAuthInAzure` (verified E2E against a live Azure workspace). **M2M** (secret): `useDatabricksOAuthInAzure: true` (or non-Azure) → `OAuthM2m` (workspace-OIDC client-credentials); an Azure host with `useDatabricksOAuthInAzure` absent/`false` → the Entra-direct Azure service-principal M2M (`AzureSpM2m`, the Entra SP creds ride `oauthClientId`/`oauthClientSecret`, `azureTenantId` optional and auto-discovered when omitted). On a non-Azure host `useDatabricksOAuthInAzure` is inert. The `AzureSpM2m` path requires a `databricks-sql-kernel` native module that exposes the Azure SP surface ([databricks-sql-kernel#280](https://github.com/databricks/databricks-sql-kernel/pull/280)); U2M works on any kernel build. (PECOBLR-4141 / PECOBLR-4120)
+
 ## 2.0.0
 
 **Breaking changes — completes the security cleanup that 1.17.0 could not do without breaking changes.**

KERNEL_REV

@@ -1 +1 @@
-eff8950428f4e6cc9975c663ec919f334962f7d0
+1495174dac3aae39f23c34040fe639c54a3c3f35

lib/kernel/KernelAuth.ts

@@ -252,6 +252,14 @@ export type KernelNativeConnectionOptions = KernelSessionDefaults &
         oauthScopes?: Array<string>;
         oauthClientId?: string;
       }
+    | {
+        hostName: string;
+        httpPath: string;
+        authMode: 'AzureSpM2m';
+        azureClientId: string;
+        azureClientSecret: string;
+        azureTenantId?: string;
+      }
   );
 
 function prependSlash(str: string): string {
@@ -261,6 +269,28 @@ function prependSlash(str: string): string {
   return str;
 }
 
+/**
+ * Azure Databricks host suffixes — the superset the Thrift driver's
+ * `OAuthManager.getManager` recognises (`.azuredatabricks.net`,
+ * `.databricks.azure.us`, `.databricks.azure.cn`). Used to decide whether an
+ * OAuth connection is on Azure and therefore subject to the in-house-vs-
+ * Entra-direct split.
+ */
+const AZURE_HOST_SUFFIXES = ['.azuredatabricks.net', '.databricks.azure.us', '.databricks.azure.cn'];
+
+/**
+ * True when `host` is an Azure Databricks workspace host. Normalises the input
+ * the same way `getManager` does (lowercase, strip scheme + any path) so a
+ * caller passing a bare host or a full URL is treated identically.
+ */
+function isAzureHost(host: string): boolean {
+  const normalized = host
+    .toLowerCase()
+    .replace(/^https?:\/\//, '')
+    .split('/')[0];
+  return AZURE_HOST_SUFFIXES.some((suffix) => normalized.endsWith(suffix));
+}
+
 /**
  * Reject inputs that pass `typeof === 'string' && length > 0` but are
  * structurally useless as credentials: whitespace-only strings, and the
@@ -481,11 +511,25 @@ export function buildKernelHttpOptions(options: ConnectionOptions): KernelHttpOp
  *     binding makes them, happen below the TypeScript layer and are not
  *     observable from this repo.
  *
+ * Azure (Entra) on the OAuth path. The kernel runs a single, cloud-blind
+ * in-house U2M flow and workspace-OIDC M2M; only Entra-direct **M2M** gets a
+ * dedicated kernel mode:
+ *   - **U2M (no secret), any cloud, any `useDatabricksOAuthInAzure`** →
+ *     `OAuthU2m`. The kernel uses the workspace's OIDC-discovered authorize
+ *     endpoint (`{host}/oidc/v1/authorize`) verbatim; that in-house
+ *     workspace-federated flow works against Azure workspaces too (they federate
+ *     the browser login to Entra server-side — verified E2E). So Azure U2M is
+ *     NOT special-cased and NOT rejected — it forwards the in-house app
+ *     (`databricks-sql-connector`) + `sql offline_access`, exactly like AWS/GCP.
+ *   - **M2M (secret) with `useDatabricksOAuthInAzure: true`** (or non-Azure) →
+ *     `OAuthM2m` (workspace-OIDC client-credentials).
+ *   - **M2M (secret) on an Azure host with `useDatabricksOAuthInAzure` absent/
+ *     `false`** (Entra-direct) → Azure service-principal M2M (`AzureSpM2m`); the
+ *     Entra SP creds ride `oauthClientId`/`oauthClientSecret`, `azureTenantId`
+ *     optional (kernel auto-discovers).
+ *   - On a non-Azure host `useDatabricksOAuthInAzure` is inert.
+ *
  * Out of scope on the OAuth paths (rejected with a clear error):
- *   - `azureTenantId` / `useDatabricksOAuthInAzure` → Microsoft Entra
- *     direct flow. The kernel uses workspace-OIDC discovery (which works
- *     against Azure workspaces too — they serve `/oidc/.well-known/...`)
- *     and does not implement the Entra-direct scope-rewrite path.
  *   - `persistence` on M2M → M2M tokens are not cached (re-issuing is
  *     cheap; no refresh token).
  *   - `persistence` on U2M → custom token store is a parity gap;
@@ -499,7 +543,7 @@ export function buildKernelHttpOptions(options: ConnectionOptions): KernelHttpOp
  *
  * Throws:
  *   - `AuthenticationError` for missing/blank required credentials.
- *   - `HiveDriverError` for unsupported auth modes / Azure-direct /
+ *   - `HiveDriverError` for unsupported auth modes / Entra-direct U2M /
  *     custom persistence / ambiguous combinations.
  */
 /**
@@ -667,12 +711,46 @@ export function buildKernelConnectionOptions(options: ConnectionOptions): Kernel
       );
     }
 
-    if (oauth.azureTenantId !== undefined || oauth.useDatabricksOAuthInAzure === true) {
-      throw new HiveDriverError(
-        'kernel backend: Azure-direct OAuth (azureTenantId / useDatabricksOAuthInAzure) ' +
-          'is not supported. The workspace-OIDC discovery path handles Azure workspaces ' +
-          'today without these options.',
-      );
+    // Azure Entra-direct **M2M** → the kernel's dedicated azure-sp-m2m. Mirroring
+    // the Thrift driver's `OAuthManager.getManager`, an Azure host with
+    // `useDatabricksOAuthInAzure` NOT set to true (the Entra-direct default) plus a
+    // secret is an Entra service-principal client-credentials flow: the Entra SP
+    // credentials ride the generic `oauthClientId` / `oauthClientSecret` (Thrift
+    // convention); forward them as `azureClientId` / `azureClientSecret`.
+    // `azureTenantId` is optional — the kernel auto-discovers it from the workspace
+    // `/aad/auth` redirect when omitted.
+    //
+    // Azure **U2M** is deliberately NOT special-cased and NOT rejected. The kernel
+    // runs a single, cloud-blind in-house U2M flow: it uses the workspace's
+    // OIDC-discovered authorize endpoint (`{host}/oidc/v1/authorize`) verbatim, and
+    // that in-house workspace-federated flow works against Azure workspaces (the
+    // workspace federates the browser login to Entra server-side; verified E2E). So
+    // ALL U2M — including Azure, with or without `useDatabricksOAuthInAzure` — falls
+    // through to the standard `OAuthU2m` path below, which forwards the in-house app
+    // (`databricks-sql-connector`) + `sql offline_access` scopes, exactly like
+    // AWS/GCP. Handing the kernel the Thrift Azure Entra-direct app / scope instead
+    // would derail its in-house flow to a broken AAD authorize URL.
+    // The `oauthClientSecret !== undefined` check is inline (not extracted to a
+    // const) so TypeScript narrows it to `string` for the AzureSpM2m literal below.
+    if (
+      isAzureHost(options.host) &&
+      oauth.useDatabricksOAuthInAzure !== true &&
+      oauth.oauthClientSecret !== undefined
+    ) {
+      const azureClientId = oauth.oauthClientId;
+      if (azureClientId === undefined) {
+        throw new HiveDriverError(
+          'kernel backend: Azure service-principal M2M requires `oauthClientId` (the Entra ' +
+            'app-registration client id) alongside `oauthClientSecret`.',
+        );
+      }
+      const azure = {
+        ...base,
+        authMode: 'AzureSpM2m' as const,
+        azureClientId,
+        azureClientSecret: oauth.oauthClientSecret,
+      };
+      return oauth.azureTenantId !== undefined ? { ...azure, azureTenantId: oauth.azureTenantId } : azure;
     }
 
     // Flow selector + client-id resolution mirror the Thrift driver EXACTLY

tests/unit/kernel/auth-m2m.test.ts

@@ -127,23 +127,56 @@ describe('KernelAuth + KernelBackend — OAuth M2M auth flow', () => {
       expect((native as { oauthClientId?: string }).oauthClientId).to.equal('client-uuid');
     });
 
-    it('rejects azureTenantId with a clear Entra-direct-out-of-scope error', () => {
+    it('routes Azure host + secret (Entra-direct default) to azure-sp-m2m, forwarding the tenant', () => {
+      // On an Azure host with no `useDatabricksOAuthInAzure`, the default is the
+      // Entra-direct flow — for M2M that is the kernel's azure-sp-m2m. The Entra
+      // SP creds ride oauthClientId/oauthClientSecret; azureTenantId is forwarded
+      // (optional — the kernel auto-discovers it when omitted).
       const opts: ConnectionOptions = {
         host: 'adb-12345.0.azuredatabricks.net',
         path: '/sql/1.0/warehouses/abc',
         authType: 'databricks-oauth',
-        oauthClientId: 'client-uuid',
-        oauthClientSecret: 'dose-fake-secret',
+        oauthClientId: 'entra-app-id',
+        oauthClientSecret: 'entra-secret',
         azureTenantId: 'tenant-uuid',
       };
 
-      expect(() => buildKernelConnectionOptions(opts)).to.throw(
-        HiveDriverError,
-        /Azure-direct OAuth.*is not supported/,
-      );
+      const native = buildKernelConnectionOptions(opts);
+      expectNativeConnectionOptions(native, {
+        hostName: 'adb-12345.0.azuredatabricks.net',
+        httpPath: '/sql/1.0/warehouses/abc',
+        intervalsAsString: true,
+        authMode: 'AzureSpM2m',
+        azureClientId: 'entra-app-id',
+        azureClientSecret: 'entra-secret',
+        azureTenantId: 'tenant-uuid',
+      });
+    });
+
+    it('routes Azure host + secret without a tenant to azure-sp-m2m (kernel auto-discovers)', () => {
+      const opts: ConnectionOptions = {
+        host: 'adb-12345.0.azuredatabricks.net',
+        path: '/sql/1.0/warehouses/abc',
+        authType: 'databricks-oauth',
+        oauthClientId: 'entra-app-id',
+        oauthClientSecret: 'entra-secret',
+      };
+
+      const native = buildKernelConnectionOptions(opts);
+      expectNativeConnectionOptions(native, {
+        hostName: 'adb-12345.0.azuredatabricks.net',
+        httpPath: '/sql/1.0/warehouses/abc',
+        intervalsAsString: true,
+        authMode: 'AzureSpM2m',
+        azureClientId: 'entra-app-id',
+        azureClientSecret: 'entra-secret',
+      });
     });
 
-    it('rejects useDatabricksOAuthInAzure with the same Entra-direct error', () => {
+    it('routes Azure host + useDatabricksOAuthInAzure:true + secret to in-house OAuthM2m', () => {
+      // `useDatabricksOAuthInAzure: true` opts into the in-house
+      // (workspace-federated) flow — for M2M that is the kernel's generic
+      // workspace-OIDC client-credentials (OAuthM2m), which works on Azure hosts.
       const opts: ConnectionOptions = {
         host: 'adb-12345.0.azuredatabricks.net',
         path: '/sql/1.0/warehouses/abc',
@@ -153,10 +186,16 @@ describe('KernelAuth + KernelBackend — OAuth M2M auth flow', () => {
         useDatabricksOAuthInAzure: true,
       };
 
-      expect(() => buildKernelConnectionOptions(opts)).to.throw(
-        HiveDriverError,
-        /Azure-direct OAuth.*is not supported/,
-      );
+      const native = buildKernelConnectionOptions(opts);
+      expectNativeConnectionOptions(native, {
+        hostName: 'adb-12345.0.azuredatabricks.net',
+        httpPath: '/sql/1.0/warehouses/abc',
+        intervalsAsString: true,
+        authMode: 'OAuthM2m',
+        oauthClientId: 'client-uuid',
+        oauthClientSecret: 'dose-fake-secret',
+        oauthScopes: ['all-apis'],
+      });
     });
 
     it('rejects a `persistence` hook on M2M (no cache needed)', () => {

tests/unit/kernel/auth-u2m.test.ts

@@ -100,32 +100,51 @@ describe('KernelAuth + KernelBackend — OAuth U2M auth flow', () => {
       expect(native.httpPath).to.equal('/sql/1.0/warehouses/abc');
     });
 
-    it('rejects azureTenantId on the U2M path with the Entra-direct error', () => {
+    it('routes Azure U2M (no secret, no useDatabricksOAuthInAzure) to in-house OAuthU2m', () => {
+      // Azure U2M is NOT rejected and NOT special-cased: the kernel runs a single
+      // cloud-blind in-house workspace-federated U2M flow (it uses the workspace's
+      // OIDC-discovered authorize endpoint verbatim), which works against Azure
+      // workspaces. So it routes to OAuthU2m with the in-house app +
+      // sql/offline_access regardless of useDatabricksOAuthInAzure. azureTenantId
+      // is inert on the kernel U2M path.
       const opts: ConnectionOptions = {
         host: 'adb-12345.0.azuredatabricks.net',
         path: '/sql/1.0/warehouses/abc',
         authType: 'databricks-oauth',
         azureTenantId: 'tenant-uuid',
       };
 
-      expect(() => buildKernelConnectionOptions(opts)).to.throw(
-        HiveDriverError,
-        /Azure-direct OAuth.*is not supported/,
-      );
+      const native = buildKernelConnectionOptions(opts);
+      expectNativeConnectionOptions(native, {
+        hostName: 'adb-12345.0.azuredatabricks.net',
+        httpPath: '/sql/1.0/warehouses/abc',
+        intervalsAsString: true,
+        authMode: 'OAuthU2m',
+        oauthRedirectPort: 8030,
+        oauthScopes: ['sql', 'offline_access'],
+      });
     });
 
-    it('rejects useDatabricksOAuthInAzure on the U2M path', () => {
+    it('routes Azure host + useDatabricksOAuthInAzure:true (no secret) to in-house OAuthU2m', () => {
+      // `useDatabricksOAuthInAzure: true` opts into the in-house
+      // (workspace-federated) browser flow, which the kernel runs against Azure
+      // Databricks workspaces — so this is the U2M happy path, not a rejection.
       const opts: ConnectionOptions = {
         host: 'adb-12345.0.azuredatabricks.net',
         path: '/sql/1.0/warehouses/abc',
         authType: 'databricks-oauth',
         useDatabricksOAuthInAzure: true,
       };
 
-      expect(() => buildKernelConnectionOptions(opts)).to.throw(
-        HiveDriverError,
-        /Azure-direct OAuth.*is not supported/,
-      );
+      const native = buildKernelConnectionOptions(opts);
+      expectNativeConnectionOptions(native, {
+        hostName: 'adb-12345.0.azuredatabricks.net',
+        httpPath: '/sql/1.0/warehouses/abc',
+        intervalsAsString: true,
+        authMode: 'OAuthU2m',
+        oauthRedirectPort: 8030,
+        oauthScopes: ['sql', 'offline_access'],
+      });
     });
 
     it('rejects a `persistence` hook on U2M citing the AuthConfig::External kernel-plumbing gap', () => {