diff --git a/.changeset/integrate-migration-tool-into-cli.md b/.changeset/integrate-migration-tool-into-cli.md new file mode 100644 index 000000000..b066925da --- /dev/null +++ b/.changeset/integrate-migration-tool-into-cli.md @@ -0,0 +1,13 @@ +--- +"clerk": minor +--- + +Add `clerk migrate` for moving users into Clerk from Clerk, Auth0, Supabase, Auth.js, Better Auth, Firebase or WorkOS. + +- `migrate export ` writes a self-describing file. `migrate import ` checks every user against the instance before writing (`--dry-run`, `--allow-partial`, `--skip-legal-checks`), asks before it writes, and continues where an interrupted or partial run stopped. +- `migrate import --reserve-unverified` creates the emails and phones a source never verified as reserved (usable for sign-in, locked to the user) instead of unverified. At a terminal, the import asks. +- `migrate runs` shows what every run did, `migrate undo ` deletes the users an import created, and `migrate sources` shows what each source carries, including sources you write yourself (`--source ./my-source.ts`). +- A request that cannot connect now names the host it could not reach. +- `clerk init` warns when the project uses WorkOS, and points to the migration guide. +- Multiselect prompts list `a: all` in their key legend. +- `clerk users` dry runs name the instance the key reaches and where the key came from, such as `--app` or the `CLERK_SECRET_KEY` env var. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eea38e67..6bd186b56 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: key: bun-${{ runner.os }}-${{ hashFiles('bun.lock') }} restore-keys: bun-${{ runner.os }}- - run: bun install --frozen-lockfile - - run: bun run build + - run: bun run build:compile lint: name: Lint diff --git a/.gitignore b/.gitignore index 91df74696..9ab506561 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ coverage # logs logs +!packages/cli-core/src/commands/migrate/logs/ +!packages/cli-core/src/commands/migrate/logs/** _.log report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json @@ -44,3 +46,7 @@ test/e2e/.har # Local planning/spec docs docs/superpowers/ + +# Local migration exports and the credentials that produced them +exports/ +*service-account*.json diff --git a/CLAUDE.md b/CLAUDE.md index 5a38ae8ca..d1c548a47 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,7 +30,7 @@ Default to using Bun instead of Node.js. - `Bun.serve()` supports WebSockets, HTTPS, and routes. Don't use `express`. - `bun:sqlite` for SQLite. Don't use `better-sqlite3`. - `Bun.redis` for Redis. Don't use `ioredis`. -- `Bun.sql` for Postgres. Don't use `pg` or `postgres.js`. +- `Bun.sql` for Postgres and MySQL. Don't use `pg`, `postgres.js`, or `mysql2`. - `WebSocket` is built-in. Don't use `ws`. - Prefer `Bun.file` over `node:fs`'s readFile/writeFile - Bun.$`ls` instead of execa. @@ -48,7 +48,7 @@ bun run test:e2e:op # Run E2E tests with secrets resolved from 1Password (prefe bun run test:e2e # Run E2E tests with env vars already set (used by CI) ``` -Locally, prefer `bun run test:e2e:op` so secrets are injected from 1Password in-memory and never written to disk. `bun run test:e2e` is for CI or for cases where the required env vars are already exported. +Locally, prefer `bun run test:e2e:op` so secrets are injected from 1Password in-memory and never written to disk. `bun run test:e2e` is for CI or for cases where the required env vars are already exported. Both run `bun test` with `--no-env-file`, so a `.env.local` pointing the CLI at a local `clerk_go` stack can't send the production test secrets there. CI runs `bun run format:check` (fails if unformatted), `bun run lint`, `bun run test`, and `bun run test:e2e` on every PR to `main`. E2E tests only run for PRs from the same repository (not external forks) and target the production Clerk API with a dedicated test application. @@ -56,6 +56,8 @@ When running multiple test files directly with `bun test`, always pass `--isolat These flags require Bun >= 1.3.13 — older versions silently ignore them and lose isolation. `bun run test` and `bun run test:e2e` run `scripts/check-bun-version.ts` first, which fails fast when the installed Bun is older than the `engines.bun` floor in package.json. +The same floor also covers `Bun.sql`'s MySQL adapter used by the DB-backed export commands: MySQL support landed in Bun 1.2.21, but binary columns (password hashes) only decoded correctly from 1.3.6. See the header of `scripts/check-bun-version.ts`. + ## Versioning The `CLI_VERSION` global is injected at compile time via `bun build --compile --define "CLI_VERSION=..."`. The CI release workflow injects the real version. diff --git a/README.md b/README.md index 817fcdaab..56d40fd14 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Commands: init [options] Initialize Clerk in your project link [options] Link this project to a Clerk application mcp Manage the Clerk remote MCP server connection for AI editors and CLIs + migrate Migrate users into Clerk from another auth provider or another Clerk instance open Open Clerk resources in your browser telemetry Control CLI usage telemetry (status, disable, enable) unlink [options] Unlink this project from its Clerk application diff --git a/bun.lock b/bun.lock index fe10159c4..3cd0ad6d1 100644 --- a/bun.lock +++ b/bun.lock @@ -1,9 +1,9 @@ { "lockfileVersion": 1, - "configVersion": 0, + "configVersion": 1, "workspaces": { "": { - "name": "marseille", + "name": "@clerk/cli-workspace", "devDependencies": { "@changesets/cli": "^2.31.1", "@clerk/testing": "^2.2.31", @@ -20,7 +20,7 @@ }, "packages/cli": { "name": "clerk", - "version": "3.3.0", + "version": "3.4.0", "bin": { "clerk": "./bin/clerk", }, @@ -37,11 +37,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.4", "semver": "^7.8.5", "yaml": "^2.9.0", + "zod": "^4.4.3", }, "devDependencies": { "@clerk/shared": "^4.30.2", @@ -58,9 +60,6 @@ }, }, }, - "patchedDependencies": { - "playwright-core@1.62.1": "patches/playwright-core@1.62.1.patch", - }, "overrides": { "tmp": "^0.2.6", }, @@ -69,9 +68,9 @@ "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], - "@babel/parser": ["@babel/parser@7.29.8", "", { "dependencies": { "@babel/types": "^7.29.8" }, "bin": "./bin/babel-parser.js" }, "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA=="], + "@babel/parser": ["@babel/parser@7.29.9", "", { "dependencies": { "@babel/types": "^7.29.8" }, "bin": "./bin/babel-parser.js" }, "sha512-CjXrNHTnvqBVqHgdBysY3vk2T8tpJHb5/RMeHJBTyVa9xgugCB0CJTx/3oO8RV2QRQP391RWpB7D6hLjm8V9uA=="], - "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], "@babel/types": ["@babel/types@7.29.8", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg=="], @@ -109,23 +108,23 @@ "@changesets/write": ["@changesets/write@0.4.0", "", { "dependencies": { "@changesets/types": "^6.1.0", "fs-extra": "^7.0.1", "human-id": "^4.1.1", "prettier": "^2.7.1" } }, "sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q=="], - "@clack/core": ["@clack/core@1.4.3", "", { "dependencies": { "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-/kr3UWNtdJfxZtPgDqUOmG2pvwlmcLGheex5yiZKdwbzZJxhV+HMNR9QNmyY5cGwTNV6LrR7Jtp+KjhUAP1qBQ=="], + "@clack/core": ["@clack/core@1.5.1", "", { "dependencies": { "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-iHTrHA8MtVuLl2TfZySmcKv1qO2PoyC9Z7pfSDozEuV5vtY3/wcOPKJXlqJ5Oq2Cx5DDGQGAMVx6HZfRRoVEbQ=="], - "@clack/prompts": ["@clack/prompts@1.7.0", "", { "dependencies": { "@clack/core": "1.4.3", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-y7/yvZ2TPAnR9+jnc00klvNNLkJiXFFrQA/hlLCcxA9a2A4zQIOimyFQ9XfwYKiGD1fb5GY8vbKIIgO8d5Tb2A=="], + "@clack/prompts": ["@clack/prompts@1.8.1", "", { "dependencies": { "@clack/core": "1.5.1", "fast-string-width": "^3.0.2", "fast-wrap-ansi": "^0.2.0", "sisteransi": "^1.0.5" } }, "sha512-dlT1m5e/0yUL0kRNcQn7yGLVThkgbB0Ga/1AmfDDC/8ik6AIiSf2QLQO2zPYvefsHP0aFgxO93cVLCCfDp7kzQ=="], - "@clerk/backend": ["@clerk/backend@3.16.13", "", { "dependencies": { "@clerk/shared": "^4.30.2", "standardwebhooks": "^1.0.0", "tslib": "2.8.1" } }, "sha512-L6+48rB8WVoxoiFDKv49KWxc58aGwuYnbuHBxNdqgTZO6PTyEH3KLpnqI/WaCqIxQW56MJE24EJREsU8WlWrWw=="], + "@clerk/backend": ["@clerk/backend@3.22.0", "", { "dependencies": { "@clerk/shared": "^4.38.0", "standardwebhooks": "^1.0.0", "tslib": "2.8.1" } }, "sha512-YBJtclx3ytvQGNYhf0/pG+WHUk1ams7Aai3D2AB8c96AkSnzUE+i1Q98HyupIulPDr3fUWeQJtex4DYyO9D4fA=="], "@clerk/cli-core": ["@clerk/cli-core@workspace:packages/cli-core"], "@clerk/cli-extras": ["@clerk/cli-extras@workspace:packages/extras"], - "@clerk/shared": ["@clerk/shared@4.30.2", "", { "dependencies": { "@tanstack/query-core": "^5.100.6", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.7" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-2U8jE2Gno2uRxg5b2fv5uY3zxzxAvmBzjiaOPAfoAVUgJ2Ys9kPu+6k2TaBNTkeF0ncQ/5erECIx6dPt0d8XyQ=="], + "@clerk/shared": ["@clerk/shared@4.38.0", "", { "dependencies": { "@tanstack/query-core": "^5.101.4", "dequal": "2.0.3", "glob-to-regexp": "0.4.1", "js-cookie": "3.0.8" }, "peerDependencies": { "react": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0", "react-dom": "^18.0.0 || ~19.0.3 || ~19.1.4 || ~19.2.3 || ~19.3.0-0" }, "optionalPeers": ["react", "react-dom"] }, "sha512-jVHz03682SJ3xsjsSjNX8YyWHllnr91qQEQZpZjCHGUIwGz3rYVavh69cofSuvPrqwOtXReukVEMtodzG2vVCQ=="], - "@clerk/testing": ["@clerk/testing@2.2.31", "", { "dependencies": { "@clerk/backend": "^3.16.13", "@clerk/shared": "^4.30.2", "dotenv": "17.2.2" }, "peerDependencies": { "@playwright/test": "^1", "cypress": "^13 || ^14 || ^15" }, "optionalPeers": ["@playwright/test", "cypress"] }, "sha512-aiwlGSDNzvEBZ9VH9U1k3cVz+HziAeNanSlbayD2ZKOZaB6FEk9z5X+0Ahh2SXZEz88yFS8kyFErAeR4lr15PQ=="], + "@clerk/testing": ["@clerk/testing@2.2.42", "", { "dependencies": { "@clerk/backend": "^3.22.0", "@clerk/shared": "^4.38.0", "dotenv": "17.2.2" }, "peerDependencies": { "@playwright/test": "^1", "cypress": "^13 || ^14 || ^15" }, "optionalPeers": ["@playwright/test", "cypress"] }, "sha512-TlbtOk+WBNbOVXI1Mfyyk3qg2NQTqMfG6d3uWEVzIA5AQX3gbGStify4wLo8J2MMgjYrZTN2AsHAyDDcvax55g=="], "@commander-js/extra-typings": ["@commander-js/extra-typings@15.0.0", "", { "peerDependencies": { "commander": "~15.0.0" } }, "sha512-yeJlba62xqmkgELUsn7356MEnzLLu/fw2x4lofFqGnXh6YysRdEs2BaLeLtg1+KU0AXvMeqQvTTp+3hBEBK+EA=="], - "@hono/node-server": ["@hono/node-server@2.1.1", "", { "peerDependencies": { "hono": "^4" } }, "sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg=="], + "@hono/node-server": ["@hono/node-server@2.1.3", "", { "peerDependencies": { "hono": "^4" } }, "sha512-TA//nWMqPhbfdfneACk6t5a9eqbS9lABEPyKn0/xZTah3H3U2XaVg85rJFl0/Fyit0I552YDHgXGVSf3GwqbUw=="], "@inquirer/external-editor": ["@inquirer/external-editor@1.0.3", "", { "dependencies": { "chardet": "^2.1.1", "iconv-lite": "^0.7.0" }, "peerDependencies": { "@types/node": ">=18" }, "optionalPeers": ["@types/node"] }, "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA=="], @@ -133,7 +132,7 @@ "@manypkg/get-packages": ["@manypkg/get-packages@1.1.3", "", { "dependencies": { "@babel/runtime": "^7.5.5", "@changesets/types": "^4.0.1", "@manypkg/find-root": "^1.1.0", "fs-extra": "^8.1.0", "globby": "^11.0.0", "read-yaml-file": "^1.1.0" } }, "sha512-fo+QhuU3qE/2TQMQmbVMqaQ6EWbMhi4ABWP+O4AM1NqPBuy0OrApV5LO6BrrgnhtAHS2NH6RrVk9OL181tTi8A=="], - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.30.0", "", { "dependencies": { "@hono/node-server": "^1.19.9 || ^2.0.5", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-xKd8OIzlqNzcqcNumGAa6g+PW2kjD5vrpcKOnfldAUPP3j7lnqMPwlTXQm8gF+UwH72z0lqaRbjr9hqGz0eITA=="], + "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.32.0", "", { "dependencies": { "@hono/node-server": "^1.19.9 || ^2.0.5", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-8BviX/hK4Gd2eL1KdTwm0gfl4d3wdoErW0Jd00h6nomUAtk6IJk5vwQJc6kfWtJTzN5OEV/lgIiJKI8fiDGAlA=="], "@napi-rs/keyring": ["@napi-rs/keyring@1.3.0", "", { "optionalDependencies": { "@napi-rs/keyring-darwin-arm64": "1.3.0", "@napi-rs/keyring-darwin-x64": "1.3.0", "@napi-rs/keyring-freebsd-x64": "1.3.0", "@napi-rs/keyring-linux-arm-gnueabihf": "1.3.0", "@napi-rs/keyring-linux-arm64-gnu": "1.3.0", "@napi-rs/keyring-linux-arm64-musl": "1.3.0", "@napi-rs/keyring-linux-riscv64-gnu": "1.3.0", "@napi-rs/keyring-linux-x64-gnu": "1.3.0", "@napi-rs/keyring-linux-x64-musl": "1.3.0", "@napi-rs/keyring-win32-arm64-msvc": "1.3.0", "@napi-rs/keyring-win32-ia32-msvc": "1.3.0", "@napi-rs/keyring-win32-x64-msvc": "1.3.0" } }, "sha512-WrOw/bcXm0f9qHkumlT1QlArXSTWqaY9sunsDpOk+yCCorCKMxvWT/a3xko4EYHVdeZoh00yI2TydXn6eyICDA=="], @@ -205,63 +204,63 @@ "@oxfmt/binding-win32-x64-msvc": ["@oxfmt/binding-win32-x64-msvc@0.64.0", "", { "os": "win32", "cpu": "x64" }, "sha512-BtmbtL/QjMtF1a6C3CqoDluH2IfB6fJt62E+B9RFfUPtFk4Iz9PFS6+y/SzzOvSxc7aUk2Kphwg7Dh8lMbwu6g=="], - "@oxlint-tsgolint/darwin-arm64": ["@oxlint-tsgolint/darwin-arm64@7.0.2001", "", { "os": "darwin", "cpu": "arm64" }, "sha512-CUJEdbSZ54+Xy9OXqOhWLTKZKV0BBiV7C2i/ygyVmXtkUNXx5YCzN8DpSSshTAKktoL7S+tnQ/ftFG/i7X896w=="], + "@oxlint-tsgolint/darwin-arm64": ["@oxlint-tsgolint/darwin-arm64@7.0.2003", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TgV33rXr6ueXBwvc+0nssUkTBSXHJxv77I8p4RCjDjCnvexHtmoPiudVRDfj3S3puMDdeQdHW6jdUgUPy3nr/w=="], - "@oxlint-tsgolint/darwin-x64": ["@oxlint-tsgolint/darwin-x64@7.0.2001", "", { "os": "darwin", "cpu": "x64" }, "sha512-pXfBb5BqONCcgrXQNUZWXgiYmRSWJzd97S8i41VVOh6ut0tyo+cJ5FKFpczDHxiVNfj/3e7c9B4MtztNdpIVCw=="], + "@oxlint-tsgolint/darwin-x64": ["@oxlint-tsgolint/darwin-x64@7.0.2003", "", { "os": "darwin", "cpu": "x64" }, "sha512-hY3FMAjIaPDdK3FNxyRXRfHPOFeoBn6dmnLyKZgQ2IbTTD1adXhl6PfCIqHhCPvurigv7nf7mYuWZZ19MmJzmg=="], - "@oxlint-tsgolint/linux-arm64": ["@oxlint-tsgolint/linux-arm64@7.0.2001", "", { "os": "linux", "cpu": "arm64" }, "sha512-roP7zujb/QDPzDwEKsFFpzNHHy91/Y7oX9vQXk78ekyZtcQj1QXDIMH33gjDdHBfRl4K9pZ36xhRgrP4Zr+R8A=="], + "@oxlint-tsgolint/linux-arm64": ["@oxlint-tsgolint/linux-arm64@7.0.2003", "", { "os": "linux", "cpu": "arm64" }, "sha512-eAET4JpyfBbg8SO0K74o4R55tEjVC292pIyxGOw2XGf8x/HrPNyxwQ478jMYOM54FIVOHc8sJ6VRHxluy6lucw=="], - "@oxlint-tsgolint/linux-x64": ["@oxlint-tsgolint/linux-x64@7.0.2001", "", { "os": "linux", "cpu": "x64" }, "sha512-UDezNqdECVmngu2TPnjaS1YoAmcTaBoI5lV9vk3VahBxoi+I5r9k3iJTT7qZoYWOXTD/7T7bNcwRgrocR6BscQ=="], + "@oxlint-tsgolint/linux-x64": ["@oxlint-tsgolint/linux-x64@7.0.2003", "", { "os": "linux", "cpu": "x64" }, "sha512-GXdyO/XyqDJ3s/llR/oOktLsYNjZtWSQBy0JMc+/0gsNPvTcseKPhn9c6KcFyWmnr6H4vljYALbujonqSzzEGw=="], - "@oxlint-tsgolint/win32-arm64": ["@oxlint-tsgolint/win32-arm64@7.0.2001", "", { "os": "win32", "cpu": "arm64" }, "sha512-uJZhqB6pdXLuN+AD1F5082byyQti/NPmJA77GtcFlmT2HzRelqbNls3SaIqxpjdFgvSBF9g0yOKGBkGFg7kX8Q=="], + "@oxlint-tsgolint/win32-arm64": ["@oxlint-tsgolint/win32-arm64@7.0.2003", "", { "os": "win32", "cpu": "arm64" }, "sha512-TWauXnPfet0VgpmrstoAK58eJ4gbuwPUuUTQmYQAzE/RG1CpnxXtrKUJULf6lyerPE4KN3gM+DflIA1hOH+38Q=="], - "@oxlint-tsgolint/win32-x64": ["@oxlint-tsgolint/win32-x64@7.0.2001", "", { "os": "win32", "cpu": "x64" }, "sha512-FkDRm8hx9OwzGQqyWG1tO5QrTLRApff9DzSgpz9QZau37BR8d1VYKOxMLGf6shPZntJFoTwIIJYT68VndYDCog=="], + "@oxlint-tsgolint/win32-x64": ["@oxlint-tsgolint/win32-x64@7.0.2003", "", { "os": "win32", "cpu": "x64" }, "sha512-DoRmfe7j8VqNlukp8liRVW45GQDhzRccNenjD/pdzelgtffW47pCMd1xbJLkPaPbKnTwID3onn3VZlL7JbebEA=="], - "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.79.0", "", { "os": "android", "cpu": "arm" }, "sha512-TebFaaMklO/RXzTv7PucaCq9l3X6D1gA+C8H6K4njtjFOV+zWE9MKLpulcJZN9bzytbUbQIY0mZuz12nQ5Kv4Q=="], + "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.86.0", "", { "os": "android", "cpu": "arm" }, "sha512-63Ozq6yQn79B2UxjSZ9huJmI+3/ZPI7hxAe6TyFh9oQL447KnIMoffOdtog+h4KILbQr61ZRvFPW5gYbAUgpgQ=="], - "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.79.0", "", { "os": "android", "cpu": "arm64" }, "sha512-KqqnOtAVgNsPPF0YSodkFZA1O80jcKoCZCTu3bgsszxA+MrMP9TLzfXitKjEj1FmrPprKDMdRDMmY3weESO9sg=="], + "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.86.0", "", { "os": "android", "cpu": "arm64" }, "sha512-JdLGp44phbf/0Du236Dt06yfK6Qw8xhTFeDvHJdjk167r59OfP5+BL31mgMYVnswS8e+j6tFuccSY0QDC/lgCw=="], - "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.79.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-BVC2nsMzqQzRDPc5RhixkZ+m1p7iH4bxRRvqkbwDXX0PlQKm1BPy8J8cRjnAFafOq2QzI+BfO3vE8w2GZ3CBag=="], + "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.86.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-h+vkOr4ik6KLFCXdtNVEj9xfnXfNUpPoF95QjORdhiSpLqQePzcN81kGHjAgsw1/G7WBWK5KsFKbI2UK8EyzMA=="], - "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.79.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-p6Lm+snmhGuLKL1+CpCV8L6ijkE/qJzK2H2jG9+eKJT0n31RbY4FLsdhexekgP3bLpw4Kgde+9DZuDZQ4yIInA=="], + "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.86.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-wbglRuH5eKsp68keTOOeKDHU1astWSRYFsh2bwKQTAPb44rD6kVgzYI1ZrAJRTc6/7PH55ZXFCQx0rQXrj0UdA=="], - "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.79.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-qDMm0dXZnoHyRqSL4N4xUq82T4sqK5cbKSjvd/dF/YbMUXc2R1wEPf+vmA5S0qUmi0nwXfNbjXBtZaIqzQLIMg=="], + "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.86.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-yTQ7QSOlNfQsrfbeVweTiegQJLKPjUKgXp/JpGiFzza6Hp29S9LAoH77TM3L0RjK39iQRZB5lk5iq2t241Mfag=="], - "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.79.0", "", { "os": "linux", "cpu": "arm" }, "sha512-2od7s0nuKPzqyUZAWk9KkCyGg7eI9dwFPZg+20lB15fKFkVZ0c9ZFxqPfiBAyDTlTkh9stPI0t+JlPCqMbItVA=="], + "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.86.0", "", { "os": "linux", "cpu": "arm" }, "sha512-QYJGy0E2Tv8GAayyBIeHFTwQb8u+zrvrOYrifuDGZDr9ad5XsDhiMScpZymhS9fwWjqrgG2ZUwT4vpY0ISqp4Q=="], - "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.79.0", "", { "os": "linux", "cpu": "arm" }, "sha512-ZOQUjkzDnvlhSE3+tWC3YXx94MMl+sYMlwH+u1+YGApGHOJP/YAc8ZBRFOXZ6eOBmxtXAWuS/fBcdZr8qqNO1A=="], + "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.86.0", "", { "os": "linux", "cpu": "arm" }, "sha512-oQo83CafLT8p3qh0AGkN0tSUIh5Hm63mfDsoqUmBQUBnSZLNBAbyRaAftfGY5a0cKZ34dG6jAtMYgcY5kZqt5g=="], - "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.79.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-lu158FR4nGqGeRS3BQvtG85wRgU/Fy4MD5Cxp1hzJXizGiLo6u2742wJSCDKh8cFcZntvX7fcxlq4mMmfryH1g=="], + "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.86.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-EM6wy5c2UM12qPb7iDJDBAZvHyfEJGH6iosFgaFiaRZpirz99//EHcUyD2KAskz07x1TULuiAtsojf4nvWWksw=="], - "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.79.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-mbpKQeE2aflTjddaHK7MP8KP/OFbUM++lt5M635ENM8IyIdK0jm2t9pb+2v9mVVIvhF6TqA4l7F79Pll1mi+uw=="], + "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.86.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-vCiUQb9ZNzZalxYRU4mlZwUqT5f+c7Sk9DwUOplWHtik3Dl4r3DVnUB5tBeRuTAUe6MUVCTylNW7TkkN8J1oTw=="], - "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.79.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WpGNua7gaxaHnpSDeog2ji8IDHn/QLPl9LPzwkR/FvVv58vT5BcXjRXnU+wbu3N75cpeha8CdC7ho/U2OIsB4g=="], + "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.86.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-T0AzE89yjTYmOhB/hb6i+rWTpe2mYt+xiguBhs9DEPIJTZ1H8OYqsE4VxJHHw2Wlq8Mztwvp2P8NV87ZOuhPRw=="], - "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.79.0", "", { "os": "linux", "cpu": "none" }, "sha512-tK1E93A5LVzISg4ngpKJnfTs7EqtIUceGI7MQ4GyDjJiLi8wPCkEyKlj2xkyKWZ1yzkDJyLHTBJ5/iFWRdnJvg=="], + "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.86.0", "", { "os": "linux", "cpu": "none" }, "sha512-s49r0K2f5Oc7fJoMUO1WkdhYK5Ob79uYRwWQ/XNoGysaOq3RUGmGUbHSuXiyydTTA+P9xEdDEnnlZVXiaSJhlw=="], - "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.79.0", "", { "os": "linux", "cpu": "none" }, "sha512-qhQvUIrngXivA2A9pQ+xPCychztn/5qUv7yS3gDwXv3w7Rag+eTeeXWmRyx+t7XsW5x6LuY/8AsTq36UgFIblg=="], + "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.86.0", "", { "os": "linux", "cpu": "none" }, "sha512-JIa2tYR8ayb3nvcMSZzn4O/xTy13S7MHllu7cBD+B5qDQXXSRy/Vwk1aNpiTjZmBJGwKCHOy/NvnY5Xcwg27uQ=="], - "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.79.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-sv6AaVgU/eE6u+6WFiQVDcPPwTxP6IJMSB9k701W2r/r6Tx465e8vPvVyRxquNH4Vy6KwRNu90mVbxXJN8+5gg=="], + "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.86.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-XwXMyBsAuWZkhZU74UNcfQKr1CJJAKnS80dZzOnjl6A4fKcf91IjFgidegQugMHA5SN8ulwDXEDV1yPntjXgQQ=="], - "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.79.0", "", { "os": "linux", "cpu": "x64" }, "sha512-iFZL02deziHslb3jEX9KdqlAkYoo4fGyotchKDzdfK1f5mxlIBeiQeHhvK3iFpuEJSB4ma/qeFn9oxPiwnhUPQ=="], + "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.86.0", "", { "os": "linux", "cpu": "x64" }, "sha512-C1WjukSyMnr66b+w1/tV8RFVv6d9v0MzDf4p9IxVXknqgmTHBgZh1pccN1eHzFr0b9Tbb3OXoPsAAdAuHAQfeA=="], - "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.79.0", "", { "os": "linux", "cpu": "x64" }, "sha512-3DtZR2raqObnh7wXZoFYFd0Fw7skBvcb3f7A+/lkEiDuh8hrE6vv9b/62Qxao1a9/OeHLw/FcXlXzgsW9wTRFg=="], + "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.86.0", "", { "os": "linux", "cpu": "x64" }, "sha512-ap6KLmvC38c6MdYzsIh25cXQupqYvjd37tMNftzrX1DCtkX1Gcf2B+S2B17dD4lKa5c5gJog7DQJyDo82PBKyw=="], - "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.79.0", "", { "os": "none", "cpu": "arm64" }, "sha512-Oatt4GuA1WJkqzk2ozx4HrWROOi7opV3AKDw/U8qDIqeTqzsjn5K2x3REJMNjU3/KU/Bkq96Zi3CknaiDTaC/Q=="], + "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.86.0", "", { "os": "none", "cpu": "arm64" }, "sha512-996Q9w/2GLwNyy5Djs39Jt09aFlr4b7TPhoj7dvo5FHmcyiF68nPz73HNc5UAYKIYeK+M9JzPQ4Wg1r1dXvP6A=="], - "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.79.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-NAgZr9Qp8nIA9rpo0JEvwiabTF/2UVqBNnupBG9X4kxXcQoScJUTi+qHhvabb9s/thgj5wQ4XcIaJvb+ZMgoKw=="], + "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.86.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-9uTCHYTknNkug5+46ViSoUMuOuGWK61D87QnblU6CXaWKJC630Z7dbls61vdHxya/cR2ycQZ6Mm6ckgF6Cag5g=="], - "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.79.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-+KyXjIvcpaXmWW/j9NNY5yWjrIVxaX18VyIheQy3jwc2GSYgpCr7MGI/HxIGQ/shAL5IWEKbhsqoMpAO5Stiog=="], + "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.86.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-a47ga7EKVfMU8rO0i1m3mGwfNRj3FhQyt1d/ukO/E9u1HHF3KpR3tyb3VhuJBGEVPF8uR4exSyHUyx7xbE+uxw=="], - "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.79.0", "", { "os": "win32", "cpu": "x64" }, "sha512-mEelcCMMBS57sIXh2veGMNy+pQwuGtcMxHxGIZWQ5Ba9pJ5jCCUFOZB9E2JhBaxGsURe+WGe0zJp4RVre52gpQ=="], + "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.86.0", "", { "os": "win32", "cpu": "x64" }, "sha512-az4dzifCRzES1crwu6pDFXzJTIanrdxXFL9TbiJWrx9LKcuacbR83aPvxDA1LKLNswi/RoqitC7Afcflhu3zBQ=="], "@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="], - "@tanstack/query-core": ["@tanstack/query-core@5.100.10", "", {}, "sha512-8UR0yJR+GiQ40m3lPhUr0xbfAupe6GSQiksSBSa9SM2NjezFyxXCIA69/lz8cSoNKZLrw1/PktIyQBJcVeMi3w=="], + "@tanstack/query-core": ["@tanstack/query-core@5.104.1", "", {}, "sha512-TiRghcrGkUTE+M4JBxvpPSpBhxvIzQairw+cksrHYGr+2uwC77yW1SY9nYxnYRYOppvucf8rEg6b5byoz4rTpw=="], - "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], - "@types/node": ["@types/node@25.2.3", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ=="], + "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], "@types/semver": ["@types/semver@7.8.0", "", {}, "sha512-1mAINjtQCXXeLkJ9ehXkwOcBpqtLxiVtKhpUf83DdRNdQKV0iXZpaHYqRr7nj+wvxuJzoAmAwXI+sCNMv1CzLQ=="], @@ -325,7 +324,7 @@ "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], - "bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], @@ -351,6 +350,8 @@ "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "csv-parser": ["csv-parser@3.2.1", "", { "bin": { "csv-parser": "bin/csv-parser" } }, "sha512-v8RPMSglouR9od735SnwSxLBbCJqEPSbgm1R5qfr8yIiMUCEFjox56kRZid0SvgHJEkxeIEu3+a9QS3YRh7CuA=="], + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], @@ -387,11 +388,11 @@ "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], - "eventsource-parser": ["eventsource-parser@3.1.0", "", {}, "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg=="], + "eventsource-parser": ["eventsource-parser@3.1.1", "", {}, "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ=="], "express": ["express@5.2.1", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "depd": "^2.0.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw=="], - "express-rate-limit": ["express-rate-limit@8.6.0", "", { "dependencies": { "debug": "^4.4.3", "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-XKJXDsASUOo0LLtFwW5hCcQGH0N4WQc/Rn8/Pvoia+TJFOkkFPvrtW9lZOeeNcxQJspvOIERMwiRLsVFlhHEkA=="], + "express-rate-limit": ["express-rate-limit@8.7.0", "", { "dependencies": { "debug": "^4.4.3", "ip-address": "^10.2.0" }, "peerDependencies": { "express": ">= 4.11" } }, "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g=="], "extendable-error": ["extendable-error@0.1.7", "", {}, "sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg=="], @@ -407,11 +408,11 @@ "fast-string-width": ["fast-string-width@3.0.2", "", { "dependencies": { "fast-string-truncated-width": "^3.0.2" } }, "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg=="], - "fast-uri": ["fast-uri@3.1.6", "", {}, "sha512-7Ical1vFEMr0onbVzEDIreM22I4khW+fzyQPwvAFWBp1iwdshSZRsL4jjRvPG9JP1uiqMHRto+YU6R2/CzDz5Q=="], + "fast-uri": ["fast-uri@3.1.8", "", {}, "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg=="], - "fast-wrap-ansi": ["fast-wrap-ansi@0.2.0", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w=="], + "fast-wrap-ansi": ["fast-wrap-ansi@0.2.2", "", { "dependencies": { "fast-string-width": "^3.0.2" } }, "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q=="], - "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + "fastq": ["fastq@1.20.3", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw=="], "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], @@ -425,8 +426,6 @@ "fs-extra": ["fs-extra@7.0.1", "", { "dependencies": { "graceful-fs": "^4.1.2", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw=="], - "fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], - "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], @@ -447,11 +446,11 @@ "hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], - "hono": ["hono@4.13.5", "", {}, "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw=="], + "hono": ["hono@4.13.12", "", {}, "sha512-6E2QDAc9Ick9Sq77ZrGS/dk2WUYni91aufTw6LJKpV7w8kW5/GxVUc650FOADOmlwg3K+f7Pun6XlV+pYmW6gw=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "human-id": ["human-id@4.1.3", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-tsYlhAYpjCKa//8rXZ9DqKEawhPoSytweBC2eNvcaDK+57RZLHGqNs3PZTQO6yekLFSuvA6AlnAfrw1uBvtb+Q=="], + "human-id": ["human-id@4.2.1", "", { "bin": { "human-id": "dist/cli.js" } }, "sha512-zPGsiS+dWoTZtZ4AtpA9Y+BdSFSNWvnouNlWNoUFyAM6xHOHmdCvqO3k8AIbdamCOv4gUFUVNPf6rJFfc4UiJw=="], "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], @@ -459,7 +458,7 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - "ip-address": ["ip-address@10.5.0", "", {}, "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g=="], + "ip-address": ["ip-address@10.7.3", "", {}, "sha512-A1kdq/tSb5QjvKvAMgIoEvDBIgL7qaqVP/jkvSwYYRZ9iEzvPpopxp2wQfu3SuZRHtpHNxMn8Fs0bS+gf5Xmwg=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -479,9 +478,9 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "jose": ["jose@6.2.3", "", {}, "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw=="], + "jose": ["jose@6.2.12", "", {}, "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw=="], - "js-cookie": ["js-cookie@3.0.7", "", {}, "sha512-z/wZZgDrkNV1eA0ULjM/F9/50Ya8fbzgKneSpoPsXSGd0KnpdtHfOZWK+GcwLk+EZbS4F9RBhU+K2RgzuDaItw=="], + "js-cookie": ["js-cookie@3.0.8", "", {}, "sha512-yeJd4aNAdYZQjaon2bpD/Gb0B/omw7HQOsynXXcOiWVCacbBcPlgn8S/d1X6blFSaHao7ozqtW7NZW19xpCtIw=="], "js-yaml": ["js-yaml@4.3.2", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA=="], @@ -495,11 +494,11 @@ "lodash.startcase": ["lodash.startcase@4.4.0", "", {}, "sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg=="], - "magicast": ["magicast@0.5.4", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "source-map-js": "^1.2.1" } }, "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w=="], + "magicast": ["magicast@0.5.5", "", { "dependencies": { "@babel/parser": "^7.29.7", "@babel/types": "^7.29.7", "source-map-js": "^1.2.1" } }, "sha512-UicdXN8zQ3JHlxVq+28afMXPr1z7WNY6+7EJnzTdQWkTAlMLF5fNCCKxJHBQwGaNGR11581EiQmQzx73+MvszA=="], "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], - "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + "media-typer": ["media-typer@1.1.1", "", {}, "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ=="], "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], @@ -517,7 +516,7 @@ "nano-staged": ["nano-staged@1.0.2", "", { "bin": { "nano-staged": "lib/bin.js" } }, "sha512-Fytar3zHLY99nlMfqPPbraxZodqQAHPpdPRyYaplL+lB9DCR6pUrafxbG+Btz4+7fO5Rm/+DO4ZeDO/nLSUMhw=="], - "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + "negotiator": ["negotiator@1.1.0", "", { "dependencies": { "content-type": "^2.1.0" } }, "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg=="], "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], @@ -531,9 +530,9 @@ "oxfmt": ["oxfmt@0.64.0", "", { "dependencies": { "tinypool": "2.1.0" }, "optionalDependencies": { "@oxfmt/binding-android-arm-eabi": "0.64.0", "@oxfmt/binding-android-arm64": "0.64.0", "@oxfmt/binding-darwin-arm64": "0.64.0", "@oxfmt/binding-darwin-x64": "0.64.0", "@oxfmt/binding-freebsd-x64": "0.64.0", "@oxfmt/binding-linux-arm-gnueabihf": "0.64.0", "@oxfmt/binding-linux-arm-musleabihf": "0.64.0", "@oxfmt/binding-linux-arm64-gnu": "0.64.0", "@oxfmt/binding-linux-arm64-musl": "0.64.0", "@oxfmt/binding-linux-ppc64-gnu": "0.64.0", "@oxfmt/binding-linux-riscv64-gnu": "0.64.0", "@oxfmt/binding-linux-riscv64-musl": "0.64.0", "@oxfmt/binding-linux-s390x-gnu": "0.64.0", "@oxfmt/binding-linux-x64-gnu": "0.64.0", "@oxfmt/binding-linux-x64-musl": "0.64.0", "@oxfmt/binding-openharmony-arm64": "0.64.0", "@oxfmt/binding-win32-arm64-msvc": "0.64.0", "@oxfmt/binding-win32-ia32-msvc": "0.64.0", "@oxfmt/binding-win32-x64-msvc": "0.64.0" }, "peerDependencies": { "svelte": "^5.0.0", "vite-plus": "*" }, "optionalPeers": ["svelte", "vite-plus"], "bin": { "oxfmt": "bin/oxfmt" } }, "sha512-XZ4GFBN/PLbXKq+0zrgpQfPKYuJlUuj+nzZJY7UpIbFMNyefNLCdN9EwViycNqnYcv0wrn0jXcQLlqJp8RCKBg=="], - "oxlint": ["oxlint@1.79.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.79.0", "@oxlint/binding-android-arm64": "1.79.0", "@oxlint/binding-darwin-arm64": "1.79.0", "@oxlint/binding-darwin-x64": "1.79.0", "@oxlint/binding-freebsd-x64": "1.79.0", "@oxlint/binding-linux-arm-gnueabihf": "1.79.0", "@oxlint/binding-linux-arm-musleabihf": "1.79.0", "@oxlint/binding-linux-arm64-gnu": "1.79.0", "@oxlint/binding-linux-arm64-musl": "1.79.0", "@oxlint/binding-linux-ppc64-gnu": "1.79.0", "@oxlint/binding-linux-riscv64-gnu": "1.79.0", "@oxlint/binding-linux-riscv64-musl": "1.79.0", "@oxlint/binding-linux-s390x-gnu": "1.79.0", "@oxlint/binding-linux-x64-gnu": "1.79.0", "@oxlint/binding-linux-x64-musl": "1.79.0", "@oxlint/binding-openharmony-arm64": "1.79.0", "@oxlint/binding-win32-arm64-msvc": "1.79.0", "@oxlint/binding-win32-ia32-msvc": "1.79.0", "@oxlint/binding-win32-x64-msvc": "1.79.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-hVJ9hq9m2unPS+Of4eJJgCPdIeCC+3DHEUX3tkmrPJr3OK2hz7PhXwgC+ZP71ZcYu8cCDEtQrqLxWNvxBppBVg=="], + "oxlint": ["oxlint@1.86.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.86.0", "@oxlint/binding-android-arm64": "1.86.0", "@oxlint/binding-darwin-arm64": "1.86.0", "@oxlint/binding-darwin-x64": "1.86.0", "@oxlint/binding-freebsd-x64": "1.86.0", "@oxlint/binding-linux-arm-gnueabihf": "1.86.0", "@oxlint/binding-linux-arm-musleabihf": "1.86.0", "@oxlint/binding-linux-arm64-gnu": "1.86.0", "@oxlint/binding-linux-arm64-musl": "1.86.0", "@oxlint/binding-linux-ppc64-gnu": "1.86.0", "@oxlint/binding-linux-riscv64-gnu": "1.86.0", "@oxlint/binding-linux-riscv64-musl": "1.86.0", "@oxlint/binding-linux-s390x-gnu": "1.86.0", "@oxlint/binding-linux-x64-gnu": "1.86.0", "@oxlint/binding-linux-x64-musl": "1.86.0", "@oxlint/binding-openharmony-arm64": "1.86.0", "@oxlint/binding-win32-arm64-msvc": "1.86.0", "@oxlint/binding-win32-ia32-msvc": "1.86.0", "@oxlint/binding-win32-x64-msvc": "1.86.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2003", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-og0lhgvZfgGF//gOOmZXvtr+GmBbAGEnbEhv/QUg7UW2Wi4wHMnJbnMD+zuHgPdJxdklfgpPupdlAaAySyxrZg=="], - "oxlint-tsgolint": ["oxlint-tsgolint@7.0.2001", "", { "optionalDependencies": { "@oxlint-tsgolint/darwin-arm64": "7.0.2001", "@oxlint-tsgolint/darwin-x64": "7.0.2001", "@oxlint-tsgolint/linux-arm64": "7.0.2001", "@oxlint-tsgolint/linux-x64": "7.0.2001", "@oxlint-tsgolint/win32-arm64": "7.0.2001", "@oxlint-tsgolint/win32-x64": "7.0.2001" }, "bin": { "tsgolint": "./bin/tsgolint.js" } }, "sha512-KjK/XLcXr1DSyonKhsuFqJRiuKqcyG9j3LJ8nkOsrLzGvodBPqzHOKauy10asLMDI0sUpvb+1sxlzff3udZvfg=="], + "oxlint-tsgolint": ["oxlint-tsgolint@7.0.2003", "", { "optionalDependencies": { "@oxlint-tsgolint/darwin-arm64": "7.0.2003", "@oxlint-tsgolint/darwin-x64": "7.0.2003", "@oxlint-tsgolint/linux-arm64": "7.0.2003", "@oxlint-tsgolint/linux-x64": "7.0.2003", "@oxlint-tsgolint/win32-arm64": "7.0.2003", "@oxlint-tsgolint/win32-x64": "7.0.2003" }, "bin": { "tsgolint": "./bin/tsgolint.js" } }, "sha512-VnK4zlqgmgq/7ZcjzCk/WpN8kKFsYGcB85Io9qT3wL4K8Un3RKmJkp793crNETieMusPMKOA4a+Mw2wbteI6TQ=="], "p-filter": ["p-filter@2.1.0", "", { "dependencies": { "p-map": "^2.0.0" } }, "sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw=="], @@ -565,15 +564,15 @@ "pkce-challenge": ["pkce-challenge@5.0.1", "", {}, "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ=="], - "playwright": ["playwright@1.62.1", "", { "dependencies": { "playwright-core": "1.62.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg=="], + "playwright": ["playwright@1.63.0", "", { "dependencies": { "playwright-core": "1.63.0" }, "bin": { "playwright": "cli.js" } }, "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg=="], - "playwright-core": ["playwright-core@1.62.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw=="], + "playwright-core": ["playwright-core@1.63.0", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg=="], "prettier": ["prettier@2.8.8", "", { "bin": { "prettier": "bin-prettier.js" } }, "sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q=="], - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-addr": ["proxy-addr@2.0.8", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ=="], - "qs": ["qs@6.15.3", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-O9gl3zCl5h5blw1KGUzQKhA5oUXSl8rwUIM5o0S3nCXMliSvy5Dzx7/DJcI+SwgICv+IneSZwhBh1oSyEHA71A=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "quansync": ["quansync@0.2.11", "", {}, "sha512-AifT7QEbW9Nri4tAwR5M/uzpBuqfZf+zwaEM/QkzEjj7NBuFD2rBuy0K3dE+8wltbezDV7JMA0WfnCPYRSYbXA=="], @@ -623,13 +622,13 @@ "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], - "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "source-map-js": ["source-map-js@1.2.2", "", {}, "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw=="], "spawndamnit": ["spawndamnit@3.0.1", "", { "dependencies": { "cross-spawn": "^7.0.5", "signal-exit": "^4.0.1" } }, "sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg=="], "sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="], - "standardwebhooks": ["standardwebhooks@1.0.0", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg=="], + "standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="], "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], @@ -653,7 +652,7 @@ "typescript": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], - "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], + "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], "universalify": ["universalify@0.1.2", "", {}, "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="], @@ -665,21 +664,15 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], - "yaml": ["yaml@2.9.0", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA=="], + "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], - "zod": ["zod@4.4.3", "", {}, "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ=="], + "zod": ["zod@4.6.5", "", {}, "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], - "@changesets/apply-release-plan/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - - "@changesets/assemble-release-plan/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], + "@inquirer/external-editor/chardet": ["chardet@2.2.0", "", {}, "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA=="], - "@changesets/get-dependents-graph/semver": ["semver@7.7.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - - "@inquirer/external-editor/chardet": ["chardet@2.1.1", "", {}, "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ=="], - - "@inquirer/external-editor/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "@inquirer/external-editor/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "@manypkg/find-root/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="], @@ -689,15 +682,17 @@ "@manypkg/get-packages/fs-extra": ["fs-extra@8.1.0", "", { "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^4.0.0", "universalify": "^0.1.0" } }, "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g=="], - "body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + "body-parser/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], + + "body-parser/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], - "body-parser/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "negotiator/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], - "raw-body/iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], + "raw-body/iconv-lite": ["iconv-lite@0.7.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ=="], "read-yaml-file/js-yaml": ["js-yaml@3.15.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w=="], - "type-is/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="], + "type-is/content-type": ["content-type@2.1.0", "", {}, "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag=="], "read-yaml-file/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], } diff --git a/package.json b/package.json index c6d5830ae..f72dc31b2 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,10 @@ "packages/*" ], "scripts": { - "build": "bun run --filter @clerk/cli-core build", + "build": "bun run build:compile", "dev": "bun run --cwd packages/cli-core dev", "test": "bun run scripts/check-bun-version.ts && bun test 'packages/cli-core/src/' 'packages/extras/src/' 'scripts/' --parallel --only-failures", - "test:e2e": "bun run scripts/check-bun-version.ts && bun test 'test/e2e/' --retry 1 --parallel --only-failures", + "test:e2e": "bun run scripts/check-bun-version.ts && bun --no-env-file test 'test/e2e/' --retry 1 --parallel --only-failures", "test:e2e:op": "bun run scripts/run-e2e-op.ts", "e2e:refresh-fixtures": "bun run scripts/refresh-e2e-fixtures.ts", "e2e:audit-fixtures": "bun run scripts/audit-e2e-fixtures.ts", diff --git a/packages/cli-core/package.json b/packages/cli-core/package.json index 280e41031..666ccbcf1 100644 --- a/packages/cli-core/package.json +++ b/packages/cli-core/package.json @@ -7,7 +7,6 @@ }, "type": "module", "scripts": { - "build": "bun build ./src/cli.ts --outfile ./dist/cli.js --target node --external @napi-rs/keyring --minify", "build:compile": "bun build --compile --minify --no-compile-autoload-dotenv --no-compile-autoload-bunfig ./src/cli.ts --outfile ./dist/clerk", "dev": "bun run ./src/cli.ts", "typecheck": "tsc --noEmit -p tsconfig.json", @@ -22,11 +21,13 @@ "@commander-js/extra-typings": "^15.0.0", "@napi-rs/keyring": "^1.3.0", "commander": "^15.0.0", + "csv-parser": "^3.2.1", "env-paths": "^4.0.0", "external-editor": "^3.1.0", "magicast": "^0.5.4", "semver": "^7.8.5", - "yaml": "^2.9.0" + "yaml": "^2.9.0", + "zod": "^4.4.3" }, "devDependencies": { "@clerk/shared": "^4.30.2", diff --git a/packages/cli-core/src/cli-program.ts b/packages/cli-core/src/cli-program.ts index 6ad326c6b..405f62363 100644 --- a/packages/cli-core/src/cli-program.ts +++ b/packages/cli-core/src/cli-program.ts @@ -24,6 +24,7 @@ import { registerCompletion } from "./commands/completion/index.ts"; import { registerUpdate } from "./commands/update/index.ts"; import { registerDeploy } from "./commands/deploy/index.ts"; import { registerWebhooks } from "./commands/webhooks/index.ts"; +import { registerMigrate } from "./commands/migrate/index.ts"; import { getEnvironment } from "./lib/config.ts"; import { setCurrentEnv, @@ -84,6 +85,7 @@ const registrants: CommandRegistrant[] = [ registerUpdate, registerDeploy, registerWebhooks, + registerMigrate, registerExtras, ]; diff --git a/packages/cli-core/src/commands/completion/__complete.ts b/packages/cli-core/src/commands/completion/__complete.ts index 7921e307e..642281f9e 100644 --- a/packages/cli-core/src/commands/completion/__complete.ts +++ b/packages/cli-core/src/commands/completion/__complete.ts @@ -1,4 +1,5 @@ import type { CommandUnknownOpts, Option } from "@commander-js/extra-typings"; +import { sources } from "../migrate/sources/registry.ts"; import { KNOWN_DASHBOARD_PATHS } from "../open/dashboard-paths.ts"; const DIRECTIVE = { @@ -33,6 +34,15 @@ const HTTP_METHOD_COMPLETIONS: Completion[] = [ { name: "DELETE", description: "Delete resource" }, ]; +/** + * The built-in migrate sources. `--source` also takes a path to a source you + * wrote, so it cannot use `.choices()`. + */ +const SOURCE_COMPLETIONS: Completion[] = sources.map((entry) => ({ + name: entry.key, + description: entry.label, +})); + /** * Hardcoded option-value completions for options that don't use Commander's `.choices()`. * Keys are the long or short flag (e.g., "--mode", "-X"). @@ -52,6 +62,7 @@ const KNOWN_OPTION_VALUES: Record = { { name: "latest", description: "Latest stable release" }, { name: "canary", description: "Latest canary (pre-release) build" }, ], + "--source": SOURCE_COMPLETIONS, "--for": [ { name: "orgs", description: "Organizations only" }, { name: "users", description: "Users only" }, @@ -66,6 +77,7 @@ const KNOWN_OPTION_VALUES: Record = { * to root, e.g. "open dashboard". */ const KNOWN_POSITIONAL_COMPLETIONS: Record = { + "migrate sources": SOURCE_COMPLETIONS, "open dashboard": KNOWN_DASHBOARD_PATHS.map((path) => ({ name: path, description: "Dashboard subpath", diff --git a/packages/cli-core/src/commands/init/scan.ts b/packages/cli-core/src/commands/init/scan.ts index 7dbc599a3..6f8458906 100644 --- a/packages/cli-core/src/commands/init/scan.ts +++ b/packages/cli-core/src/commands/init/scan.ts @@ -54,6 +54,11 @@ const AUTH_LIBRARY_SCANS: AuthLibraryScan[] = [ name: "Better Auth", docsUrl: "https://clerk.com/docs/migrations/overview", }, + { + packages: ["@workos-inc/authkit-nextjs", "@workos-inc/node"], + name: "WorkOS", + docsUrl: "https://clerk.com/docs/migrations/overview", + }, { packages: ["@kinde-oss/kinde-auth-nextjs"], name: "Kinde", diff --git a/packages/cli-core/src/commands/migrate/README.md b/packages/cli-core/src/commands/migrate/README.md new file mode 100644 index 000000000..beb8e2d34 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/README.md @@ -0,0 +1,1245 @@ +# `clerk migrate` + +Migrate users into a Clerk instance from another auth provider, or from another +Clerk instance. + +``` +clerk migrate export [platform] [-o ] [--json] +clerk migrate import [file|export-run-id] [--source ] [--dry-run] [--allow-partial] [--new-run] [--yes] [--json] +clerk migrate runs [run-id] [--json] +clerk migrate undo [--dry-run] [--yes] [--json] +clerk migrate sources [source] [--json] +clerk migrate help +``` + +A migration is usually three steps: + +```sh +clerk migrate export supabase # 1. a run, holding export.json +clerk migrate import 20260929-141502-a1b2 --dry-run # 2. check it against the instance +clerk migrate import 20260929-141502-a1b2 --yes # 3. import it +``` + +`clerk migrate undo ` takes an import back out, and `clerk migrate runs` +shows what every run did. Every subcommand but `sources` takes `--runs-dir ` +(or `CLERK_MIGRATE_DIR`) to keep its runs somewhere else. +`clerk migrate` on its own is a group name, not a command: it prints its help. + +## The rules + +Every command follows these: + +1. **Nothing writes without consent.** Consent is a yes at a terminal prompt, or + `--yes`. Without either, `import` and `undo` print what they would do and + exit 2 with the command to run. `--json` means non-interactive: it never prompts. +2. **`--dry-run` checks against the real instance, and writes nothing.** An + import's [checks](#checks) run before anything is written. Predicted + rejects stop the import unless `--allow-partial` is passed; fields that would + be dropped are warnings. +3. **State lives in one place: the [run store](#the-run-store).** Each run + records its target, its file, and every source ID → Clerk ID outcome, + including the error for each user who failed. `runs`, `undo`, re-runs and + exports all read or write it. +4. **Every command prints its target first:** the environment, app and + instance, and where the key came from. +5. **Every subcommand takes `--json`.** Exit codes: `0` all good, `1` some users + failed, `2` a usage error or a refusal, and `130` (death by SIGINT) when + Ctrl-C stops an import or an undo partway. The UI goes to stderr and data to + stdout. + + While users are created or deleted, a terminal shows a bar and the counts under it, not + a spinner: + + ``` + │ ██████████████████████████████████████████████████████░░░░░░░░░░░░░░░░░░ 75% + │ 7,500/10,000 users · ✓ 7,425 created · ✗ 75 failed · ~25s left + ``` + + A spinner takes the keyboard and exits 0 on Ctrl-C, so a script that deletes + the export once the import succeeds would delete it after an interrupted + one. Without a terminal, the counts are printed at each 10%. + +## Targeting and auth + +`clerk migrate import` resolves its Backend API key through the CLI's standard +chain: + +| Flag | Description | +| -------------------- | ------------------------------------------------------------- | +| `--secret-key ` | Use a specific Backend API secret key directly | +| `--app ` | Target an application directly, even outside a linked project | +| `--instance ` | Target `dev`, `prod`, or a full instance ID | + +Resolution order: `--secret-key` → `--app` + Platform API lookup → +`CLERK_SECRET_KEY` → the keyless project's own key → a linked project profile +from `clerk link`. + +With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks the +instance, so `import` and `undo` refuse (exit 2) an `--instance` that names a +different one. `--instance dev` next to an exported `sk_live_…` key would otherwise write +to production. + +The **instance type is read from the key**: `sk_live_…` is treated as +production, anything else as development. That choice drives the throughput +defaults and the development-instance user limit below. + +**Every command prints its target first.** `import` and `undo` name the +instance — its +environment, its app when the key came from one, and its ID from +`GET /v1/instance` — and where the key came from: `--secret-key`, `--app`, the +`CLERK_SECRET_KEY` env var, an accountless app's `.env.local`, or the linked +profile. An export names its source platform instead, and `export clerk` the +instance it reads. `runs` names the runs folder. `--json` carries the same +facts as `target`. + +``` +Target: My App (app_2x9k…), production instance ins_2x9k… +Key from: linked profile +``` + +The instance ID is what a run records, so `undo` and re-runs can tell whether +the key now in use still addresses the same instance. + +## The run store + +Every import, export and undo is a **run**, and the run store is the one place +`clerk migrate` keeps state. + +### Where runs are kept + +The first of these that is set: + +1. `--runs-dir ` +2. `CLERK_MIGRATE_DIR` +3. `/.clerk/migrate/` + +The project root is the linked profile's directory, then the git toplevel, then +the current directory. The first run written to the default location adds +`.clerk/` to the project's `.gitignore`, because run files carry user data. That +happens only once there is consent to write: a dry run or a refused import +leaves `.gitignore` alone. + +### What a run holds + +Each run is a folder named for its ID, `YYYYMMDD-HHmmss-xxxx`: + +| File | Contents | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `run.json` | Kind, status, start and finish times, the target, the source, the file and its sha256, and the counts | +| `users.ndjson` | One line per user outcome: `sourceId`, `clerkId`, `status`, and `reason`, `error`, `code`, `pending` or `passwordDropped` when present | +| `lock` | The PID of the process writing the run, while it runs | + +A user's status is `creating`, `created`, `failed`, `skipped`, `deleted` or +`exported`. The last line for each `sourceId` wins. A `429` retry, an extra +email or phone that did not attach, a first phone Clerk refused (which the +summary also counts), and a validation failure all land in the line's `error` +field. + +`creating` is written as a user's `POST /v1/users` goes out. It stays the +latest line when no answer says whether the create landed: an abort, a +network error, or a 5xx. A `created` line with `pending` lists the extra emails +and phones not yet attached. + +A run is `partial` when any user failed, was skipped, is still `creating` or +was never sent (`counts.notSent`), and `complete` otherwise. A run whose process +died, or that never recorded a finish time, lists as `interrupted`. A Ctrl-C +leaves a run that way, and prints its run ID and folder on the way out. A lock +held by another live process refuses a second writer with exit 2, and names the +lock file to delete if that process is not a migrate run. A lock holding this +process's own PID is stale: in a container the CLI often gets the same PID every +run. + +Run folders are created owner-only (`0700`), and export files `0600`: they hold +password hashes and user data. + +`users.ndjson` writes are synchronous appends, so a run interrupted with Ctrl-C +still leaves a complete record of everything already processed. A line that +cannot be written stops that user's create from going out. An export's file +lands in its run folder as `export.json` unless `--output` says otherwise. + +### Why `users.ndjson` is NDJSON + +One JSON object per line, rather than one JSON array per file. A migration is a +long append-only stream, and that format is the one that survives it: + +- **Appendable.** Each entry is written as it happens, without rewriting the + file. A JSON array would have to be re-serialized on every user. +- **Crash-safe.** Kill the process at any point and every line already written + is still valid. A truncated array is not parseable at all. +- **Streamable.** `tail -f` shows a long import progressing live, and analysis + reads line by line instead of loading a million-user record into memory. + +Which is also why it greps usefully without any tooling: + +```sh +grep '"status":"created"' .clerk/migrate/20260929-141502-a1b2/users.ndjson | wc -l +grep '"sourceId":"user_123"' .clerk/migrate/20260929-141502-a1b2/users.ndjson +``` + +## Commands + +The direction is always spelled out — `migrate import` moves users **into** +Clerk, `migrate export` gets them **out** of a source platform — so neither is +implied by the group. + +### `clerk migrate export` + +Gets users **out** of a source platform, so there is something to feed +`clerk migrate import`. + +```sh +clerk migrate export # pick a platform +clerk migrate export clerk --output users.json +clerk migrate export auth0 --domain my-tenant.us.auth0.com \ + --client-id … --client-secret … +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export firebase --service-account ./service-account.json +clerk migrate export workos --api-key sk_… +``` + +The platform is an optional positional. Omitted, you get a picker built from +the registry; given, it runs directly. Each platform resolves its own flags — +what Auth0 needs (a tenant domain and M2M credentials) has nothing in common +with what a database export needs. + +**A credential the far end rejects is asked for again.** Connection strings, +Firebase service account keys and Auth0 client secrets are all long, pasted by +hand, masked as they are typed, and wrong in ways nothing local can check: a typo'd host, a revoked key, +an expired token, the right server but the wrong database. Only the connection +or the token exchange can say, and by then the operator has answered every +other question the command asked. So that step — and only that step, +never a fetch already under way or a file already written — runs inside a +retry: the failure is explained, the prompt comes back, and the rest of the +export continues against whichever credential worked. Agent mode and a non-TTY +fail outright instead, having nobody to ask, and `-y` fails too, having been +told not to. + +| Platform | Source | Feeds | +| ------------ | -------------------------------- | --------------------- | +| `clerk` | Clerk Backend API | `--source clerk` | +| `auth0` | Auth0 Management API | `--source auth0` | +| `supabase` | Supabase Postgres (`auth.users`) | `--source supabase` | +| `authjs` | Auth.js database | `--source authjs` | +| `betterauth` | Better Auth database | `--source betterauth` | +| `firebase` | Firebase Identity Toolkit | `--source firebase` | +| `workos` | WorkOS User Management API | `--source workos` | + +Every export is a [run](#the-run-store), and the file lands in the run +folder as `export.json`. `--output` writes it somewhere else instead, +resolved against the **current directory** like every other path flag here; +the run still records where. Nothing is asked about where the file goes. + +The file is an envelope around the users: + +```json +{ + "clerkMigrate": 1, + "source": "clerk", + "exportedAt": "2026-09-29T14:15:02.000Z", + "runId": "20260929-141502-a1b2", + "users": [ … ] +} +``` + +`source` is what lets `clerk migrate import ` run with no +`--source`. A Firebase export adds `firebase`, the project's hash +parameters, so the import needs no `--firebase-*` flags. + +`--json` prints the result on stdout instead — `{ target, run, output, users, +coverage, next }` — and never prompts, so a missing credential exits 2 naming +the flag to pass. + +| Flag | Platforms | Description | +| -------------------------- | ---------------------------------- | --------------------------------------------------------- | +| `-o, --output ` | all | Write the export here instead of the run folder | +| `-y, --yes` | all | Do not prompt: fail on a rejected credential | +| `--json` | all | Print the result as JSON; never prompts | +| `--runs-dir ` | all | Where runs are kept (see [the run store](#the-run-store)) | +| `--db-url ` | `supabase`, `authjs`, `betterauth` | Postgres, MySQL, libsql/Turso or SQLite connection string | +| `--service-account ` | `firebase` | Path to a service account key JSON file | +| `--domain ` | `auth0` | Tenant domain, e.g. `my-tenant.us.auth0.com` | +| `--client-id ` | `auth0` | Machine-to-machine application client ID | +| `--client-secret ` | `auth0` | Machine-to-machine application client secret | +| `--api-key ` | `workos` | WorkOS secret API key, the one starting `sk_` | +| `--with-identities` | `workos` | Also record each user's OAuth providers | +| `--no-with-identities` | `workos` | Skip the OAuth provider fan-out without being asked | + +`export clerk` also takes the targeting flags — it reads from a Clerk instance, +so it resolves a key the same way `clerk migrate import` does, with one extra +step. The linked project is usually the migration's _destination_, so taking it +as the source without asking is how a run exports an instance and imports it +back into itself. Instead: + +- **Naming the instance runs unquestioned.** `--secret-key `, `--app`, + `--instance`, or an exported `CLERK_SECRET_KEY` — any of them is a sentence + you typed for this run, so none of them opens a picker. That is what makes + the export scriptable outside agent mode, and it keeps an exported key + outranking the linked profile here the way it does everywhere else in the + CLI. +- Anything resolved on your behalf — the linked project, a keyless app — is + never taken silently. A picker of every **instance** on your account opens + instead — one flat row each, `my-app - Production instance (ins_…)`, not an + application picker followed by an instance picker — with the resolved + application's instances listed **first** so taking one is still a single + Enter. Only when there are no instances to offer does it stop and list + `--secret-key`, `--app`/`--instance` and `clerk link` instead. +- With nothing to resolve at all (no link, no key, no flags), you get the + application picker `clerk users` uses — `Select a Clerk application to use:`, + followed by an instance picker when the application has more than one — + rather than an error about an unlinked directory. That is `clerk link`'s + picker, so it does offer `+ Create a new application`; a brand-new + application has no users to export, so it is never the answer here. + +The instance picker (the second tier) has no "create a new application" choice. +Its rows are searchable by what they show, so typing an application name, +`production`, or an instance id all narrow it. + +In agent mode the resolved instance is used without a prompt; pass +`--secret-key` or `--app`/`--instance` to be explicit. + +After each export you get a field-coverage table — which Clerk-relevant fields +were present on how many users — so you know the data is thin _before_ you +import it, not after: + +``` +Field coverage + ✓ 3/3 have an email address + ✗ 0/3 have a phone number + ! 1/3 have a username + ! 2/3 have a password (not exportable — see below) + +Exported 3 users to /project/.clerk/migrate/20260929-141502-a1b2/export.json +Run 20260929-141502-a1b2. See each user with `clerk migrate runs 20260929-141502-a1b2`. + +Import them with: + clerk migrate import 20260929-141502-a1b2 + + Imports into whichever instance the resolved secret key belongs to. + For production, add `--instance prod` or use a production secret key. +``` + +The import command prints through the same channel as the coverage table +rather than the gutter's **Next steps** outro, which is human-only — an agent +would otherwise be told what was exported and never how to import it. + +There is one command, not a development and a production variant, because no +flag's absence means "development" — the resolved key decides, through +`--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the linked +profile in that order. + +The export run has one line per exported user, so `clerk migrate runs` lists it +alongside imports. + +#### Four platforms export no passwords + +- **Clerk** never returns password digests, TOTP secrets or backup codes over + the API — only the `*_enabled` booleans. Migrated users must reset their + password in the destination instance. +- **Auth0**'s Management API does not return password hashes either; they come + only from a support request. Add a `passwordHash` field to each user before + importing, or migrate without passwords. +- **WorkOS** returns neither password hashes nor TOTP secrets, and has no + support-request escape hatch: hashes go in on import and never come back, and + `totp.secret` is returned on enrol only. There is nothing to add to the file. +- **Auth.js** core stores no passwords. An app that also uses the Credentials + provider keeps them in its own tables, which the export does not read: + migrate those separately, or have those users reset their password. + +All four say so on every run. The coverage row counts users who _have_ a +password, so the size of the gap is visible up front — `workos` prints that row +at zero unconditionally, because zero is the only value it can take. + +#### Database-backed exports (`supabase`, `authjs`, `betterauth`) + +These three read the database directly, over **`--db-url`**: + +```sh +clerk migrate export supabase --db-url "postgres://postgres:...@db.xxx.supabase.co:5432/postgres" +clerk migrate export authjs --db-url "mysql://user:...@127.0.0.1:3306/authjs" +clerk migrate export betterauth --db-url "./db.sqlite" +clerk migrate export betterauth --db-url "libsql://app-org.turso.io?authToken=..." # or set TURSO_AUTH_TOKEN +``` + +Postgres and MySQL go through `Bun.sql`; SQLite through `bun:sqlite`; +`libsql://` (Turso) over the server's HTTP pipeline endpoint, since `bun:sqlite` +only opens local files and `@libsql/client` ships native optional dependencies. +Nothing native ships in the binary — that is the whole reason the `engines.bun` +floor exists. Resolution is `--db-url`, then `SUPABASE_DB_URL` / `AUTHJS_DB_URL` +/ `BETTERAUTH_DB_URL`, then a masked prompt, since a connection string carries the password inline. A password +pasted unencoded (`#`, `@`, `/` and the like) is percent-encoded for you. A libsql +token comes from `?authToken=` on the URL, or from `TURSO_AUTH_TOKEN` / +`LIBSQL_AUTH_TOKEN`, and is redacted like a password. + +**Connection strings are redacted everywhere.** Errors show +`postgres://***@host/db`, including when the password itself contains an +unencoded `@` — the most common mistake, and exactly when the string ends up in +an error message. + +Connection failures get a hint rather than a driver error. Bun reports both an +unreachable host and a closed port as "Connection closed", so: + +| Situation | What you are told | +| ------------------------ | ------------------------------------------------------------------------------------------------- | +| Host or port unreachable | Check the host and port. On Supabase: use the pooler connection string, or enable the IPv4 add-on | +| Credentials rejected | Check the user and password | +| Table missing | Check the database name and SELECT permission. On Supabase: enable Auth, connect as `postgres` | +| SQLite file missing | Check the path and that the file is readable | + +**`supabase` reads the database rather than the Admin API** because +`encrypted_password` exists only there. An API-based export would force every +user to reset their password; this one carries the bcrypt digests across. It +also keeps `raw_app_meta_data`, which is what the import's +[checks](#checks) read for each user's providers. + +**`authjs` tries `User`, then `user`, then `users`.** Auth.js has no single +schema — Prisma capitalizes the table, Drizzle does not, and Postgres treats +the difference as significant once quoted. The run reports which one it found. +The verified column is read as `emailVerified`, or `email_verified` on a legacy +NextAuth table. Auth.js core stores no passwords, so its users arrive without +credentials. That is all an OAuth or email-link app has. An app that also uses +the Credentials provider keeps passwords in its own tables, which the export +does not read: migrate those separately, or have those users reset their +password. + +**`betterauth` reads its schema before it queries.** It finds the tables +(`user` and `account`, or `users` and `accounts` under `usePlural: true`), how +the columns are spelled (camelCase, or the snake_case Better Auth's Drizzle +generator writes by default), and which plugin columns exist. The username +plugin adds `username`, admin adds `banned` (carried only while `banExpires` is +unset or in the future), phone-number adds `phoneNumber`, and so on; selecting a column that is not there fails the whole query, and the +database answers the question better than the user can. Passwords come from a +`LEFT JOIN` onto the credential `account` row — left, not inner, so a user who +only ever signed in with OAuth is still exported. + +#### `firebase` + +```sh +clerk migrate export firebase --service-account ./service-account.json +``` + +Create a key at **Project settings → Service accounts → Generate new private +key**. The account needs to read users and the project's password hash +parameters (`signIn.hashConfig`). Either of these works, both verified live: + +- **Firebase Authentication Admin** (`roles/firebaseauth.admin`). The simplest + option, but it can also modify users and auth settings. +- **Read-only:** Firebase Authentication Viewer (`roles/firebaseauth.viewer`) + plus a custom role that adds `firebaseauth.configs.getHashConfig`: + + ```sh + gcloud iam roles create clerkMigrateHashExport --project=PROJECT_ID \ + --title="Clerk migrate hash export" \ + --permissions=firebaseauth.configs.get,firebaseauth.configs.getHashConfig,firebaseauth.users.get + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=roles/firebaseauth.viewer + gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:SA_EMAIL --role=projects/PROJECT_ID/roles/clerkMigrateHashExport + ``` + +Without `firebaseauth.configs.getHashConfig`, users still export, but the +export can't read the hash parameters. It says which permission is missing, and +the import then needs `--firebase-signer-key`, `--firebase-salt-separator`, +`--firebase-rounds` and `--firebase-mem-cost` (from **Authentication → Users → +⋮ → Password hash parameters** in the Firebase console). + +Without `--service-account` you are prompted for it, the way `export supabase` +prompts for its connection string. The answer can be a path to the downloaded +file _or_ the key's JSON pasted whole, so a key kept in a password manager or a +CI secret never has to be written to disk. The prompt is masked, since the key +carries a private key. Agent mode cannot prompt, so it names the flag instead. + +Either way the key is validated before anything reaches the network, so +downloading the web app config by mistake fails in a second with the right +console page named rather than after an auth round-trip. Key material never +appears in output. + +Firebase's scrypt is a modified variant, so a digest is worthless without the +project's four hash parameters. The export **reads them from the project** and +saves them in the export file's envelope, so the import needs nothing more: + +``` +Password hash parameters +Read from the project and saved in the export file, so the import needs nothing more. +``` + +Reading the config needs a broader role than listing users, so if it is denied +the export still succeeds and points at **Authentication → Users → (⋮) → +Password hash parameters** instead. An export with no password hashes says so +and asks for nothing. + +A user whose hash is present but whose salt is not (or the reverse) has both +dropped: half a credential produces a user nobody can sign in as. + +`FIREBASE_AUTH_EMULATOR_HOST` is honoured, so this works against the local +Firebase emulator as well as production. The target line names the emulator +when it is set. + +**No `firebase-admin`.** The spike the plan called for was run and _passed_ — a +compiled binary can import the SDK and complete `listUsers`, so the known +Firestore-under-compile bug does not reach the Auth Admin surface. It was still +not adopted: the SDK is 74 MB across 158 packages, including Firestore and +Cloud Storage, which would roughly double the ~62 MB binary every user +downloads, to serve one subcommand. What it does here is two REST calls and an +RS256 JWT, and Bun's Web Crypto signs RS256 with no dependency at all. + +#### Auth0 credentials + +Needs a machine-to-machine application with the `read:users` scope +(Applications → APIs → Auth0 Management API → Machine to Machine +Applications). Resolved from flags, then `AUTH0_DOMAIN` / `AUTH0_CLIENT_ID` / +`AUTH0_CLIENT_SECRET`, then a prompt. In agent mode a prompt is impossible, so +it exits naming **every** missing credential at once rather than one per run. + +Auth0 pages this endpoint only through the first **1000** users. Past that the +export stops and says so, pointing at Auth0's bulk export job — silently +returning the first thousand would read as "that is everyone". It still exits +0, and `--json` carries `truncated: true`. A tenant of exactly 1000 gets the +warning too: Auth0 reports a larger tenant's total as 1000, so the two look the +same. The bulk job's NDJSON file imports as it is. + +A credential the platform rejects (400, 401 or 403) is asked for again at a +terminal. An outage, a `429` or a refused connection is not: another +credential would not fix it, and it exits 1. + +#### WorkOS credentials + +Needs a secret API key — the one starting `sk_`, from the WorkOS dashboard +under API Keys. Resolved from `--api-key`, then `WORKOS_API_KEY`, then a +prompt; agent mode exits naming both instead. + +There is no `--db-url` sibling because there is no database to point it at. +WorkOS is API-only: apps commonly mirror users into their own store through +webhooks, but that mirror is a derived copy holding no credentials, so the +User Management API is the only source. Pagination is cursor-based, so unlike +Auth0 there is no record ceiling — `after` runs to the end of the tenant. + +**`--with-identities` is off by default, and it is not free.** WorkOS has no +bulk endpoint for OAuth identities, so it is one request per user: ten requests +becomes 1,010 for a thousand users. Nothing it returns can be imported — +`POST /v1/users` has no external-accounts field — so it buys a provider +breakdown in the coverage report, and an `identities` array kept in the export +file for whoever runs the migration. The interactive path asks once, after the +user count is known, defaulting to no; agent mode takes the flag's answer and +asks nothing. `-y` answers the question `yes`, so pass +`--no-with-identities` to skip the fan-out without being asked. + +The breakdown prints as its own **OAuth providers** block under the coverage +table, not as extra coverage rows: + +``` +Field coverage + ✓ 6/6 have an email address + ✗ 0/6 have a password (WorkOS returns none) + +OAuth providers + GoogleOAuth 2 users + MicrosoftOAuth 1 user + no OAuth provider 2 users + not readable 1 user + Those users have no `identities` field in the export, rather than an empty one. +``` + +Separate because the two kinds of row do not mean the same thing. A coverage +row is "N of the M users have this field"; a provider row has no such +denominator — one user holding two providers is counted under both, so the +counts can sum past the user count, and `not readable` is not a property of the +user at all. + +A lookup that fails is counted on its own row rather than folded into +`no OAuth provider`. "Lookup failed" and "has no providers" are different +facts, and flattening the first into the second would understate social +sign-in. + +Non-interactive runs get progress on stderr every 500 users during the fan-out, +and every 10 pages during the user fetch. `withSpinner` hands a no-op to +anything that is not a TTY, so without this an agent exporting a large tenant +would see nothing at all until the run finished. + +### `clerk migrate import` + +Reads an exported user file, maps it onto Clerk's user schema, checks every +user against the destination instance, and creates them through the Backend +API. + +```sh +clerk migrate import 20260929-141502-a1b2 --dry-run # check, write nothing +clerk migrate import 20260929-141502-a1b2 --yes # an export run +clerk migrate import users.json --source supabase --yes # any other file +clerk migrate import users.json --source clerk --allow-partial --yes +clerk migrate import users.json --source clerk --new-run --yes +clerk migrate import users.json --source clerk --json --yes +clerk migrate import users.json --source clerk --require-password --yes +clerk migrate import users.json --source clerk --skip-legal-checks --yes +clerk migrate import users.json --source workos --reserve-unverified --yes +clerk migrate import users.json --source firebase --firebase-signer-key SIGNER_KEY \ + --firebase-salt-separator SALT_SEPARATOR --firebase-rounds 8 --firebase-mem-cost 14 --yes +clerk migrate import users.json --source clerk --runs-dir ./runs --yes +clerk migrate import users.json --source clerk --app app_123 --instance prod --yes +clerk migrate import users.json --source clerk --secret-key sk_test_... -y +clerk migrate import # a human is asked +``` + +| Flag | Description | +| --------------------------------------- | --------------------------------------------------------------------------------------------- | +| `[file\|export-run-id]` | The export file, or the ID of the export run that wrote it | +| `--source ` | Where the file came from: a [source](#sources), or one you wrote | +| `--dry-run` | Run the [checks](#checks) against the instance, and write nothing | +| `--allow-partial` | Import the users that pass, and record the rest as skipped | +| `--new-run` | Start a new run instead of [continuing](#re-running) an earlier one | +| `--require-password` | Import only users that carry a password digest | +| `--skip-legal-checks` | Import users with no legal acceptance into an instance requiring it | +| `--reserve-unverified` | Create [unverified identifiers](#verified-vs-unverified-identifiers) reserved, not unverified | +| `--firebase-signer-key ` | Firebase base64 signer key (overrides the export file) | +| `--firebase-salt-separator ` | Firebase base64 salt separator | +| `--firebase-rounds ` | Firebase scrypt rounds | +| `--firebase-mem-cost ` | Firebase scrypt memory cost | +| `-y, --yes` | Import without prompting | +| `--json` | Output as JSON. Never prompts, so importing needs `--yes` | +| `--runs-dir ` | Where runs are kept (see [the run store](#the-run-store)) | + +Plus the targeting flags from the table above: `--secret-key`, `--app` and +`--instance`. + +An export run ID stands for the file that run wrote: if that file is gone, or +has changed since (an `--output` path another export or an edit overwrote), the +import exits 2 and imports nothing. The import records it +as `fromExport`. A file `clerk migrate export` wrote carries its source, so it +needs no `--source`, and a `--source` that contradicts it exits 2. Any other +file needs `--source`: a JSON array, Firebase's own `{ "users": [...] }`, a CSV, or NDJSON, one user per line (what +Auth0's bulk export job writes). NDJSON is read always for `.ndjson` and +`.jsonl`, and for a `.json` file that doesn't parse whole. A leading BOM is +ignored in JSON and CSV. A file that isn't valid JSON is named in the error. + +**What a human is asked, and what an agent is told.** A human at a terminal who +leaves out the file is asked for its path, and is asked for a source only when +the file does not name one. An agent, a non-TTY run, or `--json` without the +file, or without `--source` for a file that does not name one, exits 2 naming +what to pass. + +**Nothing is written without consent.** After the checks, a human is asked +`Import N users?`, and declining writes nothing. `--yes` skips the question. +Without either — an agent, a non-TTY run, `--json` — the run prints the checks +and exits 2 with the exact command to run. Printed commands shell-quote their +paths, keep `--json`, and put `` in place of a secret key and +`SIGNER_KEY`, `SALT_SEPARATOR`, `ROUNDS` and `MEM_COST` in place of the +Firebase parameters. + +**Every run prints its target first**, then which [case](#re-running) applies, +then the checks. + +Failures do not stop the run: each user's outcome is written to the +[run](#the-run-store) and the import continues. A `429` backs off — +honouring `Retry-After` when the response carries it — and retries up to 5 +times before the user is recorded as failed. The command exits 1 if any user +failed. + +`--require-password` records each user it leaves out as `skipped`, so the run +ends `partial`. + +`--json` returns `{ target, run, resume, checks, result }`. When a run stops +before importing, it carries one of `dryRun: true`, `refused: true`, +`consent: "required"` or `nothingToImport: true` in place of `result`, and a +file already imported in full returns `alreadyImported: true`. With +`--require-password`, `withoutPassword` counts the users it left out before the +checks, so `checks.total` plus it is the file's size. + +#### Re-running + +Running the same import again continues where it left off. The match is the +file's sha256, the source (and a custom source's content hash), and the +instance ID; the latest matching import run decides what happens: + +| Latest match | Re-running does | +| ----------------------------------------- | ---------------------------------------------------------------------- | +| none | a new run | +| interrupted (dead lock or no finish time) | continues the same run, skipping the users it created | +| `partial` | continues the same run, retrying the users that failed or were skipped | +| `complete` | nothing: prints "Already imported in run …" and exits 0 | +| `undone` | a new run | +| has an undo that did not finish | exits 2, naming the `clerk migrate undo` that finishes it | + +`--new-run` skips the lookup. A run another live process holds exits 2. When +Clerk cannot name the instance (`GET /v1/instance` failed, often from rate +limiting right after a large import) and a run of this file exists under a real +instance ID, it exits 2 rather than starting over: try again, or pass +`--new-run`. A continue whose run changed while it waited at the prompt, from +an undo or another continue, exits 2 too, with nothing written. + +A continued run also finishes what the last one left open: + +- A user still `creating` is looked up by `external_id`. A match that + carries this run's marker is adopted as `created`, and not created again. + A match without the marker is someone else's user: the checks reject that + record as already in the instance. Only when the lookup finds no user at + all is it created. + +Every create sends the run's ID in the user's private metadata, as +`clerkMigrateRun`, merged with any private metadata the source carries. It +is how a cut-off create is told apart from a user an app or another tool +made with the same `external_id`, and it stays on the user. A run recorded +before the marker existed adopts nothing. + +- A user whose `created` line has `pending` identifiers gets just those + attaches. + +When an import completes, it names the folders it no longer needs: the export it +read, which holds your users' data, and its own run, which only `undo` needs. +Each comes with the `rm -rf` to remove it. + +#### Checks + +Every import runs the checks before writing anything, and `--dry-run` stops +after them. They sort the users three ways: + +- **Rejected** — users Clerk would refuse. Each gets the first reason that + applies: + - it failed schema validation + - its source requested a skip (Better Auth: an anonymous guest; Supabase: a + soft-deleted user) + - its only emails are ones Clerk refuses: malformed, or a domain that can't + receive mail (`.local`, `.invalid`, `.test`, `.example`, `.arpa`, + `.internal`, `.lan`, `.corp` and the like). Such an email is dropped from + any other user, with a warning + - it lacks an identifier the instance requires. An email or phone counts + only when it is verified, because an unverified one is attached after the + user exists, or with `--reserve-unverified`, which creates it reserved + - it has no identifier left once the emails and phones Clerk would refuse + are stripped: those of an instance that neither has them on nor signs in + or does MFA with them. A username is kept: Clerk stores it with usernames + off + - it lacks a first or last name the instance requires + - it has an authenticator app secret or backup codes, and the instance has + that turned off. Importing it without them would take away its second + factor, so the checks offer to turn the setting on instead + - it has no password, and password is the instance's only way to sign in + (no code, link, social or SSO sign-in; a passkey or a password reset does + not count, as in Clerk) + - it has no legal acceptance on record, and the instance requires legal + consent. `--skip-legal-checks`, or a yes at the prompt, imports these users + without it (`skip_legal_checks`), with a warning + - its username breaks the instance's username rules (length, letters, + the allowed special characters). The checks offer the setting that allows + it. With usernames off, such a username is dropped instead, with a warning + - its password is not the shape its hasher says (`bcrypt`, with a cost up to + 15, `scrypt_firebase`, `argon2i`/`argon2id` and `scrypt_werkzeug` are + checked; other hashers are not, and Clerk refuses a bad one at create) + - Supabase: its only providers are ones Clerk has off, or doesn't offer at all + (Figma, Kakao, Keycloak, WorkOS, Zoom, Fly), and it has no verified email or + phone the instance signs in with by code or link. The checks offer to turn + on the first kind; nothing can turn on the second + - its source ID, primary email, primary phone or username repeats an + earlier user in the file that passes the checks above. Only what the + create sends counts: an extra email is attached after it, and a stripped + one never goes out. Usernames compare case-insensitively and + phones ignore punctuation. The first record in the file is kept, whatever + either holds, and the reject names it (`kept: …`) + - the instance already has a user with its source ID, email, phone or + username (a batched `GET /v1/users` lookup, 100 values a request, through + the scheduler). A user a continued run found behind its own interrupted + create does not count + - a development instance: it is past the 100-user headroom + (`CLERK_MIGRATE_DEV_USER_LIMIT` when Clerk raised it), counted in file + order +- **Imported, but not everything comes across** — emails or phones the + instance is not set up to store, fields Clerk has no place for (`Clerk won't +store: …`), passwords a source had to drop, emails Clerk refuses (malformed, + or a domain that can't receive mail), and names Clerk refuses (a phone + number, a URL, HTML, blank, or over 256 bytes: Better Auth's phone sign-up + stores the number as the name). A password, username or name whose setting + is off is stored, and works or shows once it is turned on. +- **Imported** — everyone else. + +Any reject stops the import, and it exits 2 with the command that adds +`--allow-partial`. With `--allow-partial`, the rest import and each reject is +recorded as `skipped` with its reason. `--dry-run` exits 2 when the real run +would be refused, and 0 otherwise. + +``` +Checks + 120 users checked + ✗ 12 users rejected + 12: only has an unverified email, and this instance requires an email + u_17, u_22, u_40, u_51, u_88, and 7 more + ⚠ Imported, but not everything comes across + 6 users have a username, which this instance does not use: it is stored, and works only once usernames are turned on + Clerk won't store: department (120 users) + ✓ 108 users to import + +Or change the instance instead + Make Email optional at sign-up + clerk config patch --app app_… --instance ins_… --json '{"auth_email":{"required_for_sign_up":false}}' + Enable Username + clerk config patch --app app_… --instance ins_… --json '{"auth_username":{"used_for_sign_up":true}}' +``` + +Each fix names its instance with `--instance`, so it changes the instance the +import targets, whatever the key's source. A key from `--secret-key` or +`CLERK_SECRET_KEY` names no app, so its fix reads `--app APP_ID`: fill in the +app that owns the instance. When Clerk could not name the +instance (a `key_…` fallback ID), the fix points at the Dashboard instead: in +`--json` it carries `url` in place of `command`. + +The fixes are offers, not corrections: an instance that requires an email is +configured as its owner intended, and fixing the export may be the answer. When +the instance settings cannot be read (BAPI `/v1/domains` → the instance's +Frontend API `/v1/environment`), required fields are not checked and the run +says so. When a development instance's user count cannot be read, the run says +so too, rather than checking the headroom against zero. + +#### Sign-up restrictions + +Every create sends `skip_restriction_checks: true`. The instance's allowlist, +blocklist, disposable-email and subaddress rules police new sign-ups, and these +users already signed up on the source platform. + +#### Additional identifiers + +Only the first verified email and phone go on `POST /v1/users`. Every +additional verified identifier, and every unverified one, is attached +afterwards with its own request, ahead of any create still queued, and backs +off on a `429` like the create. A refusal there is logged and the user still +counts as imported — a duplicate secondary email should not undo an otherwise +successful user. An attach with no answer stays `pending` for a re-run. + +The first phone gets the same treatment when Clerk refuses it — a country the +instance does not support, or a number that is not E.164 — and the user has an +email: the create is retried without the phone. The user counts as imported, +and the summary lists them under "Imported without their phone", by Clerk's +reason (`result.warnings` in `--json`). + +#### Throughput + +Defaults follow Clerk's documented `POST /v1/users` limits: 100 req/s for +production instances, 10 req/s for development. Concurrency defaults to ~95% of +that, assuming ~100ms of API latency. Both are overridable: + +| Variable | Effect | +| --------------------------------- | ------------------------------------------------------------ | +| `CLERK_MIGRATE_RATE_LIMIT` | Requests per second | +| `CLERK_MIGRATE_CONCURRENCY_LIMIT` | Concurrent in-flight requests | +| `CLERK_MIGRATE_DEV_USER_LIMIT` | Development-instance user limit the checks use (default 100) | + +A non-numeric or non-positive value is ignored in favour of the default. + +A development instance's user limit is checked with the other +[checks](#checks): new development instances are created with a 100-user limit, +production instances have none by default (a plan can set one; the import +stops at the first refusal), and the run reads the live count +(`GET /v1/users/count`). The limit itself is not served by any API, so for a +development instance Clerk has raised, set `CLERK_MIGRATE_DEV_USER_LIMIT` to +the raised limit; `--allow-partial` imports up to the headroom. + +Users that do exceed the limit come back in the error breakdown as +`You have reached your limit of N users`, annotated with what a development +instance can do about it. The first refusal stops the import: the users it +never sent are counted under "Not sent" (`result.notSent` in `--json`, and +`counts.notSent` in `run.json`). Once the limit is raised, +[run the import again](#re-running) to send them. + +### `clerk migrate runs` + +`runs` reads the [run store](#the-run-store). + +```sh +clerk migrate runs # every run, newest first +clerk migrate runs 20260929-141502-a1b2 # one run in full +clerk migrate runs --json +``` + +| Flag | Description | +| ------------------- | ----------------------------- | +| `[run-id]` | Show one run instead of all | +| `--json` | The same data, on stdout | +| `--runs-dir ` | Read runs from somewhere else | + +It prints the runs folder first. The listing shows each run's ID, date, kind, +status, target, file and counts. `runs ` adds the error breakdown and the +users that failed or were skipped, with the path to the full record. An unknown +ID exits 2. + +### `clerk migrate undo` + +Deletes the users an import run created. The import run is the whole record of +what to delete: every source ID whose latest line is `created`, by the Clerk ID +recorded beside it. + +The one search is for a source ID whose latest line is `creating`: the run +stopped with that user's `POST /v1/users` sent and unanswered, so Clerk may +hold the user without its ID on record. Those are looked up by `external_id`, +and only a user carrying the import run's `clerkMigrateRun` marker is deleted: +an app or another tool can make a user with the same source ID, and one +without the marker is left alone. + +```sh +clerk migrate undo 20260929-141502-a1b2 --dry-run # preview, delete nothing +clerk migrate undo 20260929-141502-a1b2 # confirms first +clerk migrate undo 20260929-141502-a1b2 --yes # no prompt +clerk migrate undo 20260929-141502-a1b2 --json --yes +``` + +| Flag | Description | +| ------------------- | -------------------------------------------------------- | +| `` | The import run to undo | +| `--dry-run` | Show the preview and delete nothing | +| `-y, --yes` | Delete without prompting | +| `--json` | Output as JSON. Never prompts, so deleting needs `--yes` | +| `--runs-dir ` | Read runs from somewhere else | + +Plus the targeting flags: `--secret-key`, `--app` and `--instance`. + +It prints the target first, then a preview: how many users will be deleted, and +how many of them have signed in since the import (from each user's +`last_sign_in_at`). When users remain to delete or record as gone, nothing is +deleted without consent: a yes at the prompt, or `--yes`. Without either — an +agent, a non-TTY run, or `--json` — it prints the preview and exits 2 with the +command to run. When none remain, undo completes without prompting. + +It refuses with exit 2, and deletes nothing, when: + +- the resolved key addresses a different instance than the run imported into + (the error names both) +- the run is not an import run +- the run has already been undone + +Deletes go through the same scheduler and `429` backoff as the import. A user +already gone from the instance counts as deleted. The undo is a run of its own, +`kind: "undo"` with `undoes: `. The import is marked `undone` only when +every user is deleted. A partial undo exits 1, and running `undo` again retries +the users that failed, in the same undo run. + +### `clerk migrate sources` + +```sh +clerk migrate sources # every source, with what it carries +clerk migrate sources betterauth # one source in full +clerk migrate sources ./my-source.ts # a source you wrote +clerk migrate sources --json +``` + +| Flag | Description | +| ---------- | ---------------------------------------------------------------- | +| `[source]` | A built-in key, or the path to a source you wrote, to show fully | +| `--json` | The same data, on stdout | + +`sources` alone prints the table under [Sources](#sources). `sources ` shows one source in +full: its export command, what it carries with a note for each, where each +field lands (`encrypted_password → password`), its fixed defaults, and any +caveats. An unknown key exits 2 and lists the valid ones. There is no +intro/outro gutter: this reads a static registry rather than running anything. + +### `clerk migrate help` + +`clerk migrate help` and `clerk migrate --help` print the help for the +group or one command, with examples. `clerk migrate help ` does the +same. + +## Sources + +A source maps one platform's export onto Clerk's user schema, and says what it +brings across. Adding a platform is one file in `sources/` plus one line in +`sources/registry.ts`; `--source`'s tab-completion reads from that array. + +| Key | Reads | Passwords | MFA | Metadata | +| ------------ | ----------------------------- | --------- | ------- | -------- | +| `clerk` | Clerk Dashboard export | partial | partial | partial | +| `auth0` | Auth0 Export Users API | partial | no | yes | +| `authjs` | Auth.js / NextAuth user table | no | no | no | +| `betterauth` | Better Auth export | yes | no | no | +| `firebase` | `firebase auth:export` | yes | no | no | +| `supabase` | Supabase `auth.users` export | yes | no | partial | +| `workos` | WorkOS User Management API | no | no | yes | + +There is no column for social sign-ins, because no source copies them and none +needs to. Every source shows the same note instead: enable the same providers in +Clerk, and a user who signs in with one is linked to their imported account by +verified email. See +[account linking](https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking). + +### `--source` + +`clerk migrate import` takes `--source `: + +- A value starting with `./`, `../` or `/`, or ending in `.ts`, `.js` or + `.mjs`, is loaded as a [custom source](#custom-sources). +- Anything else must be a built-in key. An unknown key exits 2 with the list of + valid keys. + +A file `clerk migrate export` wrote names its own source, so it needs none. + +### Custom sources + +Migrating from a platform with no built-in, without recompiling the CLI: + +```sh +clerk migrate import users.json --source ./my-platform.ts +``` + +The file lives in **your** project, not in the CLI, and is imported at runtime. +It exports the same shape the built-ins use — plain data, no imports, since +there is nothing in a compiled binary for your file to import from: + +```ts +export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform's admin console.", + transformer: { + account_ref: "userId", // required: becomes the Clerk user's external_id + contact_email: "email", + given: "firstName", + family: "lastName", + pw_bcrypt: "password", + }, + carries: { + passwords: { level: "yes", note: "bcrypt hashes from the pw_bcrypt column." }, + mfa: { level: "no", note: "Not exported." }, + metadata: { level: "no", note: "Not exported." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { + if (!user.firstName) delete user.firstName; + }, +}; +``` + +TypeScript is fine — Bun's transpiler is part of the runtime, so `interface`, +`satisfies` and `as const` all work in a file the compiled binary imports. +Plain `.js` works too. + +An import run records a custom source's key and a hash of the file, so an +edited source counts as a different source. The hash covers that file only: +a local helper it imports is not part of it, so after editing one, pass +`--new-run` rather than continue a run made with the old helper. + +#### Validation + +The file is code the CLI executes, so its shape is checked before use and +rejected with the specific problem rather than crashing mid-pipeline: + +| Problem | Message | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| Path does not exist | `No source file at /abs/path.ts.` | +| No default export, but a named one | ``has no default export. Found named export `myPlatform` — did you mean `export default`?`` | +| Does not parse | `Could not load ./f.ts: Expected identifier but found ","` | +| Nothing maps to `userId` | ``no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id`` | +| No `carries` | `` `carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note } `` | +| `key` clashes with a built-in | `key is "clerk", which is already a built-in source` | +| `preTransform` or `postTransform` is not a function | `` `preTransform` must be a function when present `` (likewise `postTransform`) | + +The `userId` check is the load-bearing one: without it the import would run to +completion and create every user with no `external_id`, which is what makes a +migration re-runnable. + +### Metadata + +Metadata a user can edit on the source platform — Auth0's `user_metadata`, +Supabase's `raw_user_meta_data`, WorkOS's `metadata` — goes to Clerk's +`unsafe_metadata`, which is the user-editable one. Public metadata is read-only +to the user, so putting it there would take away an edit the user had. Auth0's +`app_metadata` goes to `private_metadata`. A Clerk Dashboard CSV carries no metadata. + +### Better Auth passwords + +Better Auth hashes with its own scrypt by default and lets an app swap in bcrypt +or argon2, so one database can hold more than one kind. The hasher is detected +per user: + +| Stored value | Sent as | +| --------------------------- | ---------------------------------------------------------- | +| `<32 hex>:<128 hex>` | `scrypt:16384:16:1$$`, hasher `scrypt_werkzeug` | +| `$2a$`, `$2b$` or `$2y$` | `bcrypt` | +| `$argon2id$` or `$argon2i$` | `argon2id` or `argon2i` | +| anything else | dropped | + +Better Auth's scrypt uses the hex salt string as the salt and a 64-byte key, +which is what `scrypt_werkzeug` verifies once N, r and p are written inline. It +also normalizes a password to NFKC before hashing, and Clerk does not, so a +password whose NFKC form differs will not verify and that user resets it. +`clerk migrate sources betterauth` lists that caveat. + +A password Clerk cannot verify is **dropped, not rejected**: the user imports +without it and can sign in another way or reset it. Their run line carries +`passwordDropped: true`. + +### Verified vs unverified identifiers + +Every platform records verification differently, and each source declares +which style it uses. An identifier the source never confirmed is routed to +`unverifiedEmailAddresses` / `unverifiedPhoneNumbers` rather than the primary +field, because Clerk creates primary identifiers **verified** — sending an +unconfirmed address there would silently promote it. + +- **Boolean** (`auth0`, `betterauth`, `firebase`, `workos`): `true`/`false`. A CSV export stringifies these, so `"false"` is + read as false, not as a non-empty string. `TRUE`, `FALSE`, `t` and `f` read + too, as a spreadsheet or psql writes them. +- **Timestamp** (`authjs`, `supabase`): a nullable confirmation time. Any real value + means verified; `""`, `null` and `\N` do not. + +A Clerk export keeps an unverified primary email or phone unverified. + +**Unverified or reserved.** By default an unverified identifier is attached +after the user exists (`POST /v1/email_addresses`, `verified: false`). The user +cannot sign in with it, and another user can claim it by verifying it first. +`--reserve-unverified`, or a yes at the prompt a human gets when the file has +any, creates them **reserved** instead, on `POST /v1/users` through +`email_address_identification_status` / `phone_number_identification_status`. +A reserved identifier is unverified, but usable for sign-in and locked to the +user, and becomes verified the first time the user signs in with it. That is +how most source platforms treat an unconfirmed address, but it lets the user +sign in with one nobody proved they own, so it is opt-in. `-y`, `--json` and +agent mode never ask, and keep them unverified without the flag. A continued +run uses whichever the flag or answer says that time. + +### Firebase hash parameters + +Firebase uses a modified scrypt, so Clerk needs the project's four parameters +alongside each digest. Find them in the Firebase console under +**Authentication → Users → (⋮) → Password hash parameters**, and put them in +place of `SIGNER_KEY`, `SALT_SEPARATOR` and the two numbers: + +```sh +clerk migrate import users.json --source firebase -y \ + --firebase-signer-key SIGNER_KEY --firebase-salt-separator SALT_SEPARATOR \ + --firebase-rounds 8 --firebase-mem-cost 14 +``` + +All four are **required as a set** — supplying some but not all is a usage error +naming what is missing. A partial set produces a well-formed digest that +verifies against nothing, so users would import successfully and then be unable +to sign in. + +The flags are read first, then the export file's envelope, which carries the +parameters when `clerk migrate export firebase` could read them from the +project. The flags win, so a rotated key can be passed without re-exporting. +The envelope sits in the gitignored run folder, since the signer key is a +Firebase secret. + +An export with no password hashes needs no parameters at all. + +## Schema fields + +What a source maps _onto_. Every user is validated against this schema +before any request is made, so a field a source produces that is not listed +here is dropped — Zod strips unknown keys — and never reaches Clerk. The checks +warn about each one (`Clerk won't store: …`). Writing a custom source means +targeting these names exactly. + +The schema lives in `validator.ts`; adding a platform means adding a source, +not editing it. + +**Required:** `userId` (`string`). It becomes the Clerk user's `external_id`, +which is what makes a migration re-runnable. + +**Identifiers.** At least one of these must be present, or the import's checks +reject the user as invalid. Each accepts a single value or an array. + +| Field | Type | Description | +| -------------------------- | -------------------- | ---------------------------------- | +| `email` | `string \| string[]` | Primary verified email address(es) | +| `emailAddresses` | `string \| string[]` | Additional verified emails | +| `unverifiedEmailAddresses` | `string \| string[]` | Unverified emails | +| `phone` | `string \| string[]` | Primary verified phone number(s) | +| `phoneNumbers` | `string \| string[]` | Additional verified phones | +| `unverifiedPhoneNumbers` | `string \| string[]` | Unverified phones | +| `username` | `string` | Username | + +**Profile, password and 2FA.** + +| Field | Type | Description | +| -------------------- | ---------- | --------------------------------------------------- | +| `firstName` | `string` | First name | +| `lastName` | `string` | Last name | +| `password` | `string` | The hashed password from the source platform | +| `passwordHasher` | `enum` | **Required whenever `password` is set** (see below) | +| `totpSecret` | `string` | TOTP secret | +| `backupCodesEnabled` | `boolean` | Whether backup codes are enabled | +| `backupCodes` | `string[]` | Backup codes | + +Clerk verifies the digest as-is, so `passwordHasher` must name the algorithm the +source actually used: + +`argon2i`, `argon2id`, `awscognito`, `bcrypt`, `bcrypt_peppered`, +`bcrypt_sha256_django`, `hmac_sha256_utf16_b64`, `ldap_ssha`, `md5`, +`md5_phpass`, `md5_salted`, `phpass`, `pbkdf2_sha1`, `pbkdf2_sha256`, +`pbkdf2_sha256_django`, `pbkdf2_sha512`, `pbkdf2_sha512_hex`, `scrypt_firebase`, +`scrypt_werkzeug`, `sha256`, `sha256_salted`, `sha512_symfony` + +A user with an unrecognized hasher is rejected by the checks, naming the +hasher, like any other invalid user. + +**Metadata.** + +| Field | Type | Description | +| ----------------- | -------- | -------------------------------------------------------- | +| `unsafeMetadata` | `object` | Readable **and writable** by the client — never trust it | +| `publicMetadata` | `object` | Readable by the client, writable only server-side | +| `privateMetadata` | `object` | Server-side only | + +**Account state.** These are passed straight through to `POST /v1/users`, and +are how a Clerk-to-Clerk migration keeps original signup dates instead of +stamping every user with today's. + +| Field | Type | Description | +| --------------------------- | --------- | ------------------------------------------------------------------------------------ | +| `createdAt` | `string` | Original creation timestamp. A number is epoch milliseconds, or seconds below `1e11` | +| `legalAcceptedAt` | `string` | When legal terms were accepted | +| `banned` | `boolean` | Whether the user is banned | +| `bypassClientTrust` | `boolean` | Skip client trust verification | +| `createOrganizationEnabled` | `boolean` | Whether the user can create orgs | +| `createOrganizationsLimit` | `number` | Maximum orgs the user can create | +| `deleteSelfEnabled` | `boolean` | Whether the user can delete their account | +| `skipLegalChecks` | `boolean` | Skip legal acceptance checks | +| `skipPasswordChecks` | `boolean` | Skip password requirements on import | + +## API endpoints + +| Method | Path | Used by | +| -------- | ----------------------------------------------- | ------------------------------------------------------------------------------- | +| `POST` | `/v1/users` | `migrate import` — creates each user | +| `POST` | `/v1/email_addresses` | `migrate import` — attaches additional emails | +| `POST` | `/v1/phone_numbers` | `migrate import` — attaches additional phones | +| `GET` | `/v1/users?limit=&offset=&order_by=+created_at` | `migrate export clerk` — pages the whole instance, oldest first, 500 at a time | +| `GET` | `/v1/users/count` | `migrate import` — headroom against a development instance's user limit | +| `GET` | `/v1/users?external_id=…` | `migrate import` — checks for users already in the instance, 100 values a call | +| `GET` | `/v1/users?external_id=…` | `migrate undo` — finds users whose create was in flight when the import stopped | +| `GET` | `/v1/users?user_id=…` | `migrate undo` — reads the imported users back, 100 a call | +| `DELETE` | `/v1/users/{user_id}` | `migrate undo` — deletes one user | +| `GET` | `/v1/instance` | `migrate import`, `undo`, `export clerk` — names the instance behind the key | +| `GET` | `/v1/domains` | `migrate import` checks — resolves the Frontend API host | + +The checks also read the instance's Frontend API `GET /v1/environment` +(bootstrapping a dev browser first on development instances) for its +attributes and enabled social providers. Nothing in `clerk migrate` writes +instance settings: the checks print the `clerk config patch` to run instead. + +Three exports talk to their own platform rather than to Clerk: + +| Method | Path | Used by | +| ------ | ---------------------------------------------- | -------------------------------------------------------- | +| `POST` | `https:///oauth/token` | `export auth0` — Management API access token | +| `GET` | `https:///api/v2/users` | `export auth0` — 100 per page, 1000 users maximum | +| `POST` | `https://oauth2.googleapis.com/token` | `export firebase` — RS256 assertion → access token | +| `GET` | `…/v1/projects/{project_id}/accounts:batchGet` | `export firebase` — pages users, 1000 at a time | +| `GET` | `…/admin/v2/projects/{project_id}/config` | `export firebase` — reads the scrypt hash parameters | +| `GET` | `…/user_management/users` | `export workos` — 100 per page, cursor-paginated | +| `GET` | `…/user_management/users/{id}/identities` | `export workos` — `--with-identities` only, one per user | + +The two Identity Toolkit paths are on `identitytoolkit.googleapis.com`, or on +`FIREBASE_AUTH_EMULATOR_HOST` when that is set. The two WorkOS paths are on +`api.workos.com`. + +The three database exports (`supabase`, `authjs`, `betterauth`) connect over +`--db-url`. A `libsql://` URL is the one exception that goes over HTTP: it +posts to the server's pipeline endpoint. + +## Notes + +- `userId` in the source file becomes the Clerk user's `external_id`. That is + what makes a migration re-runnable and reversible. +- CSV input is coerced before validation: `a@x.dev,b@x.dev` and `["a@x.dev"]` + both become arrays, `"true"`/`1` become booleans, and JSON metadata columns + are parsed. An empty column is dropped rather than sent as null. +- A user must end up with at least one identifier (email, phone or username). + Users that do not are rejected by the checks as invalid. diff --git a/packages/cli-core/src/commands/migrate/export/auth0.test.ts b/packages/cli-core/src/commands/migrate/export/auth0.test.ts new file mode 100644 index 000000000..e40fe1690 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.test.ts @@ -0,0 +1,488 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { setAssumeYes } from "../lib/assume-yes.ts"; +import { + buildAuth0Export, + exportAuth0, + fetchAllAuth0Users, + fetchAuth0Token, + mapAuth0UserToExport, + isAuth0Domain, + normalizeAuth0Domain, + resolveAuth0Credentials, +} from "./auth0.ts"; + +const captured = useCaptureLog(); + +const CREDENTIALS = { domain: "t.auth0.com", clientId: "cid", clientSecret: "csec" }; + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expauth0-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const auth0User = (i: number, overrides: Record = {}) => ({ + user_id: `auth0|a${i}`, + email: `a${i}@x.dev`, + email_verified: true, + given_name: `Given${i}`, + family_name: `Family${i}`, + ...overrides, +}); + +/** Stubs the token exchange plus one page of users per entry in `pages`. */ +function stubAuth0(pages: Record[][], token: Response | null = null) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ? JSON.parse(init.body as string) : null }); + + if (url.includes("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/oauth/token")) { + return token ?? Response.json({ access_token: "tok" }); + } + return Response.json({ + users: pages[page++] ?? [], + total: pages.reduce((sum, p) => sum + p.length, 0), + }); + }) as unknown as typeof fetch; +} + +describe("normalizeAuth0Domain", () => { + test.each([ + ["t.auth0.com", "t.auth0.com"], + ["https://t.auth0.com", "t.auth0.com"], + ["http://t.auth0.com/", "t.auth0.com"], + [" t.auth0.com ", "t.auth0.com"], + ])("%s -> %s", (input, expected) => { + expect(normalizeAuth0Domain(input)).toBe(expected); + }); +}); + +// The client secret goes to this host, so anything that could make URL parsing +// pick a different one is refused. +describe("isAuth0Domain", () => { + test.each(["t.auth0.com", "https://t.auth0.com/", "login.example.com"])( + "accepts %s", + (domain) => { + expect(isAuth0Domain(domain)).toBe(true); + }, + ); + + test.each([ + "t.auth0.com@attacker.example", + "t.auth0.com/path", + "t.auth0.com:8443", + "t.auth0.com?x=1", + "localhost", + ])("refuses %s", (domain) => { + expect(isAuth0Domain(domain)).toBe(false); + }); +}); + +describe("resolveAuth0Credentials", () => { + test("refuses a domain that would send the secret to another host", async () => { + await expect( + resolveAuth0Credentials( + {}, + { + AUTH0_DOMAIN: "t.auth0.com@attacker.example", + AUTH0_CLIENT_ID: "c", + AUTH0_CLIENT_SECRET: "s", + }, + ), + ).rejects.toThrow(/is not an Auth0 domain/); + }); + + test("prefers flags", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "flag.auth0.com", clientId: "f", clientSecret: "s" }, + { AUTH0_DOMAIN: "env.auth0.com" }, + ); + expect(resolved.domain).toBe("flag.auth0.com"); + }); + + test("falls back to the environment", async () => { + const resolved = await resolveAuth0Credentials( + {}, + { + AUTH0_DOMAIN: "env.auth0.com", + AUTH0_CLIENT_ID: "e", + AUTH0_CLIENT_SECRET: "s", + }, + ); + expect(resolved).toEqual({ domain: "env.auth0.com", clientId: "e", clientSecret: "s" }); + }); + + test("normalizes a domain that came with a scheme", async () => { + const resolved = await resolveAuth0Credentials( + { domain: "https://t.auth0.com/", clientId: "c", clientSecret: "s" }, + {}, + ); + expect(resolved.domain).toBe("t.auth0.com"); + }); + + // `-y` is "do not prompt", at a terminal too. + test("does not ask for missing credentials under -y", async () => { + process.env.CLERK_MODE = "human"; + setAssumeYes(true); + try { + await expect(resolveAuth0Credentials({}, {})).rejects.toThrow(/cannot prompt here/); + } finally { + setAssumeYes(false); + } + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names every missing credential at once rather than one at a time", async () => { + await expect(resolveAuth0Credentials({}, {})).rejects.toThrow( + /--domain \(or AUTH0_DOMAIN\), --client-id \(or AUTH0_CLIENT_ID\), --client-secret \(or AUTH0_CLIENT_SECRET\)/, + ); + }); + + test("names only what is actually missing", async () => { + await expect( + resolveAuth0Credentials({ domain: "t.auth0.com", clientId: "c" }, {}), + ).rejects.toThrow(/Missing: --client-secret \(or AUTH0_CLIENT_SECRET\)\./); + }); +}); + +describe("fetchAuth0Token", () => { + test("exchanges client credentials for the Management API audience", async () => { + stubAuth0([[]]); + + expect(await fetchAuth0Token(CREDENTIALS)).toBe("tok"); + expect(requests[0]?.url).toBe("https://t.auth0.com/oauth/token"); + expect(requests[0]?.body).toEqual({ + grant_type: "client_credentials", + client_id: "cid", + client_secret: "csec", + audience: "https://t.auth0.com/api/v2/", + }); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + stubAuth0( + [[]], + new Response(JSON.stringify({ error_description: "Wrong client secret" }), { status: 401 }), + ); + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow( + /Auth0 rejected the credentials \(401\): Wrong client secret/, + ); + }); + + // An outage is not a bad credential: no re-prompt, and exit 1, not 2. + test("a 5xx is an outage, not a rejected credential", async () => { + stubAuth0([[]], new Response("{}", { status: 503 })); + const error = (await fetchAuth0Token(CREDENTIALS).catch((e: unknown) => e)) as CliError; + expect(error.message).toContain("Auth0 did not issue a token (503)"); + expect(error.exitCode).not.toBe(EXIT_CODE.USAGE); + }); + + test("does not follow a redirect with the client secret", async () => { + let redirect: RequestInit["redirect"]; + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + redirect = init?.redirect; + return new Response(null, { + status: 307, + headers: { location: "https://elsewhere.example/" }, + }); + }) as unknown as typeof fetch; + + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/did not issue a token \(307\)/); + expect(redirect).toBe("manual"); + }); + + test("mentions the read:users scope, the usual cause", async () => { + stubAuth0([[]], new Response("{}", { status: 403 })); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(/read:users/); + }); + + test("fails when a 200 carries no token", async () => { + stubAuth0([[]], Response.json({})); + await expect(fetchAuth0Token(CREDENTIALS)).rejects.toThrow(CliError); + }); +}); + +describe("fetchAllAuth0Users", () => { + test("pages until a short page arrives", async () => { + stubAuth0([ + Array.from({ length: 100 }, (_, i) => auth0User(i)), + Array.from({ length: 4 }, (_, i) => auth0User(100 + i)), + ]); + + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); + + expect(all).toHaveLength(104); + expect(truncated).toBe(false); + expect(requests[0]?.url).toContain("page=0"); + expect(requests[1]?.url).toContain("page=1"); + expect(requests).toHaveLength(2); + }); + + // A short page is not always the last: the total says how many there are. + test("keeps paging past a short page until the total is reached", async () => { + stubAuth0([ + Array.from({ length: 100 }, (_, i) => auth0User(i)), + Array.from({ length: 50 }, (_, i) => auth0User(100 + i)), + Array.from({ length: 30 }, (_, i) => auth0User(150 + i)), + ]); + + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); + + expect(all).toHaveLength(180); + expect(truncated).toBe(false); + expect(requests).toHaveLength(3); + }); + + test("ends on a short page when Auth0 sends no total", async () => { + const pages = [Array.from({ length: 100 }, (_, i) => auth0User(i)), [auth0User(100)]]; + let page = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push({ url: input.toString(), body: null }); + return Response.json({ users: pages[page++] ?? [] }); + }) as unknown as typeof fetch; + + const { users: all } = await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + + expect(all).toHaveLength(101); + expect(requests).toHaveLength(2); + }); + + test("asks for totals and the documented page size", async () => { + stubAuth0([[]]); + await fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" }); + expect(requests[0]?.url).toContain("per_page=100"); + expect(requests[0]?.url).toContain("include_totals=true"); + }); + + // Auth0 caps offset pagination at 1000. Returning the first thousand quietly + // would read as "that is everyone". + test("stops at Auth0's 1000-record ceiling and says so", async () => { + stubAuth0( + Array.from({ length: 12 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); + + expect(all).toHaveLength(1000); + expect(truncated).toBe(true); + expect(captured.err).toContain("only pages through the first 1000 users"); + expect(captured.err).toContain("bulk user export job"); + }); + + // Auth0 reports a larger tenant's total as 1000, so 1000 of 1000 cannot be + // told apart from 1000 of 50,000. + test("a total of exactly 1000 may hide more, and says so", async () => { + stubAuth0( + Array.from({ length: 10 }, () => Array.from({ length: 100 }, (_, i) => auth0User(i))), + ); + + const { users: all, truncated } = await fetchAllAuth0Users({ + credentials: CREDENTIALS, + token: "tok", + }); + + expect(all).toHaveLength(1000); + expect(truncated).toBe(true); + expect(captured.err).toContain("so there may be more"); + }); + + test("does not follow a redirect with the bearer token", async () => { + let redirect: RequestInit["redirect"]; + globalThis.fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + redirect = init?.redirect; + return new Response(null, { + status: 302, + headers: { location: "https://elsewhere.example/" }, + }); + }) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 302 listing users/, + ); + expect(redirect).toBe("manual"); + }); + + test("raises a clear error on a failed page request", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllAuth0Users({ credentials: CREDENTIALS, token: "tok" })).rejects.toThrow( + /Auth0 returned 500 listing users/, + ); + }); +}); + +describe("mapAuth0UserToExport", () => { + test("keeps the fields the auth0 transformer maps from", () => { + expect( + mapAuth0UserToExport(auth0User(0, { phone_number: "+1555", created_at: "2025-01-01" })), + ).toEqual({ + user_id: "auth0|a0", + email: "a0@x.dev", + given_name: "Given0", + family_name: "Family0", + phone_number: "+1555", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test.each([ + ["email_verified", false], + ["phone_verified", false], + ])("keeps %s when it is %p", (field, value) => { + const mapped = mapAuth0UserToExport(auth0User(0, { [field]: value })); + expect(mapped[field]).toBe(value); + }); + + test("drops tenant internals the import has no use for", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { + identities: [{ provider: "auth0" }], + logins_count: 42, + last_login: "2026-01-01", + multifactor: ["guardian"], + }), + ); + for (const noise of ["identities", "logins_count", "last_login", "multifactor"]) { + expect(noise in mapped).toBe(false); + } + }); + + test("keeps blocked only when it is true", () => { + expect(mapAuth0UserToExport(auth0User(0, { blocked: true })).blocked).toBe(true); + expect("blocked" in mapAuth0UserToExport(auth0User(0, { blocked: false }))).toBe(false); + }); + + test("keeps name", () => { + expect(mapAuth0UserToExport(auth0User(0, { name: "Ada Lovelace" })).name).toBe("Ada Lovelace"); + }); + + test("omits empty metadata", () => { + const mapped = mapAuth0UserToExport( + auth0User(0, { user_metadata: {}, app_metadata: { plan: "pro" } }), + ); + expect("user_metadata" in mapped).toBe(false); + expect(mapped.app_metadata).toEqual({ plan: "pro" }); + }); +}); + +describe("buildAuth0Export", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; + const { users, coverage } = buildAuth0Export( + [auth0User(0), auth0User(1, { given_name: undefined })], + (line) => lines.push(line), + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + expect(lines.map((line) => line.status)).toEqual(["exported", "exported"]); + }); +}); + +/** The envelope the one export run in this project wrote. */ +function onlyExportFile(): string { + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); + expect(entries).toHaveLength(1); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; +} + +describe("exportAuth0", () => { + test("writes the default path and reports coverage", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS }); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "auth0", + }); + const written = exportedUsers(); + expect(written[0]?.user_id).toBe("auth0|a0"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubAuth0([[auth0User(0)], []]); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportAuth0({ ...CREDENTIALS, output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); + }); + + test("--output controls the destination", async () => { + stubAuth0([[auth0User(0)], []]); + + await exportAuth0({ ...CREDENTIALS, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + // Auth0 only releases hashes through a support request; finding that out + // after the import means nobody can sign in. + test("says plainly that password hashes are not in the file", async () => { + stubAuth0([[auth0User(0)], []]); + await exportAuth0({ ...CREDENTIALS }); + expect(captured.err).toContain("does not return password hashes"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/auth0.ts b/packages/cli-core/src/commands/migrate/export/auth0.ts new file mode 100644 index 000000000..2ac88013e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/auth0.ts @@ -0,0 +1,414 @@ +/** + * `clerk migrate export auth0` — pull users out of an Auth0 tenant. + * + * Ported from the standalone migration-tool's `src/export/auth0.ts`, but + * **without the `auth0` SDK**. The SDK is 28 MB across five transitive + * dependencies — including a bundled legacy copy of itself — to make two REST + * calls, and it does its own HTTP, so nothing it sends would appear under + * `--verbose`. `.claude/rules/debug-logging.md` requires library HTTP to go + * through `loggedFetch`; two direct calls satisfy that and ship nothing extra + * inside the compiled binary. + * + * **Passwords do not come out of the Management API.** Auth0 exports password + * hashes only via a support request. The coverage report says so rather than + * leaving it to be discovered when nobody can sign in. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt, text } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { isCredentialStatus, throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; +import { finishExport, startExportRun } from "./shared.ts"; + +const PAGE_SIZE = 100; + +/** + * Auth0 caps offset pagination on `GET /api/v2/users` at 1000 records. + * Past that the tenant needs a bulk export job, so the run says so instead of + * quietly returning the first thousand as though that were everyone. + */ +const AUTH0_PAGINATION_CEILING = 1000; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/auth0"; + +export type ExportAuth0Options = { + domain?: string; + clientId?: string; + clientSecret?: string; + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +export type Auth0Credentials = { + domain: string; + clientId: string; + clientSecret: string; +}; + +/** Strips a scheme and trailing slash, so both forms of `--domain` work. */ +export function normalizeAuth0Domain(domain: string): string { + return domain + .trim() + .replace(/^https?:\/\//, "") + .replace(/\/+$/, ""); +} + +/** + * True for a bare host name. The client secret goes to this host, so userinfo + * (`tenant.auth0.com@elsewhere`), a port, a path or a query is refused rather + * than letting URL parsing pick a different host. + */ +export function isAuth0Domain(domain: string): boolean { + return /^[a-z0-9-]+(\.[a-z0-9-]+)+$/i.test(normalizeAuth0Domain(domain)); +} + +const DOMAIN_HINT = "Pass just the tenant's host name, e.g. my-tenant.us.auth0.com."; + +/** + * Resolves the tenant credentials: flags, then environment, then a prompt. + * + * @throws CliError in agent mode when anything is still missing, naming each + * absent flag rather than failing on the first one. + */ +export async function resolveAuth0Credentials( + options: ExportAuth0Options, + env: Record = process.env, +): Promise { + const resolved = { + domain: options.domain ?? env.AUTH0_DOMAIN, + clientId: options.clientId ?? env.AUTH0_CLIENT_ID, + clientSecret: options.clientSecret ?? env.AUTH0_CLIENT_SECRET, + }; + + if (resolved.domain && !isAuth0Domain(resolved.domain)) { + throwUsageError(`"${resolved.domain}" is not an Auth0 domain. ${DOMAIN_HINT}`, DOCS_URL); + } + + const missing = ( + [ + ["domain", "--domain", "AUTH0_DOMAIN"], + ["clientId", "--client-id", "AUTH0_CLIENT_ID"], + ["clientSecret", "--client-secret", "AUTH0_CLIENT_SECRET"], + ] as const + ).filter(([key]) => !resolved[key]); + + if (missing.length === 0) { + return { + domain: normalizeAuth0Domain(resolved.domain as string), + clientId: resolved.clientId as string, + clientSecret: resolved.clientSecret as string, + }; + } + + // `-y` is "do not prompt", as for the other exports. + if (options.json || isAssumeYes() || isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export auth0\` needs credentials for a machine-to-machine application and cannot prompt here.\n` + + `Missing: ${missing.map(([, flag, variable]) => `${flag} (or ${variable})`).join(", ")}.`, + DOCS_URL, + undefined, + [ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + ], + ); + } + + log.info( + "Auth0 needs a machine-to-machine application with the `read:users` scope. Create one under Applications → APIs → Auth0 Management API → Machine to Machine Applications.", + ); + + return promptAuth0Credentials(resolved); +} + +/** + * Asks for whichever of the three are still missing. + * + * Called with nothing known after Auth0 has rejected a set: its error names no + * field, and the operator may have mistyped any of them — so all three are + * asked again rather than guessing which one to keep. + */ +export async function promptAuth0Credentials( + known: Partial = {}, +): Promise { + const domain = + known.domain ?? + (await text({ + message: "Auth0 tenant domain (e.g. my-tenant.us.auth0.com)", + validate: (value) => { + if (!value?.trim()) return "A domain is required"; + return isAuth0Domain(value) ? undefined : DOMAIN_HINT; + }, + })); + const clientId = + known.clientId ?? + (await text({ + message: "Machine-to-machine client ID", + validate: (value) => (value?.trim() ? undefined : "A client ID is required"), + })); + const clientSecret = + known.clientSecret ?? + (await passwordPrompt({ + message: "Machine-to-machine client secret", + validate: (value) => (value?.trim() ? undefined : "A client secret is required"), + })); + + return { + domain: normalizeAuth0Domain(domain), + clientId: clientId.trim(), + clientSecret: clientSecret.trim(), + }; +} + +/** Exchanges the client credentials for a Management API access token. */ +export async function fetchAuth0Token(credentials: Auth0Credentials): Promise { + const url = new URL(`https://${credentials.domain}/oauth/token`); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "POST", + // A followed 307/308 would resend the client secret to wherever it points. + // Unfollowed, a 3xx is just a failed response. + redirect: "manual", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + grant_type: "client_credentials", + client_id: credentials.clientId, + client_secret: credentials.clientSecret, + audience: `https://${credentials.domain}/api/v2/`, + }), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + const detail = body.error_description ?? body.error ?? "no access token returned"; + if (!response.ok && !isCredentialStatus(response.status)) { + throwApiFailure( + response.status, + `Auth0 did not issue a token (${response.status}): ${detail}. Try again shortly.`, + DOCS_URL, + ); + } + if (!body.access_token) { + throwUsageError( + `Auth0 rejected the credentials (${response.status}): ${detail}\n` + + "Check the domain, client ID and secret, and that the application is authorized for the Management API with the `read:users` scope.", + DOCS_URL, + ); + } + + return body.access_token; +} + +type Auth0User = Record & { user_id?: string }; + +/** Fetches one page of users from the Management API. */ +async function fetchAuth0Page( + credentials: Auth0Credentials, + token: string, + page: number, +): Promise<{ users: Auth0User[]; total: number }> { + const url = new URL(`https://${credentials.domain}/api/v2/users`); + url.searchParams.set("page", String(page)); + url.searchParams.set("per_page", String(PAGE_SIZE)); + url.searchParams.set("include_totals", "true"); + + const response = await loggedFetch(url, { + tag: "auth0", + method: "GET", + redirect: "manual", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + const body = await response.text(); + throwApiFailure( + response.status, + `Auth0 returned ${response.status} listing users: ${body}`, + DOCS_URL, + ); + } + + const body = (await response.json()) as { users?: Auth0User[]; total?: number }; + return { users: body.users ?? [], total: body.total ?? 0 }; +} + +/** + * Pages through the tenant's users. + * + * Stops at Auth0's 1000-record ceiling with a warning naming the bulk export + * job — silently truncating would read as "that is everyone". + */ +export async function fetchAllAuth0Users(options: { + credentials: Auth0Credentials; + token: string; + spinner?: SpinnerControls; +}): Promise<{ users: Auth0User[]; truncated: boolean }> { + const all: Auth0User[] = []; + + for (let page = 0; ; page++) { + const { users, total } = await fetchAuth0Page(options.credentials, options.token, page); + all.push(...users); + options.spinner?.update(`Fetching users from Auth0: ${all.length} so far...`); + + // An empty page ends it whatever `total` says, so an overstated total + // cannot page forever. + if (users.length === 0) break; + + if (all.length >= AUTH0_PAGINATION_CEILING) { + // Auth0 reports a larger tenant's total as 1000 too, so reaching the + // ceiling never proves the export is complete. + log.warn( + `Auth0 only pages through the first ${AUTH0_PAGINATION_CEILING} users on this endpoint` + + (total > AUTH0_PAGINATION_CEILING + ? `, and this tenant reports ${total}` + : `, and counts no higher, so there may be more`) + + ". Exported what is reachable; use Auth0's bulk user export job for the rest.", + ); + return { users: all, truncated: true }; + } + + // Auth0 can send a short page before the last one, so `total` decides + // when it is known; a short page ends it only when Auth0 sent no total. + if (total > 0 ? all.length >= total : users.length < PAGE_SIZE) break; + } + + return { users: all, truncated: false }; +} + +/** + * Keeps the fields the `auth0` transformer maps from. + * + * Deliberately a copy rather than the raw record: an Auth0 user carries + * identities, session counts and tenant internals that would bloat the export + * and mean nothing to the import. + */ +export function mapAuth0UserToExport(user: Auth0User): Record { + const exported: Record = {}; + + for (const field of [ + "user_id", + "email", + "username", + "name", + "given_name", + "family_name", + "phone_number", + "created_at", + ] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Verification flags are meaningful when false, so they are copied on + // presence rather than on truthiness. + for (const field of ["email_verified", "phone_verified"] as const) { + if (user[field] !== undefined) exported[field] = user[field]; + } + + // Only when true: every unblocked user would otherwise carry a `false`. + if (user.blocked === true) exported.blocked = true; + + for (const field of ["user_metadata", "app_metadata"] as const) { + const value = user[field]; + if (value && typeof value === "object" && Object.keys(value).length > 0) { + exported[field] = value; + } + } + + return exported; +} + +export type Auth0ExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildAuth0Export( + users: Auth0User[], + record: (line: UserLine) => void = () => {}, +): Auth0ExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0 }; + + for (const user of users) { + const userId = String(user.user_id ?? ""); + try { + const mapped = mapAuth0UserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.username) counts.username++; + if (mapped.given_name) counts.firstName++; + if (mapped.family_name) counts.lastName++; + if (mapped.phone_number) counts.phone++; + + record({ sourceId: userId, status: "exported" }); + } catch (error) { + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +export async function exportAuth0(options: ExportAuth0Options): Promise { + const resolved = await resolveAuth0Credentials(options); + + await withGutter("Exporting users from Auth0", async () => { + if (!options.json) printTarget({ platform: "auth0" }); + // Only Auth0 can say whether these three go together, and whether the + // application carries the `read:users` scope, so a rejected set is asked + // for again here. + const { value: token, input: credentials } = await withInputRetry( + resolved, + async () => promptAuth0Credentials(), + async (candidate) => { + log.info(`Exporting from ${candidate.domain}.`); + return withSpinner("Authenticating with Auth0...", async () => fetchAuth0Token(candidate)); + }, + options, + ); + + const { users, truncated } = await withSpinner( + "Fetching users from Auth0...", + async (spinner) => fetchAllAuth0Users({ credentials, token, spinner }), + ); + + const run = await startExportRun(options, { platform: "auth0" }); + const { users: exported, coverage } = buildAuth0Export(users, run.append); + finishExport({ run, options, users: exported, coverage, truncated }); + + if (exported.length > 0) { + log.warn( + "Auth0's Management API does not return password hashes. Request a password hash export from Auth0 support and add a `passwordHash` field to each user before importing, or migrate without passwords.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/authjs.ts b/packages/cli-core/src/commands/migrate/export/authjs.ts new file mode 100644 index 000000000..79c601862 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/authjs.ts @@ -0,0 +1,190 @@ +/** + * `clerk migrate export authjs` — read users out of an Auth.js database. + * + * Ported from the standalone migration-tool's `src/export/authjs.ts`, on the + * `Bun.sql`/`bun:sqlite` client. + * + * Auth.js has no export tool and no single schema: the adapter decides the + * table name, and Prisma's `User` differs from Drizzle's `user` only in + * casing — which Postgres and SQLite treat as significant once quoted. The + * export tries the documented casing first and falls back rather than making + * the user find out from a driver error. + */ + +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import { log } from "../../../lib/log.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { finishExport, startExportRun } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** Table names to try, in order. Prisma capitalizes; Drizzle does not. */ +const TABLE_CANDIDATES = ["User", "user", "users"] as const; + +type AuthJsRow = Record & { + id?: unknown; + name?: string | null; + email?: string | null; + email_verified?: unknown; +}; + +/** + * The verified-email column, in the order tried: the current adapters' name, + * then the legacy NextAuth `users` table's. + */ +const VERIFIED_COLUMNS = ["emailVerified", "email_verified"] as const; + +/** + * Every column is qualified with the table alias: SQLite reads an unqualified + * double-quoted name that matches no column as a string literal, so a missing + * `"emailVerified"` would come back as the text "emailVerified" on every row, + * which reads as verified. + */ +export function buildAuthJsQuery( + client: DbClient, + table: string, + verifiedColumn: (typeof VERIFIED_COLUMNS)[number] = "emailVerified", +): string { + const q = (identifier: string) => client.quote(identifier); + return ( + `SELECT u.${q("id")}, u.${q("name")}, u.${q("email")}, u.${q(verifiedColumn)} AS ${q("email_verified")} ` + + `FROM ${q(table)} u ORDER BY u.${q("id")} ASC` + ); +} + +const messageOf = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +/** True for an error that means "no such column". Postgres says "does not exist" for both. */ +function isMissingColumn(error: unknown): boolean { + return /no such column|column .* does not exist|unknown column/i.test(messageOf(error)); +} + +/** True for an error that means "wrong table name", not "broken connection". */ +function isMissingTable(error: unknown): boolean { + return /does not exist|no such table|doesn't exist|unknown table/i.test(messageOf(error)); +} + +/** + * Reads the user table, trying each casing until one answers. + * + * A table that exists but lacks the columns is passed over too: Postgres + * keeps a quoted `"User"` apart from `"user"`, and an app's own `User` table + * must not hide Auth.js's. When nothing reads, the first column error is the + * one thrown, since it names a table that was there. + * + * @returns The rows and the table they came from, so the run can say which. + */ +export async function fetchAuthJsUsers( + client: DbClient, +): Promise<{ rows: AuthJsRow[]; table: string }> { + let lastError: unknown; + let columnError: unknown; + + for (const table of TABLE_CANDIDATES) { + for (const column of VERIFIED_COLUMNS) { + try { + const rows = await client.query(buildAuthJsQuery(client, table, column)); + return { rows, table }; + } catch (error) { + // The table is there: try the other name for the verified column. + if (isMissingColumn(error)) { + columnError ??= error; + continue; + } + if (!isMissingTable(error)) throw error; + lastError = error; + break; + } + } + } + + if (columnError) throw columnError; + throw lastError instanceof Error + ? new Error( + `No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}. ${lastError.message}`, + ) + : new Error(`No Auth.js user table found. Tried ${TABLE_CANDIDATES.join(", ")}.`); +} + +export function buildAuthJsExport(rows: AuthJsRow[], record: (line: UserLine) => void = () => {}) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, name: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + const user: Record = { id: userId }; + + if (row.name) { + user.name = row.name; + counts.name++; + } + if (row.email) { + user.email = row.email; + counts.email++; + } + // A nullable timestamp, not a boolean: the transformer reads presence. + if (row.email_verified) { + user.email_verified = + row.email_verified instanceof Date ? row.email_verified.toISOString() : row.email_verified; + counts.emailVerified++; + } + + users.push(user); + record({ sourceId: userId, status: "exported" }); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a name", count: counts.name }, + ], + }; +} + +const AUTHJS_DB = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Auth.js adapter uses.", +} as const satisfies ResolveConfig; + +export async function exportAuthJs(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, AUTHJS_DB); + + await withGutter("Exporting users from Auth.js", async () => { + if (!options.json) printTarget({ platform: "authjs" }); + const { + value: { rows, table }, + } = await withInputRetry( + dbUrl, + async () => promptDbUrl(AUTHJS_DB), + async (connectionString) => + withSpinner("Reading the user table...", async () => + withDbClient(connectionString, "authjs", fetchAuthJsUsers), + ), + options, + ); + log.info(`Read ${rows.length} row${rows.length === 1 ? "" : "s"} from ${table}.`); + + const run = await startExportRun(options, { platform: "authjs" }); + const { users, coverage } = buildAuthJsExport(rows, run.append); + finishExport({ run, options, users, coverage }); + + if (users.length > 0) { + log.warn( + "Auth.js core stores no passwords, so users arrive without one. That covers OAuth and email-link sign-in. " + + "If the app also uses the Credentials provider, its passwords live in the app's own tables and were not exported: " + + "migrate them separately, or have those users reset their password.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/betterauth.ts b/packages/cli-core/src/commands/migrate/export/betterauth.ts new file mode 100644 index 000000000..d06022626 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/betterauth.ts @@ -0,0 +1,318 @@ +/** + * `clerk migrate export betterauth` — read users out of a Better Auth database. + * + * Ported from the standalone migration-tool's `src/export/betterauth.ts`, on + * the `Bun.sql`/`bun:sqlite` client. + * + * Better Auth's schema depends on which plugins are enabled, so the columns + * are **detected from the schema** rather than asked for: the username plugin + * adds `username`, admin adds `banned`, phone-number adds `phoneNumber`, + * anonymous adds `isAnonymous`, and + * so on. Selecting a column that is not there fails the whole query, and + * asking the user which plugins they run is a question their database can + * already answer. + * + * Passwords live on the `account` row for the credential provider, not on the + * user, which is why the export joins. + */ + +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { withDbClient, type DbClient } from "../lib/db.ts"; +import { finishExport, startExportRun } from "./shared.ts"; +import { + promptDbUrl, + resolveDbUrl, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +/** Columns a Better Auth plugin adds to the user table. */ +export const PLUGIN_COLUMNS = [ + "username", + "displayUsername", + "phoneNumber", + "phoneNumberVerified", + "role", + "banned", + "banReason", + "banExpires", + "twoFactorEnabled", + "isAnonymous", +] as const; + +export type PluginColumn = (typeof PLUGIN_COLUMNS)[number]; + +/** Columns every Better Auth install has. */ +const CORE_COLUMNS = ["id", "email", "emailVerified", "name", "createdAt", "updatedAt"] as const; + +/** How one Better Auth database names its tables and columns. */ +export type BetterAuthSchema = { + userTable: string; + accountTable: string; + /** + * The column for a field. Better Auth's Drizzle generator writes snake_case + * (`email_verified`) unless the project sets `camelCase: true`; its Kysely + * and Prisma setups keep camelCase. + */ + column: (field: string) => string; + /** + * The same for the account table. Better Auth maps each model's fields on + * its own, so its casing is read from that table rather than assumed. + */ + accountColumn: (field: string) => string; + /** The plugin columns this database has. */ + plugins: Set; +}; + +/** `emailVerified` → `email_verified`, as Better Auth's Drizzle generator does it. */ +function toSnakeCase(field: string): string { + return field + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2") + .replace(/([a-z\d])([A-Z])/g, "$1_$2") + .toLowerCase(); +} + +/** + * Every column of a table, or an empty set when there is no such table. + * + * SQLite has no `information_schema`, so it goes through `PRAGMA` — and the + * PRAGMA takes the table name inline rather than as a bind parameter. + */ +async function tableColumns(client: DbClient, table: string): Promise> { + if (client.dbType === "sqlite") { + const rows = await client.query<{ name: string }>(`PRAGMA table_info(${client.quote(table)})`); + return new Set(rows.map((row) => row.name)); + } + if (client.dbType === "postgres") { + // `to_regclass` resolves the name through the whole search_path, as the + // export's unqualified SELECT will. `current_schema()` is only the first + // schema there that exists, which misses tables in `public` whenever a + // schema named after the role comes first. + const rows = await client.query<{ column_name: string }>( + `SELECT attname AS column_name FROM pg_attribute + WHERE attrelid = to_regclass(quote_ident(${client.placeholder(1)}::text)) + AND attnum > 0 AND NOT attisdropped`, + [table], + ); + return new Set(rows.map((row) => row.column_name)); + } + const rows = await client.query<{ column_name?: string; COLUMN_NAME?: string }>( + `SELECT column_name FROM information_schema.columns + WHERE table_name = ${client.placeholder(1)} AND table_schema = DATABASE()`, + [table], + ); + // MySQL 8 answers with an upper-case column label. + return new Set(rows.map((row) => row.column_name ?? row.COLUMN_NAME ?? "")); +} + +/** Table names to try, in order: Better Auth's default, then `usePlural: true`. */ +const TABLE_CANDIDATES = [ + ["user", "account"], + ["users", "accounts"], +] as const; + +/** + * A field's column: as named when the table has it, else its snake_case form + * when the table has that, else the table's general guess. Better Auth maps + * each field on its own, so one table can mix the two; a column that is truly + * missing still fails the query, which names it. + */ +function resolveColumn(columns: Set, snake: boolean): (field: string) => string { + return (field) => { + if (columns.has(field)) return field; + const snaked = toSnakeCase(field); + return columns.has(snaked) || snake ? snaked : field; + }; +} + +/** + * Asks the database how it names Better Auth's tables and columns, and which + * plugin columns it has. Selecting a column that is not there fails the whole + * query, so nothing is assumed. + */ +export async function detectSchema(client: DbClient): Promise { + for (const [userTable, accountTable] of TABLE_CANDIDATES) { + const columns = await tableColumns(client, userTable); + // A table without Better Auth's verified column is some other `user` + // table, so the plural one is tried next. + if (!columns.has("emailVerified") && !columns.has("email_verified")) continue; + const snake = !columns.has("emailVerified"); + const column = resolveColumn(columns, snake); + const plugins = new Set(PLUGIN_COLUMNS.filter((field) => columns.has(column(field)))); + // No account table: keep the user table's casing, and let the query's + // "no such table" error say what is missing. + const accountColumns = await tableColumns(client, accountTable); + const accountSnake = + accountColumns.size === 0 + ? snake + : !accountColumns.has("userId") && accountColumns.has("user_id"); + const accountColumn = resolveColumn(accountColumns, accountSnake); + return { userTable, accountTable, column, accountColumn, plugins }; + } + // No table found: the query names the default, and its "no such table" + // error carries the hint. + return { + userTable: "user", + accountTable: "account", + column: (field) => field, + accountColumn: (field) => field, + plugins: new Set(), + }; +} + +/** + * Builds the SELECT, including only the plugin columns that exist. Each column + * comes back under its camelCase name, however the database spells it. + * + * @param schema - From {@link detectSchema}. + */ +export function buildBetterAuthQuery(client: DbClient, schema: BetterAuthSchema): string { + const q = (identifier: string) => client.quote(identifier); + const { column, accountColumn } = schema; + const select = (field: string) => + column(field) === field ? `u.${q(field)}` : `u.${q(column(field))} AS ${q(field)}`; + const selected = [ + ...CORE_COLUMNS.map(select), + ...PLUGIN_COLUMNS.filter((field) => schema.plugins.has(field)).map(select), + ]; + + // LEFT JOIN, not INNER: a user who only ever signed in with OAuth has no + // credential account, and dropping them would silently shrink the export. + return ( + `SELECT ${selected.join(", ")}, a.${q("password")} AS ${q("password_hash")} ` + + `FROM ${q(schema.userTable)} u ` + + `LEFT JOIN ${q(schema.accountTable)} a ON a.${q(accountColumn("userId"))} = u.${q("id")} ` + + `AND a.${q(accountColumn("providerId"))} = 'credential' ` + + `ORDER BY u.${q("id")} ASC` + ); +} + +type BetterAuthRow = Record & { id?: unknown }; + +/** Renames the schema's camelCase onto what the betterauth transformer reads. */ +const FIELD_ALIASES: Record = { + id: "user_id", + emailVerified: "email_verified", + phoneNumber: "phone_number", + phoneNumberVerified: "phone_number_verified", + displayUsername: "display_username", + createdAt: "created_at", + updatedAt: "updated_at", +}; + +export function buildBetterAuthExport( + rows: BetterAuthRow[], + record: (line: UserLine) => void = () => {}, +) { + const users: Record[] = []; + const counts = { email: 0, emailVerified: 0, password: 0, name: 0, username: 0, phone: 0 }; + + // The join returns a row per credential account, and nothing in Better + // Auth's schema stops a user having two. Rows that agree are one user. Rows + // with different hashes cannot be settled here, so that user is skipped + // rather than given whichever hash came first. + const byId = new Map }>(); + for (const row of rows) { + const userId = String(row.id ?? ""); + const entry = byId.get(userId) ?? { row, accounts: 0, hashes: new Set() }; + entry.accounts++; + if (row.password_hash !== null && row.password_hash !== undefined) { + entry.hashes.add(row.password_hash); + entry.row = row; + } + byId.set(userId, entry); + } + + let conflicts = 0; + for (const [userId, { row, accounts, hashes }] of byId) { + if (hashes.size > 1) { + conflicts++; + record({ + sourceId: userId, + status: "skipped", + error: `has ${accounts} credential accounts with different password hashes; keep one in Better Auth and export again`, + }); + continue; + } + + const user: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + user[FIELD_ALIASES[key] ?? key] = value instanceof Date ? value.toISOString() : value; + } + + if (row.email) counts.email++; + if (row.emailVerified) counts.emailVerified++; + if (row.password_hash) counts.password++; + if (row.name) counts.name++; + if (row.username) counts.username++; + if (row.phoneNumber) counts.phone++; + + users.push(user); + record({ sourceId: userId, status: "exported" }); + } + + if (conflicts > 0) { + log.warn( + `Skipped ${conflicts} user${conflicts === 1 ? "" : "s"} with more than one credential account and ` + + "different password hashes. Keep one account each in Better Auth, then export again.", + ); + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.emailVerified }, + { label: "have a password hash", count: counts.password }, + { label: "have a name", count: counts.name }, + { label: "have a username", count: counts.username }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +const BETTERAUTH_DB = { + platform: "betterauth", + envVar: "BETTERAUTH_DB_URL", + prompt: "Better Auth database connection string", + hint: "Postgres, MySQL, libsql://… or a SQLite file — whichever your Better Auth install uses.", +} as const satisfies ResolveConfig; + +export async function exportBetterAuth(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, BETTERAUTH_DB); + + await withGutter("Exporting users from Better Auth", async () => { + if (!options.json) printTarget({ platform: "betterauth" }); + const { + value: { rows, plugins }, + } = await withInputRetry( + dbUrl, + async () => promptDbUrl(BETTERAUTH_DB), + async (connectionString) => + withSpinner("Reading the user table...", async () => + withDbClient(connectionString, "betterauth", async (client) => { + const schema = await detectSchema(client); + const rows = await client.query(buildBetterAuthQuery(client, schema)); + return { rows, plugins: schema.plugins }; + }), + ), + options, + ); + + log.info( + plugins.size > 0 + ? `Detected plugin columns: ${[...plugins].join(", ")}.` + : "No plugin columns detected; exporting the core user fields.", + ); + + const run = await startExportRun(options, { platform: "betterauth" }); + const { users, coverage } = buildBetterAuthExport(rows, run.append); + finishExport({ run, options, users, coverage }); + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts new file mode 100644 index 000000000..1a3105ecd --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.test.ts @@ -0,0 +1,282 @@ +import { beforeEach, describe, expect, mock, test } from "bun:test"; +import { CliError, ERROR_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +const mockDescribeBapiTarget = mock(); +const mockResolveBapiSecretKey = mock(); +mock.module("../../../lib/bapi-command.ts", () => ({ + describeBapiTarget: (...args: unknown[]) => mockDescribeBapiTarget(...args), + resolveBapiSecretKey: (...args: unknown[]) => mockResolveBapiSecretKey(...args), +})); + +const mockResolveProfile = mock(); +mock.module("../../../lib/config.ts", () => ({ + resolveProfile: (...args: unknown[]) => mockResolveProfile(...args), +})); + +const mockFetchApps = mock(); +mock.module("../../../lib/app-picker.ts", () => ({ + fetchAppsTolerantly: (...args: unknown[]) => mockFetchApps(...args), +})); + +const mockSearch = mock(); +mock.module("../../../lib/listage.ts", () => ({ + search: (...args: unknown[]) => mockSearch(...args), +})); + +const mockResolveUsersInstanceContext = mock(); +mock.module("../../users/interactive/instance-context.ts", () => ({ + resolveUsersInstanceContext: (...args: unknown[]) => mockResolveUsersInstanceContext(...args), +})); + +let human = true; +mock.module("../../../mode.ts", () => ({ + isHuman: () => human, + isAgent: () => !human, + getMode: () => (human ? "human" : "agent"), + setMode: () => {}, +})); + +const { resolveClerkSource } = await import("./clerk-source.ts"); +const { setAssumeYes } = await import("../lib/assume-yes.ts"); + +const captured = useCaptureLog(); + +beforeEach(() => { + human = true; + setAssumeYes(false); + mockDescribeBapiTarget.mockReset(); + mockResolveBapiSecretKey.mockReset(); + mockResolveProfile.mockReset(); + mockResolveProfile.mockResolvedValue(undefined); + mockResolveUsersInstanceContext.mockReset(); + mockFetchApps.mockReset(); + mockSearch.mockReset(); + delete process.env.CLERK_SECRET_KEY; +}); + +/** The linked-project case: something resolved, nobody asked for it. */ +function stubResolved(target: string | undefined, secretKey = "sk_test_resolved") { + mockDescribeBapiTarget.mockResolvedValue(target); + mockResolveBapiSecretKey.mockResolvedValue(secretKey); +} + +describe("resolveClerkSource", () => { + test("--secret-key names the instance outright and is never questioned", async () => { + stubResolved(undefined, "sk_test_explicit"); + + const source = await resolveClerkSource({ secretKey: "sk_test_explicit" }); + + expect(source).toEqual({ secretKey: "sk_test_explicit", target: undefined }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // An export has to be scriptable outside agent mode. Before this, `chosen` + // keyed off `--secret-key` alone, so every other way of naming an instance + // still stopped at a picker no flag could answer. + test.each([ + ["--app", { app: "app_1" }], + ["--instance", { instance: "prod" }], + ["--app and --instance", { app: "app_1", instance: "prod" }], + ])("%s names the instance, so nothing is asked", async (_label, options) => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource(options); + + expect(source).toEqual({ secretKey: "sk_test_resolved", target: "my-app (production)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // `resolveBapiSecretKey` puts an exported key above the linked profile for + // every other command in this family. Opening a picker here — and then + // overriding the key with whatever it returned — made this the one command + // where exporting CLERK_SECRET_KEY did less than not exporting it. + test("an exported CLERK_SECRET_KEY is not second-guessed in a linked directory", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_from_env"; + stubResolved("my-app (development)", "sk_test_from_env"); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_from_env", target: "my-app (development)" }); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // Exporting the instance that is about to be imported *into* is the failure + // this whole module exists to prevent, so a resolved instance is offered as + // one choice among the account's applications rather than taken silently. + test("offers every instance, flat, with the linked application's first", async () => { + stubResolved("my-app (development)"); + mockResolveProfile.mockResolvedValue({ profile: { appId: "app_2" } }); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [ + { instance_id: "ins_1d", environment_type: "development" }, + { instance_id: "ins_1p", environment_type: "production" }, + ], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_live_other", + appLabel: "my-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_live_other", target: "my-app (production)" }); + // One row per instance, not per application: dev and prod are different + // user pools, and exporting the wrong one is silent. + const { message, source: listSource } = mockSearch.mock.calls[0]![0]; + expect(message).toBe("What Clerk instance do you want to export users from?"); + expect(listSource("")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + { + name: "my-app - Development instance (ins_1d)", + value: { app: "app_1", instance: "ins_1d" }, + }, + { + name: "my-app - Production instance (ins_1p)", + value: { app: "app_1", instance: "ins_1p" }, + }, + ]); + // Both halves are handed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + expect(mockResolveUsersInstanceContext).toHaveBeenCalledWith({ + app: "app_1", + instance: "ins_1p", + }); + expect(captured.err).toBe(""); + }); + + // The list is searched by its rendered label, so an application id typed from + // a dashboard URL still finds its instances. + test("filters on the rendered label", async () => { + stubResolved("my-app (development)"); + mockFetchApps.mockResolvedValue([ + { + application_id: "app_1", + name: "my-app", + instances: [{ instance_id: "ins_1p", environment_type: "production" }], + }, + { + application_id: "app_2", + name: "other-app", + instances: [{ instance_id: "ins_2d", environment_type: "development" }], + }, + ]); + mockSearch.mockResolvedValue({ app: "app_1", instance: "ins_1p" }); + mockResolveUsersInstanceContext.mockResolvedValue({ secretKey: "sk_live_other" }); + + await resolveClerkSource({}); + + const { source: listSource } = mockSearch.mock.calls[0]![0]; + expect(listSource("ins_2d")).toEqual([ + { + name: "other-app - Development instance (ins_2d)", + value: { app: "app_2", instance: "ins_2d" }, + }, + ]); + expect(listSource("production")).toHaveLength(1); + }); + + // An empty list is not a picker. PLAPI being degraded looks the same as an + // account with no applications, and neither one has an instance to offer. + test("no instances to offer falls back to the flags", async () => { + stubResolved("my-app (development)"); + // An application with no instances is not an offer either. + mockFetchApps.mockResolvedValue([{ application_id: "app_1", name: "my-app", instances: [] }]); + + await expect(resolveClerkSource({})).rejects.toBeInstanceOf(UserAbortError); + + expect(mockSearch).not.toHaveBeenCalled(); + expect(captured.err).toContain("--secret-key"); + expect(captured.err).toContain("--app"); + expect(captured.err).toContain("clerk link"); + }); + + test("agent mode takes the resolved instance without prompting", async () => { + human = false; + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + // `--json` never prompts, even at a terminal: it takes what an agent would. + // `-y` is "do not prompt": it takes the resolved instance as an agent does. + test("-y takes the resolved instance without the picker", async () => { + stubResolved("my-app (production)"); + setAssumeYes(true); + + const source = await resolveClerkSource({}); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("-y in an unlinked directory fails rather than opening the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + setAssumeYes(true); + + await expect(resolveClerkSource({})).rejects.toThrow(failure); + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); + + test("--json takes the resolved instance without prompting", async () => { + stubResolved("my-app (production)"); + + const source = await resolveClerkSource({ json: true }); + + expect(source.secretKey).toBe("sk_test_resolved"); + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("--json in an unlinked directory fails rather than opening the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ json: true })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); + + test("an unlinked directory picks an application instead of failing", async () => { + mockDescribeBapiTarget.mockRejectedValue( + new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }), + ); + mockResolveUsersInstanceContext.mockResolvedValue({ + secretKey: "sk_test_picked", + appLabel: "other-app", + instanceLabel: "production", + }); + + const source = await resolveClerkSource({}); + + expect(source).toEqual({ secretKey: "sk_test_picked", target: "other-app (production)" }); + // The picker just asked which application; asking again is noise. + expect(mockSearch).not.toHaveBeenCalled(); + }); + + test("an explicit --app that fails to resolve surfaces the error, not the picker", async () => { + const failure = new CliError("No secret key found.", { code: ERROR_CODE.NO_SECRET_KEY }); + mockDescribeBapiTarget.mockRejectedValue(failure); + + await expect(resolveClerkSource({ app: "app_123" })).rejects.toThrow(failure); + + expect(mockResolveUsersInstanceContext).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk-source.ts b/packages/cli-core/src/commands/migrate/export/clerk-source.ts new file mode 100644 index 000000000..897c44170 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk-source.ts @@ -0,0 +1,194 @@ +/** + * Which Clerk instance `migrate export clerk` reads *from*. + * + * Every other resolver in the CLI answers "where do I operate?" with the linked + * project, silently. For an export that default is actively dangerous: the + * linked instance is normally the migration's *destination*, so taking it + * without asking is how a run ends up exporting an instance and importing it + * straight back into itself. + * + * So the source is resolved in three tiers: + * + * 1. The user named the instance — `--secret-key`, `--app`, `--instance`, or + * an exported `CLERK_SECRET_KEY` — and it runs unquestioned. An exported + * key outranks the linked profile everywhere else in this CLI + * (`resolveBapiSecretKey`), and this command is not the one place that + * should differ. + * 2. Anything the CLI resolved on the user's behalf (the linked project, a + * keyless app) is not taken silently: every instance on the account is + * offered, with the resolved application's instances first so "yes, that + * one" is still a single Enter. + * 3. Nothing to resolve at all — no link, no key, no flags — falls back to the + * application picker `users list` uses, rather than failing on an unlinked + * directory. That one is `clerk link`'s, so it also offers "create a new + * application"; an empty application is never the right source, but the + * alternative here is an error, not a better list. + */ + +import { fetchAppsTolerantly } from "../../../lib/app-picker.ts"; +import { describeBapiTarget, resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { resolveProfile } from "../../../lib/config.ts"; +import { CliError, ERROR_CODE, throwUserAbort } from "../../../lib/errors.ts"; +import { search } from "../../../lib/listage.ts"; +import type { ApplicationInstance } from "../../../lib/plapi.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; +import { resolveUsersInstanceContext } from "../../users/interactive/instance-context.ts"; + +/** e.g. `Development instance`. Unknown environment types print as-is. */ +function instanceLabel(instance: ApplicationInstance): string { + const type = instance.environment_type; + if (!type) return "instance"; + return `${type.charAt(0).toUpperCase()}${type.slice(1)} instance`; +} + +export type ResolveClerkSourceOptions = { + secretKey?: string; + app?: string; + instance?: string; + cwd?: string; + /** `--json` never prompts. */ + json?: boolean; +}; + +export type ClerkExportSource = { + secretKey: string; + /** + * Human-readable target, e.g. `my-app (production)`. Absent when + * `--secret-key` (or `CLERK_SECRET_KEY`) named the instance directly, since + * a bare key carries no application context to describe. + */ + target?: string; +}; + +/** {@link ClerkExportSource} plus whether the user already chose it out loud. */ +type ResolvedSource = ClerkExportSource & { chosen: boolean }; + +async function resolveSource(options: ResolveClerkSourceOptions): Promise { + // What separates the two tiers is not "did the CLI have to look anything up" + // but "did the user say which instance". A flag or an exported key is a + // sentence they typed for this run; the linked project is a choice they made + // for some other purpose, possibly months ago, and normally names the + // migration's destination rather than its source. Only the second is worth + // asking about. + // + // `--instance` counts on its own: paired with the linked application it + // names one instance, and pairing it with `--app` addresses any instance on + // the account. Without this an export could not be scripted at all outside + // agent mode — every run would stop at a picker no flag could answer. + const named = + Boolean(options.secretKey) || + Boolean(options.app) || + Boolean(options.instance) || + Boolean(process.env.CLERK_SECRET_KEY); + + try { + return { + target: await describeBapiTarget(options), + secretKey: await resolveBapiSecretKey(options), + chosen: named, + }; + } catch (error) { + if ( + options.json || + isAssumeYes() || + !isHuman() || + named || + !(error instanceof CliError) || + error.code !== ERROR_CODE.NO_SECRET_KEY + ) { + throw error; + } + + const ctx = await resolveUsersInstanceContext({}); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + // The picker just asked. Confirming the answer to a question the user + // answered one prompt ago is noise. + chosen: true, + }; + } +} + +/** + * Offers every instance on the account, flat — one row per instance rather than + * an application picker followed by an instance picker. + * + * An application is not what an export reads from; an instance is. Picking + * "Migration Test" and then "development" is two questions with one answer, and + * it hides the thing that actually matters — dev and prod are different user + * pools, and exporting the wrong one is silent. + * + * Deliberately not `pickOrCreateApp`: its "+ Create a new application" choice + * makes sense when you are choosing somewhere to *write*, and no sense at all + * as an export source — a brand-new application has no users in it. + * + * @param currentAppId the application the CLI resolved on the user's behalf. + * Its instances lead the list, because they are the likeliest answer. + * @returns undefined when there is nothing to offer, so the caller can fall + * back to telling the user which flags to pass instead of showing an empty + * list. `fetchAppsTolerantly` returns empty on a degraded PLAPI, not just on + * an account with no applications. + */ +async function pickInstance(currentAppId?: string): Promise { + const apps = await fetchAppsTolerantly(); + + const ordered = currentAppId + ? [ + ...apps.filter((app) => app.application_id === currentAppId), + ...apps.filter((app) => app.application_id !== currentAppId), + ] + : apps; + + const choices = ordered.flatMap((app) => + (app.instances ?? []).map((instance) => ({ + name: `${app.name || app.application_id} - ${instanceLabel(instance)} (${instance.instance_id})`, + value: { app: app.application_id, instance: instance.instance_id }, + })), + ); + if (choices.length === 0) return undefined; + + const picked = await search<{ app: string; instance: string }>({ + message: "What Clerk instance do you want to export users from?", + source: (term) => + term + ? choices.filter((choice) => choice.name.toLowerCase().includes(term.toLowerCase())) + : choices, + }); + + // Both halves are passed on, so the secret-key lookup runs against exactly + // the instance that was chosen and nothing prompts a second time. + const ctx = await resolveUsersInstanceContext(picked); + return { + secretKey: ctx.secretKey, + target: ctx.appLabel ? `${ctx.appLabel} (${ctx.instanceLabel})` : undefined, + }; +} + +/** The application the CLI resolved on the user's behalf, if it knows one. */ +async function currentAppId(options: ResolveClerkSourceOptions): Promise { + if (options.app) return options.app; + const resolved = await resolveProfile(options.cwd ?? process.cwd()).catch(() => undefined); + return resolved?.profile.appId; +} + +export async function resolveClerkSource( + options: ResolveClerkSourceOptions, +): Promise { + const { chosen, ...source } = await resolveSource(options); + // `-y` takes the resolved instance without the picker, as an agent does. + if (chosen || !source.target || options.json || isAssumeYes() || !isHuman()) return source; + + const picked = await pickInstance(await currentAppId(options)); + if (picked) return picked; + + log.info( + "Export from a different instance with one of:\n" + + " `--secret-key ` — the source instance's secret key\n" + + " `--app --instance ` — another application on your account\n" + + " `clerk link` — link this directory to a different application first", + ); + throwUserAbort(); +} diff --git a/packages/cli-core/src/commands/migrate/export/clerk.test.ts b/packages/cli-core/src/commands/migrate/export/clerk.test.ts new file mode 100644 index 000000000..868d912e9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.test.ts @@ -0,0 +1,415 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { + buildClerkExport, + exportClerk, + fetchAllClerkUsers, + mapClerkUserToExport, +} from "./clerk.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expclerk-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Answers `GET /v1/users` from `pages`, one page per call. */ +function stubPages(pages: unknown[][]) { + let call = 0; + globalThis.fetch = (async (input: string | URL | Request) => { + // The export names the instance it reads before paging through it. + if (new URL(input.toString()).pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); + } + requests.push(input.toString()); + return Response.json(pages[call++] ?? []); + }) as unknown as typeof fetch; +} + +const user = (overrides: Record = {}) => ({ + id: "user_1", + primary_email_address_id: "idn_1", + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + ], + phone_numbers: [], + ...overrides, +}); + +describe("mapClerkUserToExport", () => { + test("writes the field names the clerk transformer reads", () => { + expect( + mapClerkUserToExport(user({ first_name: "Ada", last_name: "L", username: "ada" })), + ).toMatchObject({ + id: "user_1", + primary_email_address: "a@x.dev", + first_name: "Ada", + last_name: "L", + username: "ada", + }); + }); + + // `migrate import` puts the first entry on POST /v1/users and attaches the rest + // afterwards, so a reordered list would change which address signs the user in. + test("keeps the primary identifier out of the additional list", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("separates unverified identifiers", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "c@x.dev", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.unverified_email_addresses).toEqual(["c@x.dev"]); + expect(mapped.verified_email_addresses).toBeUndefined(); + }); + + // Verify-at-sign-up off lets a primary stay unverified. Exported as primary, + // the import would create it verified. + test("keeps an unverified primary unverified, and promotes a verified one", () => { + const mapped = mapClerkUserToExport( + user({ + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "unverified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("b@x.dev"); + expect(mapped.unverified_email_addresses).toEqual(["a@x.dev"]); + }); + + // The import puts the old Clerk ID in external_id, so the app's own value + // would otherwise be lost. + test("moves external_id into private metadata, keeping what is there", () => { + const mapped = mapClerkUserToExport( + user({ external_id: "acct_9", private_metadata: { tier: "gold" } }), + ); + expect(mapped.private_metadata).toEqual({ tier: "gold", clerkExternalId: "acct_9" }); + }); + + test("promotes the first verified address when none is flagged primary", () => { + const mapped = mapClerkUserToExport( + user({ + primary_email_address_id: null, + email_addresses: [ + { id: "idn_1", email_address: "a@x.dev", verification: { status: "verified" } }, + { id: "idn_2", email_address: "b@x.dev", verification: { status: "verified" } }, + ], + }), + ); + expect(mapped.primary_email_address).toBe("a@x.dev"); + expect(mapped.verified_email_addresses).toEqual(["b@x.dev"]); + }); + + test("maps phone numbers the same way", () => { + const mapped = mapClerkUserToExport( + user({ + primary_phone_number_id: "pn_1", + phone_numbers: [ + { id: "pn_1", phone_number: "+15555550100", verification: { status: "verified" } }, + { id: "pn_2", phone_number: "+15555550101", verification: { status: "unverified" } }, + ], + }), + ); + expect(mapped.primary_phone_number).toBe("+15555550100"); + expect(mapped.unverified_phone_numbers).toEqual(["+15555550101"]); + }); + + test("converts BAPI's Unix-millisecond timestamps to RFC3339", () => { + const mapped = mapClerkUserToExport(user({ created_at: 1704067200000 })); + expect(mapped.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits empty metadata rather than writing empty objects", () => { + const mapped = mapClerkUserToExport( + user({ public_metadata: {}, private_metadata: { plan: "pro" } }), + ); + expect("public_metadata" in mapped).toBe(false); + expect(mapped.private_metadata).toEqual({ plan: "pro" }); + }); + + test("carries the account-state fields the import accepts", () => { + const mapped = mapClerkUserToExport( + user({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }), + ); + expect(mapped).toMatchObject({ + banned: true, + create_organization_enabled: false, + create_organizations_limit: 3, + delete_self_enabled: true, + }); + }); +}); + +describe("fetchAllClerkUsers", () => { + test("pages until a short page arrives", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + Array.from({ length: 12 }, (_, i) => user({ id: `v${i}` })), + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(512); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("offset=480"); + }); + + // A user deleted after the first page shifts every later row back one. The + // overlap re-reads the boundary, so the row that slid onto it isn't lost. + test("keeps the user a mid-export deletion slides onto a page boundary", async () => { + const users = Array.from({ length: 600 }, (_, i) => user({ id: `u${i}` })); + let pages = 0; + // A server that answers by offset, and loses u10 after the first page. + globalThis.fetch = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_src", environment_type: "production" }); + } + const offset = Number(url.searchParams.get("offset")); + const rows = pages++ === 0 ? users : users.filter((entry) => entry.id !== "u10"); + return Response.json(rows.slice(offset, offset + 500)); + }) as unknown as typeof fetch; + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all.map((entry) => entry.id)).toContain("u500"); + expect(all).toHaveLength(600); + }); + + // Oldest first, so a sign-up mid-export lands at the end; a repeat the + // overlap reads again is dropped. + test("pages oldest first and drops a user a shifted page repeats", async () => { + stubPages([ + Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), + [user({ id: "u499" }), user({ id: "v0" })], + ]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(501); + expect(requests[0]).toContain("order_by=%2Bcreated_at"); + }); + + // A full final page must still trigger one more request, or an instance whose + // size is an exact multiple of the page size would look short by one page. + test("makes one more request when the last page is exactly full", async () => { + stubPages([Array.from({ length: 500 }, (_, i) => user({ id: `u${i}` })), []]); + + const all = await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + + expect(all).toHaveLength(500); + expect(requests).toHaveLength(2); + }); + + test("asks for BAPI's maximum page size", async () => { + stubPages([[]]); + await fetchAllClerkUsers({ secretKey: "sk_test_x" }); + expect(requests[0]).toContain("limit=500"); + }); + + test("copes with an instance that has no users", async () => { + stubPages([[]]); + expect(await fetchAllClerkUsers({ secretKey: "sk_test_x" })).toEqual([]); + }); +}); + +describe("buildClerkExport", () => { + test("counts coverage per field", () => { + const { coverage } = buildClerkExport([ + user({ id: "u1", first_name: "Ada", password_enabled: true }), + user({ id: "u2" }), + ]); + + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + expect(byLabel["have a password (not exportable — see below)"]).toBe(1); + }); + + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildClerkExport([user({ id: "u1" }), user({ id: "u2" })], (line) => lines.push(line)); + + expect(lines).toEqual([ + { sourceId: "u1", status: "exported" }, + { sourceId: "u2", status: "exported" }, + ]); + }); +}); + +/** The envelope the one export run in this project wrote. */ +function onlyExportFile(): string { + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); + expect(entries).toHaveLength(1); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; +} + +describe("exportClerk", () => { + test("writes the default path and reports coverage", async () => { + stubPages([[user({ id: "u1", first_name: "Ada" })], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "clerk", + }); + const written = exportedUsers(); + expect(written).toHaveLength(1); + expect(written[0]?.id).toBe("u1"); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("Exported 1 user"); + }); + + test("names the command that consumes the file", async () => { + stubPages([[user()], []]); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); + }); + + // The key decides the instance; --instance naming a different one would + // label a read of one user pool with another's name. + test("refuses an --instance the key does not address, before reading any user", async () => { + stubPages([[user()], []]); + + await expect(exportClerk({ secretKey: "sk_test_x", instance: "dev" })).rejects.toThrow( + /--instance dev does not match the key from --secret-key/, + ); + expect(requests).toEqual([]); + }); + + test("names the instance it reads from first, and records it on the run", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(Bun.stripANSI(captured.err)).toContain("Source: Clerk, production instance ins_src"); + expect( + JSON.parse(fs.readFileSync(onlyExportFile().replace("export.json", "run.json"), "utf-8")), + ).toMatchObject({ + target: { + platform: "clerk", + instanceId: "ins_src", + keySource: "the instance behind --secret-key", + }, + }); + }); + + test("--output controls the destination, relative to the working directory", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x", output: "somewhere/mine.json" }); + + expect(fs.existsSync(path.join(workDir, "somewhere", "mine.json"))).toBe(true); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + // Silence here would be the worst outcome: the operator finds out when + // nobody can sign in to the destination instance. + test("says plainly that passwords are not in the file", async () => { + stubPages([[user({ password_enabled: true })], []]); + await exportClerk({ secretKey: "sk_test_x" }); + expect(captured.err).toContain("never returns password digests"); + }); + + test("writes an empty file and says so when the instance has no users", async () => { + stubPages([[]]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("No users found to export"); + expect(exportedUsers()).toEqual([]); + }); + + test("an empty export warns but does not suggest importing it", async () => { + stubPages([[]]); + const originalMode = getMode(); + setMode("human"); + try { + await exportClerk({ secretKey: "sk_test_x", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + + expect(captured.err).toContain("No users found to export"); + expect(captured.err).not.toContain("Next steps"); + expect(captured.err).not.toContain("Import them with"); + }); + + test("agent mode suppresses the Next steps block", async () => { + stubPages([[user()], []]); + + await exportClerk({ secretKey: "sk_test_x" }); + + expect(captured.err).toContain("Exported 1 user"); + expect(captured.err).not.toContain("Next steps"); + // The import command still prints, where an agent can read it. + expect(captured.err).toContain("Import them with"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/clerk.ts b/packages/cli-core/src/commands/migrate/export/clerk.ts new file mode 100644 index 000000000..1139c3284 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/clerk.ts @@ -0,0 +1,305 @@ +/** + * `clerk migrate export clerk` — pull users out of a Clerk instance. + * + * Ported from the standalone migration-tool's `src/export/clerk.ts`, rewritten + * onto `bapiRequest` instead of `@clerk/backend` so it shares the CLI's auth + * resolution, `--verbose` request tracing and error taxonomy. + * + * The output feeds `clerk migrate import --transformer clerk` unedited, which is + * what makes development → production a two-command operation. + * + * **Passwords do not come out of this endpoint.** Clerk never returns password + * digests, TOTP secrets or backup codes over the API; only the `*_enabled` + * booleans. The coverage report says how many users *have* a password so the + * gap is visible before the import, not after. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { retryOn429 } from "../lib/retry.ts"; +import { assertInstanceFlagMatches, fetchInstanceIdentity, printTarget } from "../lib/target.ts"; +import { resolveClerkSource } from "./clerk-source.ts"; +import { finishExport, startExportRun } from "./shared.ts"; + +/** BAPI's maximum page size for `GET /v1/users`. */ +const PAGE_SIZE = 500; + +/** + * How many rows each page re-reads from the end of the one before. A user + * deleted mid-export shifts every later row back one, so without it the row + * at each page boundary would be skipped; the Map drops the repeats. + */ +const PAGE_OVERLAP = 20; + +export type ExportClerkOptions = { + output?: string; + secretKey?: string; + app?: string; + instance?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +type BapiIdentifier = { + email_address?: string; + phone_number?: string; + verification?: { status?: string } | null; +}; + +type BapiUser = { + id: string; + external_id?: string | null; + username?: string | null; + first_name?: string | null; + last_name?: string | null; + email_addresses?: BapiIdentifier[]; + phone_numbers?: BapiIdentifier[]; + primary_email_address_id?: string | null; + primary_phone_number_id?: string | null; + public_metadata?: Record; + private_metadata?: Record; + unsafe_metadata?: Record; + password_enabled?: boolean; + totp_enabled?: boolean; + banned?: boolean; + create_organization_enabled?: boolean; + create_organizations_limit?: number | null; + delete_self_enabled?: boolean; + created_at?: number; + legal_accepted_at?: number | null; +}; + +type IdentifierWithId = BapiIdentifier & { id?: string }; + +/** + * Splits identifiers into verified and unverified, primary first. + * + * The primary has to lead: `migrate import` puts the first entry on + * `POST /v1/users` and attaches the rest afterwards, so a reordered list would + * silently change which address the user signs in with. + */ +function splitIdentifiers( + entries: IdentifierWithId[] | undefined, + primaryId: string | null | undefined, + read: (entry: BapiIdentifier) => string | undefined, +): { primary?: string; verified: string[]; unverified: string[] } { + const verified: string[] = []; + const unverified: string[] = []; + let primary: string | undefined; + + for (const entry of entries ?? []) { + const value = read(entry); + if (!value) continue; + + const isVerified = entry.verification?.status === "verified"; + // An unverified primary stays unverified: the import puts a primary on + // POST /v1/users, which creates it verified. + if (entry.id && entry.id === primaryId && isVerified) { + primary = value; + continue; + } + if (isVerified) verified.push(value); + else unverified.push(value); + } + + // No verified primary: promote the first verified one so the export still + // has an identifier the import can lead with. + if (!primary && verified.length > 0) primary = verified.shift(); + + return { primary, verified, unverified }; +} + +/** Maps a BAPI user onto the shape the `clerk` transformer reads. */ +export function mapClerkUserToExport(user: BapiUser): Record { + const exported: Record = { id: user.id }; + + const emails = splitIdentifiers( + user.email_addresses, + user.primary_email_address_id, + (entry) => entry.email_address, + ); + if (emails.primary) exported.primary_email_address = emails.primary; + if (emails.verified.length > 0) exported.verified_email_addresses = emails.verified; + if (emails.unverified.length > 0) exported.unverified_email_addresses = emails.unverified; + + const phones = splitIdentifiers( + user.phone_numbers, + user.primary_phone_number_id, + (entry) => entry.phone_number, + ); + if (phones.primary) exported.primary_phone_number = phones.primary; + if (phones.verified.length > 0) exported.verified_phone_numbers = phones.verified; + if (phones.unverified.length > 0) exported.unverified_phone_numbers = phones.unverified; + + if (user.username) exported.username = user.username; + if (user.first_name) exported.first_name = user.first_name; + if (user.last_name) exported.last_name = user.last_name; + + for (const [source, target] of [ + ["public_metadata", "public_metadata"], + ["private_metadata", "private_metadata"], + ["unsafe_metadata", "unsafe_metadata"], + ] as const) { + const value = user[source]; + if (value && Object.keys(value).length > 0) exported[target] = value; + } + + // The import sets external_id to the old Clerk ID, so an app's own + // external_id moves to private metadata rather than being lost. + if (user.external_id) { + exported.private_metadata = { + ...(exported.private_metadata as Record | undefined), + clerkExternalId: user.external_id, + }; + } + + if (user.banned) exported.banned = true; + if (user.create_organization_enabled !== undefined) { + exported.create_organization_enabled = user.create_organization_enabled; + } + if (user.create_organizations_limit !== null && user.create_organizations_limit !== undefined) { + exported.create_organizations_limit = user.create_organizations_limit; + } + if (user.delete_self_enabled !== undefined) { + exported.delete_self_enabled = user.delete_self_enabled; + } + + // BAPI reports timestamps as Unix milliseconds; the schema wants RFC3339. + if (user.created_at) exported.created_at = new Date(user.created_at).toISOString(); + if (user.legal_accepted_at) { + exported.legal_accepted_at = new Date(user.legal_accepted_at).toISOString(); + } + + return exported; +} + +/** Pages through every user in the instance. */ +export async function fetchAllClerkUsers(options: { + secretKey: string; + spinner?: SpinnerControls; +}): Promise { + const all = new Map(); + + // Oldest first, so a sign-up during the export lands at the end instead of + // shifting every later page by one (BAPI's default is newest first). A + // deletion shifts later rows back, which would skip the row at the next + // page boundary, so each page overlaps the last. + // ponytail: offset paging, safe for up to PAGE_OVERLAP deletions per page + // mid-export; /v1/users has no cursor to page by instead. + for (let offset = 0; ; offset += PAGE_SIZE - PAGE_OVERLAP) { + const response = await retryOn429(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?limit=${PAGE_SIZE}&offset=${offset}&order_by=%2Bcreated_at`, + secretKey: options.secretKey, + }), + ); + + const page = Array.isArray(response.body) ? (response.body as BapiUser[]) : []; + for (const user of page) all.set(user.id, user); + options.spinner?.update(`Fetching users from Clerk: ${all.size} so far...`); + + // A short page means the end; anything else would loop forever on an + // instance whose size happens to be a multiple of the page size. + if (page.length < PAGE_SIZE) break; + } + + return [...all.values()]; +} + +export type ClerkExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +/** Maps every user and counts what the export actually contains. */ +export function buildClerkExport( + users: BapiUser[], + record: (line: UserLine) => void = () => {}, +): ClerkExportResult { + const exported: Record[] = []; + const counts = { email: 0, username: 0, firstName: 0, lastName: 0, phone: 0, password: 0 }; + + for (const user of users) { + try { + const mapped = mapClerkUserToExport(user); + exported.push(mapped); + + if (mapped.primary_email_address) counts.email++; + if (mapped.username) counts.username++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.primary_phone_number) counts.phone++; + if (user.password_enabled) counts.password++; + + record({ sourceId: user.id, status: "exported" }); + } catch (error) { + record({ sourceId: user.id, status: "skipped", error: (error as Error).message }); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a phone number", count: counts.phone }, + { label: "have a username", count: counts.username }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have a password (not exportable — see below)", count: counts.password }, + ], + }; +} + +export async function exportClerk(options: ExportClerkOptions): Promise { + // Resolved before the gutter opens, the way `export auth0` resolves its + // credentials: confirming the source is a question about whether to run at + // all, not a step of the run. + const source = await resolveClerkSource({ + secretKey: options.secretKey, + app: options.app, + instance: options.instance, + json: options.json, + }); + + await withGutter("Exporting users from Clerk", async () => { + const identity = await fetchInstanceIdentity(source.secretKey); + // A key picks the instance on its own and ignores --instance, so the two + // must agree, as on import: otherwise this reads the wrong user pool under + // the requested instance's name. + const keyFrom = options.secretKey + ? "--secret-key" + : !options.app && process.env.CLERK_SECRET_KEY + ? "CLERK_SECRET_KEY env var" + : undefined; + if (keyFrom) assertInstanceFlagMatches({ instance: options.instance }, keyFrom, identity); + // describeBapiTarget already names where the key came from, env key included. + const keySource = source.target ?? keyFrom; + const target = { + platform: "clerk", + env: identity.env, + instanceId: identity.instanceId, + ...(keySource ? { keySource } : {}), + }; + if (!options.json) printTarget(target); + + const users = await withSpinner("Fetching users from Clerk...", async (spinner) => + fetchAllClerkUsers({ secretKey: source.secretKey, spinner }), + ); + + const run = await startExportRun(options, target); + const { users: exported, coverage } = buildClerkExport(users, run.append); + finishExport({ run, options, users: exported, coverage }); + + if (exported.length > 0) { + log.warn( + "Clerk's API never returns password digests, TOTP secrets or backup codes, so they are not in this file. " + + "Users will need to reset their password in the destination instance.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/db-exports.test.ts b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts new file mode 100644 index 000000000..a386554f8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-exports.test.ts @@ -0,0 +1,601 @@ +/** + * The three database-backed exports, driven against a real SQLite database. + * + * SQLite because it is the one engine that needs no container, and it + * exercises the same client, the same query building and the same plugin + * detection path (via `PRAGMA` rather than `information_schema`). Postgres and + * MySQL are covered by the manual matrix run recorded in the ticket. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { createDbClient, type DbClient } from "../lib/db.ts"; +import { buildAuthJsExport, buildAuthJsQuery, exportAuthJs, fetchAuthJsUsers } from "./authjs.ts"; +import { + buildBetterAuthExport, + buildBetterAuthQuery, + detectSchema, + exportBetterAuth, + PLUGIN_COLUMNS, +} from "./betterauth.ts"; +import { buildSupabaseExport, exportSupabase, fetchSupabaseUsers } from "./supabase.ts"; +import { + looksLikeConnectionString, + normalizeConnectionString, + resolveDbUrl, +} from "./db-options.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-dbexp-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +/** Builds a fresh SQLite file so each test starts from a known schema. */ +function makeDb(build: (db: Database) => void): string { + const file = path.join(workDir, `db-${counter++}.sqlite`); + const db = new Database(file, { create: true }); + build(db); + db.close(); + return file; +} + +function betterAuthDb(pluginColumns: string[], rows: Record[] = []): string { + return makeDb((db) => { + const extra = pluginColumns.map((column) => `, "${column}" TEXT`).join(""); + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT${extra})`, + ); + db.run(`CREATE TABLE "account" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + for (const row of rows) { + const keys = Object.keys(row); + db.run( + `INSERT INTO "user" (${keys.map((k) => `"${k}"`).join(",")}) VALUES (${keys.map(() => "?").join(",")})`, + keys.map((k) => row[k]) as never[], + ); + } + }); +} + +async function withClient(file: string, work: (client: DbClient) => Promise): Promise { + const client = await createDbClient(file); + try { + return await work(client); + } finally { + await client.close(); + } +} + +describe("looksLikeConnectionString", () => { + test.each([ + ["postgres://u:p@h:5432/db", true], + ["mysql://u:p@h:3306/db", true], + ["./db.sqlite", true], + ["file:./db.sqlite", true], + ["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/abs/app.db", true], + ["", false], + [" ", false], + ["just some words", false], + ["postgres://", false], + ])("%p -> %p", (input, expected) => { + expect(looksLikeConnectionString(input)).toBe(expected); + }); +}); + +describe("normalizeConnectionString", () => { + test("encodes a password pasted in raw", () => { + const raw = "postgres://postgres:aB#c%92^d@db.example.supabase.co:5432/postgres"; + const normalized = normalizeConnectionString(raw); + + expect(looksLikeConnectionString(normalized)).toBe(true); + expect(decodeURIComponent(new URL(normalized).password)).toBe("aB#c%92^d"); + expect(new URL(normalized).hostname).toBe("db.example.supabase.co"); + }); + + test("encodes an unencoded @ in the password", () => { + const normalized = normalizeConnectionString("postgres://u:p@ss@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("p@ss"); + expect(new URL(normalized).hostname).toBe("host"); + }); + + // A bare `%` parses as a URL but fails the driver's decode with "URI error". + test("encodes a bare % in a password", () => { + const normalized = normalizeConnectionString("postgres://u:50%off@host:5432/db"); + + expect(decodeURIComponent(new URL(normalized).password)).toBe("50%off"); + }); + + test("leaves an already-valid string alone", () => { + const encoded = "postgres://u:p%40ss@host:5432/db"; + expect(normalizeConnectionString(encoded)).toBe(encoded); + }); + + test("leaves non-URL forms alone", () => { + expect(normalizeConnectionString(" ./db.sqlite ")).toBe("./db.sqlite"); + }); +}); + +describe("resolveDbUrl", () => { + const config = { platform: "authjs" as const, envVar: "AUTHJS_DB_URL", prompt: "url" }; + + test("prefers the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p@h/db" }, config, { + AUTHJS_DB_URL: "mysql://u:p@h/db", + }); + expect(url).toBe("postgres://u:p@h/db"); + }); + + test("falls back to the environment variable", async () => { + expect(await resolveDbUrl({}, config, { AUTHJS_DB_URL: "mysql://u:p@h/db" })).toBe( + "mysql://u:p@h/db", + ); + }); + + test("encodes a raw password passed to the flag", async () => { + const url = await resolveDbUrl({ dbUrl: "postgres://u:p#ss@host:5432/db" }, config, {}); + expect(decodeURIComponent(new URL(url).password)).toBe("p#ss"); + }); + + test("rejects a flag that is not a connection string, naming the encoding trap", async () => { + await expect(resolveDbUrl({ dbUrl: "not a url" }, config, {})).rejects.toThrow(/URL-encode it/); + }); + + test("warns and moves on when the environment variable is unusable", async () => { + // Tests run non-TTY, so it then hits the agent-mode branch. + await expect(resolveDbUrl({}, config, { AUTHJS_DB_URL: "garbage" })).rejects.toThrow( + /cannot prompt here/, + ); + expect(captured.err).toContain("AUTHJS_DB_URL is not a valid connection string"); + }); + + test("names both the flag and the variable when it cannot prompt", async () => { + await expect(resolveDbUrl({}, config, {})).rejects.toThrow(/--db-url.*AUTHJS_DB_URL/s); + }); +}); + +describe("authjs export", () => { + const authJsDb = (table: string) => + makeDb((db) => { + db.run( + `CREATE TABLE "${table}" (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, [ + "aj1", + "Jane Doe", + "jane@x.dev", + "2024-01-15", + ]); + db.run(`INSERT INTO "${table}" VALUES (?,?,?,?)`, ["aj2", "John Smith", "john@x.dev", null]); + }); + + test("quotes identifiers for the dialect", async () => { + await withClient(authJsDb("User"), async (client) => { + expect(buildAuthJsQuery(client, "User")).toContain('"User" u'); + expect(buildAuthJsQuery(client, "User")).toContain('u."emailVerified" AS "email_verified"'); + }); + }); + + // Prisma capitalizes the table, Drizzle does not, and Auth.js has no single + // schema — so the export tries rather than making the user guess. + test.each([["User"], ["user"], ["users"]])("finds the %s table", async (table) => { + const { rows } = await withClient(authJsDb(table), fetchAuthJsUsers); + expect(rows).toHaveLength(2); + }); + + // SQLite reads an unqualified "emailVerified" that matches no column as the + // string "emailVerified", which would mark every email verified. + test("reads a legacy NextAuth table's email_verified, and keeps null unverified", async () => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, email TEXT, email_verified TEXT)`, + ); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["n1", "Nv", "nv@x.dev", null]); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["n2", "V", "v@x.dev", "2024-01-15"]); + }); + + const { rows, table } = await withClient(file, fetchAuthJsUsers); + + expect(table).toBe("users"); + expect(rows.map((row) => row.email_verified)).toEqual([null, "2024-01-15"]); + }); + + // Postgres keeps a quoted "User" apart from "user"; SQLite does not, so the + // plural table stands in for the later candidate here. + test("passes over a candidate table without the columns for a later one", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "User" (id TEXT PRIMARY KEY, handle TEXT)`); + db.run( + `CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, email TEXT, "emailVerified" TEXT)`, + ); + db.run(`INSERT INTO users VALUES (?,?,?,?)`, ["u1", "U", "u@x.dev", null]); + }); + + const { rows, table } = await withClient(file, fetchAuthJsUsers); + + expect(table).toBe("users"); + expect(rows).toHaveLength(1); + }); + + test("a missing column is an error, not a literal", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "User" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" TEXT)`); + db.run(`INSERT INTO "User" VALUES (?,?,?)`, ["a", "a@x.dev", null]); + }); + + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow(/no such column/); + }); + + test("fails clearly when no candidate table exists", async () => { + const file = makeDb((db) => db.run(`CREATE TABLE unrelated (id TEXT)`)); + await expect(withClient(file, fetchAuthJsUsers)).rejects.toThrow( + /No Auth.js user table found. Tried User, user, users/, + ); + }); + + test("treats email_verified as a nullable timestamp, not a boolean", () => { + const { users } = buildAuthJsExport([ + { id: "a", email: "a@x.dev", email_verified: "2024-01-15" }, + { id: "b", email: "b@x.dev", email_verified: null }, + ]); + expect(users[0]?.email_verified).toBe("2024-01-15"); + expect("email_verified" in (users[1] ?? {})).toBe(false); + }); + + test("counts coverage", () => { + const { coverage } = buildAuthJsExport([ + { id: "a", email: "a@x.dev", name: "A", email_verified: "2024-01-01" }, + { id: "b" }, + ]); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(1); + expect(byLabel["have a verified email"]).toBe(1); + }); + + test("exports end to end and says which table it read", async () => { + await exportAuthJs({ dbUrl: authJsDb("User"), output: "authjs.json" }); + + const written = JSON.parse(fs.readFileSync(path.join(workDir, "authjs.json"), "utf-8")); + expect(written).toMatchObject({ clerkMigrate: 1, source: "authjs" }); + expect(written.users).toHaveLength(2); + expect(captured.err).toContain("Read 2 rows from"); + expect(captured.err).toContain("stores no passwords"); + expect(captured.err).toContain("Credentials provider"); + }); +}); + +describe("betterauth export", () => { + test("detects only the plugin columns that exist", async () => { + await withClient(betterAuthDb(["username", "banned"]), async (client) => { + expect([...(await detectSchema(client)).plugins].sort()).toEqual(["banned", "username"]); + }); + }); + + test("detects nothing on a core-only schema", async () => { + await withClient(betterAuthDb([]), async (client) => { + expect((await detectSchema(client)).plugins.size).toBe(0); + }); + }); + + test("detects every plugin column when all are present", async () => { + await withClient(betterAuthDb([...PLUGIN_COLUMNS]), async (client) => { + expect((await detectSchema(client)).plugins.size).toBe(PLUGIN_COLUMNS.length); + }); + }); + + // Selecting a column that is not there fails the whole query, which is why + // the columns are detected rather than assumed. + test("selects only detected columns", async () => { + await withClient(betterAuthDb(["username"]), async (client) => { + const query = buildBetterAuthQuery(client, await detectSchema(client)); + expect(query).toContain('"username"'); + expect(query).not.toContain('"twoFactorEnabled"'); + }); + }); + + test("the built query actually runs against the schema it was built for", async () => { + const file = betterAuthDb( + ["username", "role"], + [{ id: "u1", email: "a@x.dev", username: "a" }], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + expect(rows).toHaveLength(1); + }); + + // A user who only ever signed in with OAuth has no credential account; + // an INNER JOIN would drop them and silently shrink the export. + test("keeps a user with no credential account", async () => { + const file = betterAuthDb( + [], + [ + { id: "u1", email: "a@x.dev" }, + { id: "u2", email: "b@x.dev" }, + ], + ); + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + expect(rows).toHaveLength(2); + }); + + // Better Auth's Drizzle generator writes snake_case unless `camelCase: true`, + // and `usePlural: true` pluralizes the tables. + test.each([ + ["snake_case columns", "user", "account"], + ["snake_case columns and plural tables", "users", "accounts"], + ])("reads a Drizzle schema with %s", async (_label, userTable, accountTable) => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE "${userTable}" (id TEXT PRIMARY KEY, email TEXT, email_verified INTEGER, name TEXT, + created_at TEXT, updated_at TEXT, ban_expires TEXT, banned INTEGER)`, + ); + db.run( + `CREATE TABLE "${accountTable}" (id TEXT, user_id TEXT, provider_id TEXT, password TEXT)`, + ); + db.run( + `INSERT INTO "${userTable}" (id, email, email_verified, banned) VALUES ('u1', 'a@x.dev', 1, 1)`, + ); + db.run(`INSERT INTO "${accountTable}" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const { rows, schema } = await withClient(file, async (client) => { + const detected = await detectSchema(client); + return { rows: await client.query(buildBetterAuthQuery(client, detected)), schema: detected }; + }); + + expect([...schema.plugins].sort()).toEqual(["banExpires", "banned"]); + // Read back under the camelCase names the rest of the export expects. + expect(rows).toEqual([ + expect.objectContaining({ + id: "u1", + emailVerified: 1, + banned: 1, + password_hash: "salt:hash", + }), + ]); + }); + + // Better Auth maps each model's fields on its own, so the two tables can + // disagree on casing. + test.each([ + ["snake_case user table, camelCase account table", "snake", "userId", "providerId"], + ["camelCase user table, snake_case account table", "camel", "user_id", "provider_id"], + ])("joins a %s", async (_label, userCase, userId, providerId) => { + const [verified, created, updated] = + userCase === "snake" + ? ["email_verified", "created_at", "updated_at"] + : ["emailVerified", "createdAt", "updatedAt"]; + const file = makeDb((db) => { + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "${verified}" INTEGER, name TEXT, + "${created}" TEXT, "${updated}" TEXT)`, + ); + db.run( + `CREATE TABLE "account" (id TEXT, "${userId}" TEXT, "${providerId}" TEXT, password TEXT)`, + ); + db.run(`INSERT INTO "user" (id, email, "${verified}") VALUES ('u1', 'a@x.dev', 1)`); + db.run(`INSERT INTO "account" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + + expect(rows).toEqual([ + expect.objectContaining({ id: "u1", emailVerified: 1, password_hash: "salt:hash" }), + ]); + }); + + // Each field is mapped on its own, so one table can mix the two casings. + test("resolves each column on its own when a table mixes casings", async () => { + const file = makeDb((db) => { + db.run( + `CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, email_verified INTEGER, name TEXT, + "createdAt" TEXT, updated_at TEXT)`, + ); + db.run(`CREATE TABLE "account" (id TEXT, user_id TEXT, "providerId" TEXT, password TEXT)`); + db.run(`INSERT INTO "user" (id, email, email_verified) VALUES ('u1', 'a@x.dev', 1)`); + db.run(`INSERT INTO "account" VALUES ('a1', 'u1', 'credential', 'salt:hash')`); + }); + + const rows = await withClient(file, async (client) => + client.query(buildBetterAuthQuery(client, await detectSchema(client))), + ); + + expect(rows).toEqual([ + expect.objectContaining({ id: "u1", emailVerified: 1, password_hash: "salt:hash" }), + ]); + }); + + test("passes over a `user` table that is not Better Auth's for the plural one", async () => { + const file = makeDb((db) => { + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, handle TEXT)`); + db.run( + `CREATE TABLE "users" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER, name TEXT, + "createdAt" TEXT, "updatedAt" TEXT)`, + ); + db.run(`CREATE TABLE "accounts" (id TEXT, "userId" TEXT, "providerId" TEXT, password TEXT)`); + db.run(`INSERT INTO "users" (id, email) VALUES ('u1', 'a@x.dev')`); + }); + + const schema = await withClient(file, async (client) => detectSchema(client)); + + expect(schema).toMatchObject({ userTable: "users", accountTable: "accounts" }); + }); + + test("exports a user with two matching credential accounts once", () => { + const lines: UserLine[] = []; + const { users, coverage } = buildBetterAuthExport( + [ + { id: "u1", email: "a@x.dev", password_hash: "salt:hash" }, + { id: "u1", email: "a@x.dev", password_hash: "salt:hash" }, + { id: "u2", email: "b@x.dev", password_hash: null }, + ], + (line) => lines.push(line), + ); + + expect(users.map((user) => user.user_id)).toEqual(["u1", "u2"]); + expect(coverage.find((row) => row.label === "have a password hash")?.count).toBe(1); + expect(lines).toEqual([ + { sourceId: "u1", status: "exported" }, + { sourceId: "u2", status: "exported" }, + ]); + }); + + // Picking one hash would leave the user unable to sign in with the other. + test("skips a user whose credential accounts hold different hashes, and says so", () => { + const lines: UserLine[] = []; + const { users } = buildBetterAuthExport( + [ + { id: "u1", email: "a@x.dev", password_hash: "salt:one" }, + { id: "u1", email: "a@x.dev", password_hash: "salt:two" }, + ], + (line) => lines.push(line), + ); + + expect(users).toEqual([]); + expect(lines).toEqual([ + expect.objectContaining({ + sourceId: "u1", + status: "skipped", + error: expect.stringContaining("2 credential accounts with different password hashes"), + }), + ]); + expect(captured.err).toContain("Skipped 1 user with more than one credential account"); + }); + + test("renames camelCase columns onto what the transformer reads", () => { + const { users } = buildBetterAuthExport([ + { id: "u1", emailVerified: 1, phoneNumber: "+1555", createdAt: "2025-01-01" }, + ]); + expect(users[0]).toMatchObject({ + user_id: "u1", + email_verified: 1, + phone_number: "+1555", + created_at: "2025-01-01", + }); + }); + + test("exports end to end and reports the detected plugins", async () => { + const file = betterAuthDb(["username"], [{ id: "u1", email: "a@x.dev", username: "ada" }]); + + await exportBetterAuth({ dbUrl: file, output: "ba.json" }); + + expect(captured.err).toContain("Detected plugin columns: username"); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "ba.json"), "utf-8")).users).toHaveLength( + 1, + ); + }); + + test("says so plainly when no plugins are in use", async () => { + await exportBetterAuth({ dbUrl: betterAuthDb([]), output: "ba2.json" }); + expect(captured.err).toContain("No plugin columns detected"); + }); +}); + +describe("supabase export", () => { + // The source splits a display or OAuth name, but only when first_name is + // empty; coalescing them into first_name here would stop it. + test("selects only the metadata's own first_name as first_name", async () => { + let sql = ""; + await fetchSupabaseUsers({ + query: async (query: string) => { + sql = query; + return []; + }, + } as unknown as DbClient); + expect(sql).toContain("raw_user_meta_data->>'first_name' AS first_name"); + expect(sql).not.toContain("display_name"); + }); + + test("serializes timestamps the transformer can parse", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", created_at: new Date("2024-01-01T00:00:00Z") }, + ]); + expect(users[0]?.created_at).toBe("2024-01-01T00:00:00.000Z"); + }); + + test("omits null columns rather than exporting them", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", phone: null, last_name: null }, + ]); + expect("phone" in (users[0] ?? {})).toBe(false); + expect("last_name" in (users[0] ?? {})).toBe(false); + }); + + test("counts the password hashes, the reason this reads the database", () => { + const { coverage } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", encrypted_password: "$2b$10$x" }, + { id: "u2", email: "b@x.dev" }, + ]); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + }); + + test("keeps raw_app_meta_data, which the import checks read for providers", () => { + const { users } = buildSupabaseExport([ + { id: "u1", email: "a@x.dev", raw_app_meta_data: { providers: ["discord"] } }, + ]); + expect(users[0]?.raw_app_meta_data).toEqual({ providers: ["discord"] }); + }); + + test("records one line per exported user", () => { + const lines: UserLine[] = []; + buildSupabaseExport([{ id: "u1" }, { id: "u2" }], (line) => lines.push(line)); + + expect(lines).toHaveLength(2); + }); +}); + +describe("connection failures", () => { + afterEach(() => { + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); + }); + + test("a missing SQLite file fails before anything is written", async () => { + await expect(exportAuthJs({ dbUrl: "./definitely-not-here.sqlite" })).rejects.toThrow(CliError); + expect(fs.existsSync(path.join(workDir, "exports"))).toBe(false); + }); + + // Supabase is Postgres: anything else would connect and then fail. + test.each([["mysql://u:p@127.0.0.1:1/db"], ["./definitely-not-here.sqlite"]])( + "refuses %s before connecting or writing a run", + async (dbUrl) => { + await expect(exportSupabase({ dbUrl })).rejects.toThrow(/Supabase's database is Postgres/); + expect(fs.existsSync(path.join(workDir, ".clerk"))).toBe(false); + }, + ); + + test("the failure never contains the password", async () => { + await expect( + exportBetterAuth({ dbUrl: "postgres://user:hunter2@127.0.0.1:1/db" }), + ).rejects.toThrow( + expect.objectContaining({ message: expect.not.stringContaining("hunter2") }) as Error, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/db-options.ts b/packages/cli-core/src/commands/migrate/export/db-options.ts new file mode 100644 index 000000000..96c895f65 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/db-options.ts @@ -0,0 +1,221 @@ +/** + * Resolving a `--db-url` for the database-backed exports. + * + * Shared by supabase, authjs and betterauth: all three take one connection + * string, from a flag, an environment variable, or a prompt. + */ + +import { CliError, throwUsageError } from "../../../lib/errors.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { withSpinner } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + isLibsqlUrl, + redactConnectionString, + redactDbMessage, + type DbClient, + type DbPlatform, +} from "../lib/db.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; +import { withInputRetry } from "../lib/input-retry.ts"; + +export type DbExportOptions = { + dbUrl?: string; + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +export type ResolveConfig = { + platform: DbPlatform; + /** Environment variable checked when `--db-url` is absent. */ + envVar: string; + prompt: string; + /** Extra guidance shown before prompting. */ + hint?: string; +}; + +const URL_SCHEME = /^(postgresql|postgres|mysql|mysql2|libsql):\/\//i; + +/** + * True when the string parses as a URL with a host. + * + * A hostname is required: `postgres://` alone parses as a valid URL, and + * accepting it only defers the failure into the driver. + */ +function parsesAsUrl(value: string): boolean { + try { + const url = new URL(value); + // A bare `%` in the password parses, then fails the driver's decode with a + // cryptic "URI error"; treat it as unparsed so it gets encoded. + decodeURIComponent(url.username); + decodeURIComponent(url.password); + return url.hostname.length > 0; + } catch { + return false; + } +} + +/** + * Percent-encodes the credentials when the raw string will not parse as a URL. + * + * Dashboards hand out `postgres://user:[YOUR-PASSWORD]@host/db` and people + * paste their real password in verbatim. A `#`, `@`, `/` or `^` in it makes the + * whole string unparseable — here and later inside `Bun.SQL` — so encode it for + * them rather than bouncing a paste they cannot even see (the prompt is + * masked). Strings that already parse are returned untouched, so a password + * that was correctly encoded is never double-encoded. + */ +export function normalizeConnectionString(value: string): string { + const trimmed = value.trim(); + if (!URL_SCHEME.test(trimmed) || parsesAsUrl(trimmed)) return trimmed; + + // Greedy up to the LAST `@`: everything before it is userinfo, so an + // unencoded `@` inside the password does not split the string early. + const match = /^([a-z0-9+]+:\/\/)(.*)@([^@]*)$/i.exec(trimmed); + if (!match) return trimmed; + + const [, scheme = "", userinfo = "", rest = ""] = match; + const separator = userinfo.indexOf(":"); + const user = separator === -1 ? userinfo : userinfo.slice(0, separator); + const secret = separator === -1 ? undefined : userinfo.slice(separator + 1); + const credentials = + secret === undefined + ? encodeURIComponent(user) + : `${encodeURIComponent(user)}:${encodeURIComponent(secret)}`; + + const encoded = `${scheme}${credentials}@${rest}`; + return parsesAsUrl(encoded) ? encoded : trimmed; +} + +/** True for something that could plausibly be a connection string. */ +export function looksLikeConnectionString(value: string): boolean { + const trimmed = value.trim(); + if (!trimmed) return false; + + if (URL_SCHEME.test(trimmed)) return parsesAsUrl(trimmed); + + return ( + trimmed.startsWith("file:") || /\.(sqlite3?|db)$/i.test(trimmed) || trimmed.startsWith("./") + ); +} + +/** + * Resolves the connection string: flag, then environment, then a prompt. + * + * Prompted as a password so it is not echoed — a connection string carries the + * database password inline. + */ +export async function resolveDbUrl( + options: DbExportOptions, + config: ResolveConfig, + env: Record = process.env, +): Promise { + const fromFlag = options.dbUrl ? normalizeConnectionString(options.dbUrl) : undefined; + if (fromFlag) { + if (!looksLikeConnectionString(fromFlag)) { + throwUsageError( + `--db-url does not look like a connection string. Expected postgres://…, mysql://…, libsql://… or a SQLite file path.\n` + + "If the password contains @, # or /, URL-encode it.", + ); + } + return fromFlag; + } + + const raw = env[config.envVar]; + const fromEnv = raw ? normalizeConnectionString(raw) : undefined; + if (fromEnv) { + if (looksLikeConnectionString(fromEnv)) return fromEnv; + // Falling through silently would make the prompt look unexplained. + log.warn(`${config.envVar} is not a valid connection string; ignoring it.`); + } + + // `-y` is "do not prompt": it gets the usage error naming --db-url too. + if (options.json || isAssumeYes() || isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export ${config.platform}\` needs a database connection and cannot prompt here.\n` + + `Pass --db-url, or set ${config.envVar}.`, + undefined, + undefined, + [ + { + command: `clerk migrate export ${config.platform} --db-url "postgres://user:password@host:5432/db"`, + description: "Export from Postgres", + }, + ], + ); + } + + if (config.hint) log.info(dim(config.hint)); + + return promptDbUrl(config); +} + +/** + * Asks for a connection string, masked. + * + * Masked because a connection string carries the database password inline. The + * validator runs on the normalized value, so a password that needed encoding is + * judged as the driver will see it, not as it was typed. + */ +export async function promptDbUrl(config: ResolveConfig): Promise { + const answer = await passwordPrompt({ + message: config.prompt, + validate: (value) => + looksLikeConnectionString(normalizeConnectionString(value ?? "")) + ? undefined + : "Expected postgres://…, mysql://…, libsql://… or a SQLite file path", + }); + + return normalizeConnectionString(answer); +} + +/** Describes the target for the run's opening line, credentials removed. */ +export function describeTarget(connectionString: string): string { + const label = isLibsqlUrl(connectionString) ? "libsql" : detectDbType(connectionString); + return `${label} at ${redactConnectionString(connectionString)}`; +} + +/** + * Connects, asking for the connection string again if the server refuses it, + * then runs `work` on the client and closes it. + * + * Only the connect is retried. Once it has worked, the string is right: a + * statement timeout or a dropped connection partway through a large read is + * not fixed by asking for it again, so it fails (exit 1) with the hint for it. + */ +export async function withDbConnection( + dbUrl: string, + config: ResolveConfig, + options: { json?: boolean }, + work: (client: DbClient) => Promise, +): Promise { + const { value: client, input: connectionString } = await withInputRetry( + dbUrl, + async () => promptDbUrl(config), + async (candidate) => + withSpinner("Connecting to the database...", async () => + createDbClient(candidate, config.platform), + ), + options, + ); + try { + return await work(client); + } catch (error) { + if (error instanceof CliError) throw error; + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); + throw new CliError(`${message}\n\n${describeDbError(error, config.platform)}`); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/firebase.test.ts b/packages/cli-core/src/commands/migrate/export/firebase.test.ts new file mode 100644 index 000000000..0a97ba570 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.test.ts @@ -0,0 +1,590 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { setAssumeYes } from "../lib/assume-yes.ts"; +import { + buildFirebaseExport, + exportFirebase, + fetchAccessToken, + fetchAllFirebaseUsers, + fetchHashConfig, + formatHashConfigGuidance, + loadServiceAccount, + mapFirebaseUserToExport, + readServiceAccount, + signServiceAccountJwt, + type ServiceAccount, +} from "./firebase.ts"; + +const captured = useCaptureLog(); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { url: string; body: unknown }[]; +let account: ServiceAccount; + +beforeAll(async () => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-fb-"))); + process.chdir(workDir); + + // A real RSA key, so the signing path is genuinely exercised. + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ); + const pkcs8 = await crypto.subtle.exportKey("pkcs8", pair.privateKey); + const body = btoa(String.fromCharCode(...new Uint8Array(pkcs8))).replace(/(.{64})/g, "$1\n"); + + account = { + project_id: "demo-fb", + client_email: "exp@demo-fb.iam.gserviceaccount.com", + private_key: `-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`, + }; + fs.writeFileSync( + path.join(workDir, "sa.json"), + JSON.stringify({ type: "service_account", ...account }), + ); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a + // leaked "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; +}); + +/** Answers the token exchange, then one page per entry in `pages`. */ +function stubFirebase(pages: Record[][], hashConfig?: unknown) { + let page = 0; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ url, body: init?.body ?? null }); + + if (url.includes("oauth2.googleapis.com/token")) { + return Response.json({ access_token: "tok" }); + } + if (url.includes("/config")) { + return hashConfig === undefined + ? new Response("forbidden", { status: 403 }) + : Response.json(hashConfig); + } + const current = pages[page++] ?? []; + const hasMore = page < pages.length; + return Response.json({ users: current, ...(hasMore ? { nextPageToken: `p${page}` } : {}) }); + }) as unknown as typeof fetch; +} + +const fbUser = (i: number, overrides: Record = {}) => ({ + localId: `fb${i}`, + email: `u${i}@fb.dev`, + emailVerified: true, + displayName: `User ${i}`, + passwordHash: `SGFzaA${i}`, + salt: `U2FsdA${i}`, + createdAt: "1704067200000", + ...overrides, +}); + +describe("loadServiceAccount", () => { + // The prompt takes either, so a key pasted out of a password manager never + // has to be written to disk first. + test("accepts the key JSON pasted whole", () => { + expect(loadServiceAccount(` ${JSON.stringify(account)} `).project_id).toBe("demo-fb"); + }); + + test("accepts a path to the key file", () => { + expect(loadServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("rejects a paste that is not valid JSON", () => { + expect(() => loadServiceAccount('{"project_id":')).toThrow(/pasted key is not valid JSON/); + }); + + test("rejects a paste missing a required field", () => { + expect(() => loadServiceAccount('{"type":"service_account"}')).toThrow( + /The pasted key is not a usable service account key: "project_id" is missing/, + ); + }); +}); + +describe("readServiceAccount", () => { + test("reads a valid key file", () => { + expect(readServiceAccount("./sa.json").project_id).toBe("demo-fb"); + }); + + test("reports a path that is not there", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(/No service account file at/); + }); + + test("reports a file that is not JSON", () => { + fs.writeFileSync(path.join(workDir, "bad.json"), "not json"); + expect(() => readServiceAccount("./bad.json")).toThrow(/is not valid JSON/); + }); + + // Downloading the web app config instead of a service account key is the + // usual mistake, and the two files look similar at a glance. + test("points at the right console page for a web app config", () => { + fs.writeFileSync(path.join(workDir, "web.json"), JSON.stringify({ apiKey: "x" })); + expect(() => readServiceAccount("./web.json")).toThrow(/"project_id" is missing/); + }); + + test("names a wrong type explicitly", () => { + fs.writeFileSync(path.join(workDir, "wrong.json"), JSON.stringify({ type: "authorized_user" })); + expect(() => readServiceAccount("./wrong.json")).toThrow( + /"type" is "authorized_user".*Generate new private key/s, + ); + }); + + test.each([["project_id"], ["client_email"], ["private_key"]])( + "reports a missing %s", + (field) => { + const partial: Record = { type: "service_account", ...account }; + delete partial[field]; + fs.writeFileSync(path.join(workDir, `no-${field}.json`), JSON.stringify(partial)); + expect(() => readServiceAccount(`./no-${field}.json`)).toThrow( + new RegExp(`"${field}" is missing`), + ); + }, + ); + + // Pasting a key through a form that eats newlines is common, and the failure + // would otherwise surface as an opaque crypto error. + test("catches a private key whose newlines were mangled", () => { + fs.writeFileSync( + path.join(workDir, "mangled.json"), + JSON.stringify({ type: "service_account", ...account, private_key: "mangled" }), + ); + expect(() => readServiceAccount("./mangled.json")).toThrow(/newlines survived copying/); + }); + + test("raises CliError so the global handler formats it", () => { + expect(() => readServiceAccount("./nope.json")).toThrow(CliError); + }); +}); + +describe("signServiceAccountJwt", () => { + test("produces a three-segment RS256 JWT", async () => { + const jwt = await signServiceAccountJwt(account); + expect(jwt.split(".")).toHaveLength(3); + }); + + test("claims the right issuer, audience and scopes", async () => { + const jwt = await signServiceAccountJwt(account, 1_700_000_000); + const claims = JSON.parse( + atob((jwt.split(".")[1] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + + expect(claims).toMatchObject({ + iss: "exp@demo-fb.iam.gserviceaccount.com", + aud: "https://oauth2.googleapis.com/token", + iat: 1_700_000_000, + exp: 1_700_003_600, + }); + expect(claims.scope).toContain("cloud-platform"); + }); + + test("declares RS256 in the header", async () => { + const jwt = await signServiceAccountJwt(account); + const header = JSON.parse( + atob((jwt.split(".")[0] as string).replace(/-/g, "+").replace(/_/g, "/")), + ); + expect(header).toEqual({ alg: "RS256", typ: "JWT" }); + }); + + test("rejects a private key that is not valid base64", async () => { + await expect( + signServiceAccountJwt({ + ...account, + private_key: "-----BEGIN PRIVATE KEY-----\n!!!\n-----END PRIVATE KEY-----", + }), + ).rejects.toThrow(CliError); + }); +}); + +describe("fetchAccessToken", () => { + test("exchanges the assertion for a token", async () => { + stubFirebase([[]]); + + expect(await fetchAccessToken(account)).toBe("tok"); + expect(String(requests[0]?.body)).toContain("grant-type%3Ajwt-bearer"); + }); + + test("explains a rejection rather than surfacing a raw status", async () => { + globalThis.fetch = (async () => + new Response(JSON.stringify({ error_description: "Invalid JWT Signature" }), { + status: 400, + })) as unknown as typeof fetch; + + await expect(fetchAccessToken(account)).rejects.toThrow( + /Google rejected the service account \(400\): Invalid JWT Signature/, + ); + }); + + test("names the role the service account usually lacks", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 403 })) as unknown as typeof fetch; + await expect(fetchAccessToken(account)).rejects.toThrow(/revoked or deleted/); + }); + + // The emulator has no token endpoint; `firebase-admin` uses the same bearer. + test("skips the exchange entirely against the emulator", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + globalThis.fetch = (async () => { + throw new Error("should not have been called"); + }) as unknown as typeof fetch; + + expect(await fetchAccessToken(account)).toBe("owner"); + }); +}); + +describe("fetchAllFirebaseUsers", () => { + test("follows nextPageToken until it stops coming", async () => { + stubFirebase([ + Array.from({ length: 1000 }, (_, i) => fbUser(i)), + Array.from({ length: 7 }, (_, i) => fbUser(1000 + i)), + ]); + + const all = await fetchAllFirebaseUsers({ account, token: "tok" }); + + expect(all).toHaveLength(1007); + expect(requests[1]?.url).toContain("nextPageToken=p1"); + }); + + test("asks for the endpoint's maximum page size", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("maxResults=1000"); + }); + + test("targets the project named in the key", async () => { + stubFirebase([[]]); + await fetchAllFirebaseUsers({ account, token: "tok" }); + expect(requests[0]?.url).toContain("/projects/demo-fb/accounts:batchGet"); + }); + + test("routes through the emulator when one is configured", async () => { + process.env.FIREBASE_AUTH_EMULATOR_HOST = "127.0.0.1:9099"; + stubFirebase([[]]); + + await fetchAllFirebaseUsers({ account, token: "owner" }); + + expect(requests[0]?.url).toStartWith("http://127.0.0.1:9099/"); + }); + + test("raises a clear error on a failed page", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + await expect(fetchAllFirebaseUsers({ account, token: "tok" })).rejects.toThrow( + /Firebase returned 500 listing users/, + ); + }); +}); + +describe("mapFirebaseUserToExport", () => { + test("keeps the fields the firebase transformer maps from", () => { + expect(mapFirebaseUserToExport(fbUser(0))).toEqual({ + localId: "fb0", + email: "u0@fb.dev", + displayName: "User 0", + createdAt: "1704067200000", + emailVerified: true, + passwordHash: "SGFzaA0", + salt: "U2FsdA0", + }); + }); + + test("drops project internals the import has no use for", () => { + const mapped = mapFirebaseUserToExport( + fbUser(0, { + providerUserInfo: [{ providerId: "password" }], + lastLoginAt: "1704153600000", + customAttributes: '{"role":"x"}', + validSince: "1704067200", + }), + ); + for (const noise of ["providerUserInfo", "lastLoginAt", "customAttributes", "validSince"]) { + expect(noise in mapped).toBe(false); + } + }); + + // A digest without its salt cannot be verified, so exporting one alone would + // produce a user nobody can sign in as. + test.each([ + ["hash without salt", { passwordHash: "H", salt: undefined }], + ["salt without hash", { passwordHash: undefined, salt: "S" }], + ])("drops a %s", (_label, overrides) => { + const mapped = mapFirebaseUserToExport(fbUser(0, overrides)); + expect("passwordHash" in mapped).toBe(false); + expect("salt" in mapped).toBe(false); + }); + + test("keeps emailVerified when it is false", () => { + expect(mapFirebaseUserToExport(fbUser(0, { emailVerified: false })).emailVerified).toBe(false); + }); + + test("copes with a phone-only user", () => { + const mapped = mapFirebaseUserToExport({ localId: "fb9", phoneNumber: "+15555550100" }); + expect(mapped).toEqual({ localId: "fb9", phoneNumber: "+15555550100" }); + }); +}); + +test("mapFirebaseUserToExport keeps disabled only when it is true", () => { + expect(mapFirebaseUserToExport(fbUser(0, { disabled: true })).disabled).toBe(true); + expect("disabled" in mapFirebaseUserToExport(fbUser(0, { disabled: false }))).toBe(false); +}); + +describe("buildFirebaseExport", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; + const { users, coverage } = buildFirebaseExport( + [fbUser(0), { localId: "fb1", phoneNumber: "+1555" }], + (line) => lines.push(line), + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have a password hash"]).toBe(1); + expect(byLabel["have a phone number"]).toBe(1); + expect(lines).toHaveLength(2); + }); + + // Firebase returns an empty passwordHash for users it did not hash itself. + test("counts password users whose hash Firebase did not return", () => { + const { users, unreadablePasswords } = buildFirebaseExport([ + fbUser(0, { providerUserInfo: [{ providerId: "password" }] }), + fbUser(1, { passwordHash: "", providerUserInfo: [{ providerId: "password" }] }), + fbUser(2, { passwordHash: "", providerUserInfo: [{ providerId: "google.com" }] }), + ]); + expect(unreadablePasswords).toBe(1); + expect("passwordHash" in users[1]!).toBe(false); + }); + + // Firebase sends base64 "REDACTED" when the caller may not read hashes: a + // digest that could never verify, which must not be exported as one. + test("treats a redacted hash as none, and counts it", () => { + const { users, redactedPasswords } = buildFirebaseExport([ + fbUser(0, { passwordHash: "UkVEQUNURUQ=", providerUserInfo: [{ providerId: "password" }] }), + ]); + expect(redactedPasswords).toBe(1); + expect("passwordHash" in users[0]!).toBe(false); + expect("salt" in users[0]!).toBe(false); + }); +}); + +describe("fetchHashConfig", () => { + test("reads the project's scrypt parameters", async () => { + stubFirebase([[]], { + signIn: { + hashConfig: { signerKey: "KEY=", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }, + }, + }); + + expect(await fetchHashConfig(account, "tok")).toEqual({ + signerKey: "KEY=", + saltSeparator: "Bw==", + rounds: 8, + memoryCost: 14, + }); + }); + + // Firebase leaves the field out when the project has no separator. + test("reads a project with no salt separator as an empty one", async () => { + stubFirebase([[]], { + signIn: { hashConfig: { signerKey: "KEY=", rounds: 8, memoryCost: 14 } }, + }); + + expect(await fetchHashConfig(account, "tok")).toEqual({ + signerKey: "KEY=", + saltSeparator: "", + rounds: 8, + memoryCost: 14, + }); + }); + + // Reading the config needs a broader role than listing users, so a project + // where it is denied must still export. + test("returns null rather than failing when the call is not permitted", async () => { + stubFirebase([[]]); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + // Names the permission, so the operator can grant it rather than guess. + expect(captured.err).toContain("firebaseauth.configs.getHashConfig"); + }); + + // Every digest carries these, so a set Clerk would refuse is not written. + test("leaves out parameters Clerk would refuse, and says so", async () => { + stubFirebase([[]], { + signIn: { + hashConfig: { signerKey: "KEY=", saltSeparator: "Bw==", rounds: 17, memoryCost: 14 }, + }, + }); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + expect(captured.err).toContain( + "won't work in Clerk (rounds must be a whole number from 1 to 16", + ); + }); + + test("returns null when the response carries no hash config", async () => { + stubFirebase([[]], { signIn: {} }); + expect(await fetchHashConfig(account, "tok")).toBeNull(); + }); +}); + +describe("formatHashConfigGuidance", () => { + const config = { signerKey: "KEY==", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }; + + // They are in the envelope now, so the import needs no flags for them. + test("says the parameters travel in the export file when the project gave them", () => { + const text = formatHashConfigGuidance(config, 3).join("\n"); + expect(text).toContain("saved in the export file"); + expect(text).not.toContain("--firebase-signer-key"); + }); + + test("says where to find them when the project would not say", () => { + const text = formatHashConfigGuidance(null, 3).join("\n"); + expect(text).toContain("Password hash parameters"); + expect(text).toContain("Authentication → Users"); + expect(text).toContain("--firebase-signer-key"); + }); + + // Nothing to configure, so nothing to tell them to configure. + test("says nothing is needed when the export has no hashes", () => { + expect(formatHashConfigGuidance(null, 0).join("\n")).toContain("no hash parameters are needed"); + }); +}); + +/** The envelope the one export run in this project wrote. */ +function onlyExportFile(): string { + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); + expect(entries).toHaveLength(1); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; +} + +describe("exportFirebase", () => { + // `-y` is "do not prompt", at a terminal too. + test("does not ask for a service account key under -y", async () => { + process.env.CLERK_MODE = "human"; + setAssumeYes(true); + try { + await expect(exportFirebase({})).rejects.toThrow(/cannot prompt here/); + } finally { + setAssumeYes(false); + } + }); + + test("exports end to end and reports coverage", async () => { + stubFirebase([[fbUser(0), fbUser(1)]], { + signIn: { + hashConfig: { signerKey: "KEY=", saltSeparator: "Bw==", rounds: 8, memoryCost: 14 }, + }, + }); + + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "firebase", + firebase: { + base64_signer_key: "KEY=", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }); + const written = exportedUsers(); + expect(written).toHaveLength(2); + expect(captured.err).toContain("Field coverage"); + expect(captured.err).toContain("demo-fb project"); + }); + + // Firebase's own variable, so it's honoured, but an emulator's users are + // not the project's: the target line says which. + test("names the emulator in the target line when one is set", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + process.env.FIREBASE_AUTH_EMULATOR_HOST = "localhost:9099"; + try { + await exportFirebase({ serviceAccount: "./sa.json" }); + } finally { + delete process.env.FIREBASE_AUTH_EMULATOR_HOST; + } + expect(captured.err).toContain("Source: firebase (emulator at localhost:9099)"); + }); + + test("names the command that consumes the file", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + // The suggestion now rides the gutter's Next steps block, which only + // renders in human mode. + const originalMode = getMode(); + setMode("human"); + try { + await exportFirebase({ serviceAccount: "./sa.json", output: "exports/mine.json" }); + } finally { + setMode(originalMode); + } + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); + }); + + test("--output controls the destination", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json", output: "fb.json" }); + expect(fs.existsSync(path.join(workDir, "fb.json"))).toBe(true); + }); + + // Human runs get a prompt instead; an agent has nobody to ask, so it is told + // which flag to pass. + test("agent mode names the flag rather than prompting", async () => { + await expect(exportFirebase({})).rejects.toThrow(/needs a service account key file/); + }); + + test("validates the key file before making any request", async () => { + stubFirebase([[fbUser(0)]]); + await expect(exportFirebase({ serviceAccount: "./nope.json" })).rejects.toThrow(CliError); + expect(requests).toHaveLength(0); + }); + + test("never puts key material in the output", async () => { + stubFirebase([[fbUser(0)]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).not.toContain("BEGIN PRIVATE KEY"); + expect(captured.err).not.toContain(account.private_key.slice(40, 80)); + }); + + test("skips the hash-parameter section when nothing has a password", async () => { + stubFirebase([[{ localId: "fb9", phoneNumber: "+1555" }]], { signIn: {} }); + await exportFirebase({ serviceAccount: "./sa.json" }); + expect(captured.err).toContain("no hash parameters are needed"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/firebase.ts b/packages/cli-core/src/commands/migrate/export/firebase.ts new file mode 100644 index 000000000..0630fe887 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/firebase.ts @@ -0,0 +1,643 @@ +/** + * `clerk migrate export firebase` — pull users out of Firebase Authentication. + * + * Ported from the standalone migration-tool's `src/export/firebase.ts`, but + * **without `firebase-admin`**. + * + * The spike the ticket asked for was run first, and it passed: a + * `bun build --compile` binary imports `firebase-admin`, initializes it, and + * completes `listUsers` against Identity Toolkit. The known Firestore-under- + * compile bug does not reach the Auth Admin surface. + * + * The SDK was still not adopted, on the second measurement: it is **74 MB + * across 158 packages**, including `@google-cloud/firestore` and + * `@google-cloud/storage`, neither of which this command touches. The compiled + * `clerk` binary is ~65 MB today, so that roughly doubles the artifact every + * user downloads — to serve one subcommand. + * + * What the SDK actually does here is two REST calls and an RS256 JWT, and Bun's + * Web Crypto signs RS256 with no dependency at all (verified compiled). So this + * adds **zero** packages, and its HTTP goes through `loggedFetch`, so a + * `--verbose` run shows the requests — which an SDK doing its own fetch would + * not. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { bold, dim } from "../../../lib/color.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { log } from "../../../lib/log.ts"; +import { password as passwordPrompt } from "../../../lib/prompts.ts"; +import { isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; +import { firebaseHashConfigProblem } from "../lib/firebase-hash.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import type { FirebaseHashConfig } from "../types.ts"; +import { isCredentialStatus, throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; +import { finishExport, startExportRun } from "./shared.ts"; + +/** Identity Toolkit's maximum for `accounts:batchGet`. */ +const PAGE_SIZE = 1000; + +const TOKEN_URL = "https://oauth2.googleapis.com/token"; +const SCOPES = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/firebase", +].join(" "); + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/firebase"; + +export type ExportFirebaseOptions = { + serviceAccount?: string; + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +export type ServiceAccount = { + project_id: string; + client_email: string; + private_key: string; +}; + +/** + * Validates already-parsed JSON as a service-account key. + * + * Every failure names the field, because the usual causes are downloading the + * wrong JSON from the console (a web app config rather than a service account) + * or pasting a key with its newlines mangled. + * + * @param label how to refer to the source in an error — a file name, or + * "the pasted key" when it came from the prompt. + */ +function validateServiceAccount(parsed: unknown, label: string): ServiceAccount { + const account = parsed as Partial & { type?: string }; + const invalid = (problem: string): never => { + throwUsageError(`${label} is not a usable service account key: ${problem}`, DOCS_URL); + }; + + if (account.type && account.type !== "service_account") { + invalid( + `its "type" is "${account.type}", not "service_account". Download a private key from ` + + "Project settings → Service accounts → Generate new private key.", + ); + } + for (const field of ["project_id", "client_email", "private_key"] as const) { + if (typeof account[field] !== "string" || account[field].length === 0) { + invalid(`"${field}" is missing`); + } + } + if (!account.private_key?.includes("PRIVATE KEY")) { + invalid('"private_key" does not look like a PEM key — check its newlines survived copying'); + } + + return account as ServiceAccount; +} + +/** Reads and validates a service-account key file. */ +export function readServiceAccount(file: string): ServiceAccount { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No service account file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + + let parsed: unknown; + try { + parsed = JSON.parse(fs.readFileSync(resolved, "utf-8")); + } catch (error) { + throw new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + return validateServiceAccount(parsed, file); +} + +/** + * Accepts what the prompt accepts: a path to the downloaded key file, or the + * key's JSON pasted in whole. Console downloads land as a file, but a key + * copied out of a password manager or CI secret never touches disk. + */ +export function loadServiceAccount(source: string): ServiceAccount { + const trimmed = source.trim(); + if (!trimmed.startsWith("{")) return readServiceAccount(trimmed); + + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (error) { + throw new CliError(`The pasted key is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + docsUrl: DOCS_URL, + }); + } + + return validateServiceAccount(parsed, "The pasted key"); +} + +/** + * Resolves the key: the flag, then a prompt — the shape `export supabase` uses + * for its connection string. Prompted as a password: the JSON carries a private + * key, and a path typed blind is short enough to survive being masked. + */ +async function resolveServiceAccount(options: ExportFirebaseOptions): Promise { + if (options.serviceAccount) return loadServiceAccount(options.serviceAccount); + + // `-y` is "do not prompt", as for the other exports. + if (options.json || isAssumeYes() || !isHuman()) { + throwUsageError( + "`clerk migrate export firebase` needs a service account key file and cannot prompt here. " + + "Pass --service-account .", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ], + ); + } + + log.info( + dim("Firebase console → Project settings → Service accounts → Generate new private key."), + ); + + return promptServiceAccount(); +} + +/** + * Asks for the key, masked. + * + * Masked because the JSON carries a private key, and a path typed blind is + * short enough to survive it. What the file cannot tell us — whether Google + * still accepts the key — is left to the token exchange, which is why this is + * separate from {@link resolveServiceAccount}: a revoked key has to be asked + * for again after that call fails, not before it is made. + */ +async function promptServiceAccount(): Promise { + const answer = await passwordPrompt({ + message: "Path to the service account key file, or paste the key JSON", + validate: (value) => { + try { + loadServiceAccount(value ?? ""); + return undefined; + } catch (error) { + return error instanceof CliError ? error.message : String(error); + } + }, + }); + + return loadServiceAccount(answer); +} + +function base64Url(input: string | Uint8Array): string { + const binary = + typeof input === "string" ? input : String.fromCharCode(...(input as unknown as number[])); + return btoa(binary).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +/** Imports the PEM private key for RS256 signing. */ +async function importPrivateKey(pem: string): Promise { + const body = pem.replace(/-----[^-]+-----/g, "").replace(/\s+/g, ""); + let der: Uint8Array; + try { + der = Uint8Array.from(atob(body), (character) => character.charCodeAt(0)); + } catch { + throwUsageError("The service account's private_key is not valid base64.", DOCS_URL); + } + + try { + return await crypto.subtle.importKey( + "pkcs8", + der, + { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, + false, + ["sign"], + ); + } catch (error) { + throwUsageError( + `The service account's private_key could not be read: ${(error as Error).message}`, + DOCS_URL, + ); + } +} + +/** + * Signs the assertion Google exchanges for an access token. + * + * @param now - Seconds since the epoch; injectable so tests are not clock-bound. + */ +export async function signServiceAccountJwt( + account: ServiceAccount, + now: number = Math.floor(Date.now() / 1000), +): Promise { + const key = await importPrivateKey(account.private_key); + const claims = { + iss: account.client_email, + scope: SCOPES, + aud: TOKEN_URL, + iat: now, + exp: now + 3600, + }; + const body = `${base64Url(JSON.stringify({ alg: "RS256", typ: "JWT" }))}.${base64Url(JSON.stringify(claims))}`; + + const signature = await crypto.subtle.sign( + "RSASSA-PKCS1-v1_5", + key, + new TextEncoder().encode(body), + ); + + return `${body}.${base64Url(new Uint8Array(signature))}`; +} + +/** + * Exchanges the signed assertion for an Identity Toolkit access token. + * + * Against the emulator there is nothing to exchange with — Google's token + * endpoint is not part of it — so the run uses the `owner` bearer the emulator + * accepts, matching what `firebase-admin` does. + */ +export async function fetchAccessToken(account: ServiceAccount): Promise { + if (process.env.FIREBASE_AUTH_EMULATOR_HOST) return "owner"; + + const assertion = await signServiceAccountJwt(account); + + const response = await loggedFetch(new URL(TOKEN_URL), { + tag: "firebase", + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", + assertion, + }).toString(), + }); + + const body = (await response.json().catch(() => ({}))) as { + access_token?: string; + error_description?: string; + error?: string; + }; + + const detail = body.error_description ?? body.error ?? "no access token returned"; + if (!response.ok && !isCredentialStatus(response.status)) { + throwApiFailure( + response.status, + `Google did not issue a token (${response.status}): ${detail}. Try again shortly.`, + DOCS_URL, + ); + } + if (!body.access_token) { + throwUsageError( + `Google rejected the service account (${response.status}): ${detail}\n` + + "Check the key has not been revoked or deleted, in the Google Cloud console under IAM → Service accounts.", + DOCS_URL, + ); + } + + return body.access_token; +} + +/** + * Base URL for Identity Toolkit. + * + * Honours `FIREBASE_AUTH_EMULATOR_HOST`, the variable Firebase's own tooling + * uses, so this works against the local emulator as well as production. + */ +function identityToolkitBase(): string { + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + return emulator + ? `http://${emulator}/identitytoolkit.googleapis.com` + : "https://identitytoolkit.googleapis.com"; +} + +export type FirebaseUser = Record & { localId?: string }; + +/** Pages through every user in the project. */ +export async function fetchAllFirebaseUsers(options: { + account: ServiceAccount; + token: string; + spinner?: SpinnerControls; +}): Promise { + const all: FirebaseUser[] = []; + let pageToken: string | undefined; + + do { + const url = new URL( + `${identityToolkitBase()}/v1/projects/${options.account.project_id}/accounts:batchGet`, + ); + url.searchParams.set("maxResults", String(PAGE_SIZE)); + if (pageToken) url.searchParams.set("nextPageToken", pageToken); + + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${options.token}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throwApiFailure( + response.status, + `Firebase returned ${response.status} listing users: ${await response.text()}`, + DOCS_URL, + ); + } + + const body = (await response.json()) as { users?: FirebaseUser[]; nextPageToken?: string }; + all.push(...(body.users ?? [])); + options.spinner?.update(`Fetching users from Firebase: ${all.length} so far...`); + pageToken = body.nextPageToken; + } while (pageToken); + + return all; +} + +export type HashConfig = { + signerKey: string; + saltSeparator: string; + rounds: number; + memoryCost: number; +}; + +/** + * Reads the project's scrypt parameters. + * + * These are the whole reason a Firebase migration keeps its passwords: without + * them Clerk cannot verify a single digest. Fetching them here saves the user + * hunting through the console — and if the call is not permitted, the run says + * exactly where to look instead. + * + * @returns `null` when the config could not be read. + */ +export async function fetchHashConfig( + account: ServiceAccount, + token: string, +): Promise { + try { + const url = new URL(`${identityToolkitBase()}/admin/v2/projects/${account.project_id}/config`); + const response = await loggedFetch(url, { + tag: "firebase", + method: "GET", + headers: { Authorization: `Bearer ${token}`, Accept: "application/json" }, + }); + if (!response.ok) { + if (response.status === 403) { + log.warn( + "The service account cannot read the project's password hash parameters: it needs the " + + "`firebaseauth.configs.getHashConfig` permission. Grant it and export again, or pass the " + + "parameters to the import with --firebase-*.", + ); + } else { + log.debug(`firebase: ${response.status} reading the project config`); + } + return null; + } + + const body = (await response.json()) as { + signIn?: { hashConfig?: Partial & { algorithm?: string } }; + }; + const config = body.signIn?.hashConfig; + if (!config?.signerKey) return null; + + const hashConfig = { + signerKey: config.signerKey, + // Firebase omits an empty separator. + saltSeparator: config.saltSeparator ?? "", + rounds: Number(config.rounds ?? 8), + memoryCost: Number(config.memoryCost ?? 14), + }; + // Every digest the import builds carries these, so a set Clerk would + // refuse is left out rather than written: the import then asks for them. + const problem = firebaseHashConfigProblem({ + base64_signer_key: hashConfig.signerKey, + base64_salt_separator: hashConfig.saltSeparator, + rounds: hashConfig.rounds, + mem_cost: hashConfig.memoryCost, + }); + if (problem) { + log.warn( + `The project's password hash parameters won't work in Clerk (${problem}), so the export carries none. ` + + "Pass the right ones to the import with --firebase-*.", + ); + return null; + } + return hashConfig; + } catch (error) { + log.debug(`firebase: could not read the project config: ${String(error)}`); + return null; + } +} + +/** + * Keeps the fields the `firebase` transformer maps from. + * + * A Firebase user also carries provider records, custom claims and sign-in + * timestamps that would bloat the export and mean nothing to the import. + */ +export function mapFirebaseUserToExport(user: FirebaseUser): Record { + const exported: Record = {}; + + for (const field of ["localId", "email", "displayName", "phoneNumber", "createdAt"] as const) { + if (user[field]) exported[field] = user[field]; + } + // Meaningful when false, so copied on presence rather than truthiness. + if (user.emailVerified !== undefined) exported.emailVerified = user.emailVerified; + // Only when true: every active user would otherwise carry a `false`. + if (user.disabled === true) exported.disabled = true; + + // Both halves or neither: a digest without its salt cannot be verified. A + // redacted hash is no hash at all. + if (user.passwordHash && user.passwordHash !== REDACTED_HASH && user.salt) { + exported.passwordHash = user.passwordHash; + exported.salt = user.salt; + } + + return exported; +} + +/** + * What Firebase sends as `passwordHash` when the caller may not read hashes: + * base64 for "REDACTED". `firebase-admin` treats it as no hash. + */ +export const REDACTED_HASH = "UkVEQUNURUQ="; + +function hasPasswordProvider(user: FirebaseUser): boolean { + const providers = user.providerUserInfo; + return ( + Array.isArray(providers) && + providers.some((provider) => (provider as { providerId?: unknown })?.providerId === "password") + ); +} + +export function buildFirebaseExport( + users: FirebaseUser[], + record: (line: UserLine) => void = () => {}, +) { + const exported: Record[] = []; + const counts = { email: 0, verified: 0, password: 0, name: 0, phone: 0 }; + // Password users Firebase returned no hash for: it only returns digests it + // made itself (scrypt), and an empty string for users uploaded with bcrypt, + // HMAC or any other hasher. + let unreadablePasswords = 0; + // Hashes Firebase redacted because the caller may not read them. + let redactedPasswords = 0; + + for (const user of users) { + const userId = String(user.localId ?? ""); + try { + const mapped = mapFirebaseUserToExport(user); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.emailVerified) counts.verified++; + if (mapped.passwordHash) counts.password++; + else if (user.passwordHash === REDACTED_HASH) redactedPasswords++; + else if (hasPasswordProvider(user)) unreadablePasswords++; + if (mapped.displayName) counts.name++; + if (mapped.phoneNumber) counts.phone++; + + record({ sourceId: userId, status: "exported" }); + } catch (error) { + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); + } + } + + return { + users: exported, + unreadablePasswords, + redactedPasswords, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a verified email", count: counts.verified }, + { label: "have a password hash", count: counts.password }, + { label: "have a display name", count: counts.name }, + { label: "have a phone number", count: counts.phone }, + ], + }; +} + +/** + * What the reader needs to know about the hash parameters. + * + * When they were read, they are already in the export file, so the import + * needs nothing extra. When they were not, the import has to be given them. + */ +export function formatHashConfigGuidance( + config: HashConfig | null, + passwordCount: number, +): string[] { + if (passwordCount === 0) { + return [dim("No password hashes in this export, so no hash parameters are needed.")]; + } + + if (!config) { + return [ + bold("Password hash parameters"), + "This export carries password hashes, which Clerk can only verify with the project's", + "scrypt parameters. Find them in the Firebase console under", + "Authentication → Users → (⋮) → Password hash parameters, then pass them to the import:", + dim( + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + ), + ]; + } + + return [ + bold("Password hash parameters"), + dim("Read from the project and saved in the export file, so the import needs nothing more."), + ]; +} + +/** The hash parameters in the shape the import reads them. */ +export function toFirebaseHashConfig(config: HashConfig): FirebaseHashConfig { + return { + base64_signer_key: config.signerKey, + base64_salt_separator: config.saltSeparator, + rounds: config.rounds, + mem_cost: config.memoryCost, + }; +} + +export async function exportFirebase(options: ExportFirebaseOptions): Promise { + // Read and validate before anything reaches the network, so a wrong file + // fails in a second rather than after an auth round-trip. + const resolved = await resolveServiceAccount(options); + + await withGutter("Exporting users from Firebase", async () => { + // The emulator variable is Firebase's own, so it is honoured, but named: + // an export from it is not the production project's users. + const emulator = process.env.FIREBASE_AUTH_EMULATOR_HOST; + if (!options.json) { + printTarget({ platform: emulator ? `firebase (emulator at ${emulator})` : "firebase" }); + } + // Only Google can say whether a well-formed key is still a valid one, so a + // revoked or deleted key fails here and is asked for again. + const { value: token, input: account } = await withInputRetry( + resolved, + promptServiceAccount, + async (candidate) => { + log.info(`Exporting from the ${candidate.project_id} project.`); + return withSpinner("Authenticating with Google...", async () => + fetchAccessToken(candidate), + ); + }, + options, + ); + + const users = await withSpinner("Fetching users from Firebase...", async (spinner) => + fetchAllFirebaseUsers({ account, token, spinner }), + ); + + const run = await startExportRun(options, { platform: "firebase" }); + const { + users: exported, + coverage, + unreadablePasswords, + redactedPasswords, + } = buildFirebaseExport(users, run.append); + + // Read before the file is written, so the envelope carries them and the + // import needs no --firebase-* flags. + const passwordCount = coverage.find((entry) => entry.label.includes("password"))?.count ?? 0; + const hashConfig = passwordCount > 0 ? await fetchHashConfig(account, token) : null; + + finishExport({ + run, + options, + users: exported, + coverage, + ...(hashConfig ? { firebase: toFirebaseHashConfig(hashConfig) } : {}), + }); + + if (!options.json) { + log.blank(); + for (const line of formatHashConfigGuidance(hashConfig, passwordCount)) log.info(line); + } + + if (redactedPasswords > 0) { + log.warn( + `${redactedPasswords} user${redactedPasswords === 1 ? "'s" : "s'"} password hash came back redacted: ` + + "the service account can't read hashes. Grant it `firebaseauth.configs.getHashConfig` and export again, " + + "or those users are imported without a password and need to reset it.", + ); + } + + if (unreadablePasswords > 0) { + log.warn( + `${unreadablePasswords} user${unreadablePasswords === 1 ? " has" : "s have"} a password Firebase did not return. ` + + "Firebase only returns hashes it made itself; users imported into Firebase with bcrypt, HMAC or another hasher come back without one. " + + "They will be imported without a password and need to reset it.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/index.ts b/packages/cli-core/src/commands/migrate/export/index.ts new file mode 100644 index 000000000..da6190120 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/index.ts @@ -0,0 +1,231 @@ +import type { Command } from "@commander-js/extra-typings"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { select } from "../../../lib/listage.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; +import type { DbExportOptions } from "./db-options.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "../lib/run-store.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +/** + * Bare `clerk migrate export` — pick a platform, then run its export. + * + * The picker is built from the registry, so a new platform appears without a + * second place to update. Whatever the chosen platform needs beyond the + * platform name, it prompts for itself. + */ +export async function exportPicker(options: Record = {}): Promise { + if (options.json || isAgent() || !isHuman()) { + throwUsageError( + `\`clerk migrate export\` needs a platform and cannot prompt here. Name one: ${exportPlatformKeys().join(", ")}.`, + undefined, + undefined, + exportPlatforms.map((entry) => ({ + command: `clerk migrate export ${entry.key}`, + description: entry.description, + })), + ); + } + + const platform = await select({ + message: "Which platform are you exporting from?", + choices: exportPlatforms.map((entry) => ({ + name: entry.label, + value: entry.key, + description: entry.description, + })), + }); + + const entry = getExportPlatform(platform); + // Unreachable via the picker; a guard so a registry edit cannot silently + // produce a choice with nothing behind it. + if (!entry) throwUsageError(`Unknown export platform "${platform}".`); + + await entry.run(options); +} + +const handlers = { + picker: exportPicker, + clerk: exportClerk, + auth0: exportAuth0, + supabase: exportSupabase, + authjs: exportAuthJs, + betterauth: exportBetterAuth, + firebase: exportFirebase, + workos: exportWorkOs, +}; + +/** The three platforms that read a database, which share `--db-url`. */ +const DB_PLATFORMS = [ + { + key: "supabase", + summary: "Export users from a Supabase Postgres database", + envVar: "SUPABASE_DB_URL", + example: "postgres://postgres:password@db.xxx.supabase.co:5432/postgres", + }, + { + key: "authjs", + summary: "Export users from an Auth.js database", + envVar: "AUTHJS_DB_URL", + example: "mysql://user:password@127.0.0.1:3306/authjs", + }, + { + key: "betterauth", + summary: "Export users from a Better Auth database", + envVar: "BETTERAUTH_DB_URL", + example: "./db.sqlite", + }, +] as const; + +/** Registers `export [platform]` under the `migrate` group. */ +export function registerMigrateExport(migrateCommand: Command<[], Record>): void { + const exportCommand = migrateCommand + .command("export") + .description("Export users from a source platform, ready for `clerk migrate import`") + .setExamples([ + { command: "clerk migrate export", description: "Pick a platform interactively" }, + { + command: "clerk migrate export clerk", + description: "Export from a Clerk instance into a new run", + }, + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export from an Auth0 tenant", + }, + ]) + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .action(async (_opts, cmd) => + handlers.picker(cmd.optsWithGlobals() as Record), + ); + + exportCommand + .command("clerk") + .description("Export users from a Clerk instance") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .setExamples([ + { + command: "clerk migrate export clerk", + description: "Prompts for the source instance and where to save the file", + }, + { + command: "clerk migrate export clerk --secret-key sk_live_… --output prod-users.json", + description: "Name the source instance outright, skipping the picker", + }, + ]) + .action(async (_opts, cmd) => + handlers.clerk(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("auth0") + .description("Export users from an Auth0 tenant") + .option("--domain ", "Auth0 tenant domain, e.g. my-tenant.us.auth0.com") + .option("--client-id ", "Machine-to-machine application client ID") + .option("--client-secret ", "Machine-to-machine application client secret") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .setExamples([ + { + command: + "clerk migrate export auth0 --domain my-tenant.us.auth0.com --client-id … --client-secret …", + description: "Export with explicit credentials", + }, + { + command: "clerk migrate export auth0", + description: "Read AUTH0_DOMAIN, AUTH0_CLIENT_ID and AUTH0_CLIENT_SECRET, or prompt", + }, + ]) + .action(async (_opts, cmd) => + handlers.auth0(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("firebase") + .description("Export users from a Firebase project") + .option("--service-account ", "Path to a service account key JSON file") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .setExamples([ + { + command: "clerk migrate export firebase --service-account ./service-account.json", + description: "Export using a downloaded service account key", + }, + ]) + .action(async (_opts, cmd) => + handlers.firebase(cmd.optsWithGlobals() as Parameters[0]), + ); + + exportCommand + .command("workos") + .description("Export users from a WorkOS tenant") + .option("--api-key ", "WorkOS secret API key, the one starting `sk_`") + .option( + "--with-identities", + "Also record each user's OAuth providers — one extra request per user", + ) + .option("--no-with-identities", "Skip the OAuth provider fan-out without being asked") + .option("-o, --output ", "Write the export here instead of the run folder") + .option( + "-y, --yes", + "Do not prompt: fail on a rejected credential, and assume --with-identities", + ) + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .setExamples([ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + { + command: "clerk migrate export workos", + description: "Read WORKOS_API_KEY, or prompt", + }, + ]) + .action(async (_opts, cmd) => + handlers.workos(cmd.optsWithGlobals() as Parameters[0]), + ); + + // All three take exactly one connection string, so they are registered from + // a table rather than three near-identical blocks. + for (const platform of DB_PLATFORMS) { + exportCommand + .command(platform.key) + .description(platform.summary) + .option("--db-url ", "Postgres, MySQL, libsql/Turso or SQLite connection string") + .option("-o, --output ", "Write the export here instead of the run folder") + .option("-y, --yes", "Do not prompt: fail on a rejected credential") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .option("--json", "Print the result as JSON; never prompts") + .setExamples([ + { + command: `clerk migrate export ${platform.key} --db-url "${platform.example}"`, + description: "Export from an explicit database", + }, + { + command: `clerk migrate export ${platform.key}`, + description: `Read ${platform.envVar}, or prompt`, + }, + ]) + .action(async (_opts, cmd) => + handlers[platform.key](cmd.optsWithGlobals() as DbExportOptions), + ); + } +} diff --git a/packages/cli-core/src/commands/migrate/export/registry.test.ts b/packages/cli-core/src/commands/migrate/export/registry.test.ts new file mode 100644 index 000000000..fbdb8ec7b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; +import { sourceKeys } from "../sources/registry.ts"; +import { exportPlatformKeys, exportPlatforms, getExportPlatform } from "./registry.ts"; + +describe("export registry", () => { + test("registers every source platform", () => { + expect(exportPlatformKeys()).toEqual([ + "clerk", + "auth0", + "supabase", + "authjs", + "firebase", + "betterauth", + "workos", + ]); + }); + + test.each([...exportPlatforms])("$key carries a label and description", (entry) => { + expect(entry.label.length).toBeGreaterThan(0); + expect(entry.description.length).toBeGreaterThan(0); + }); + + // The picker, the docs and the "what next" line all read this, so a typo + // would send someone to a transformer that does not exist. + test.each([...exportPlatforms])("$key names a real transformer", (entry) => { + expect(sourceKeys()).toContain(entry.sourceKey); + }); + + test.each([...exportPlatforms])("$key has something to run", (entry) => { + expect(typeof entry.run).toBe("function"); + }); + + test("looks a platform up by key", () => { + expect(getExportPlatform("auth0")?.label).toBe("Auth0"); + }); + + test("returns nothing for a platform that is not registered", () => { + expect(getExportPlatform("okta")).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/registry.ts b/packages/cli-core/src/commands/migrate/export/registry.ts new file mode 100644 index 000000000..385953b20 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/registry.ts @@ -0,0 +1,88 @@ +/** + * Export registry. + * + * The picker behind a bare `clerk migrate export` is built from this array, so + * adding a platform is one file plus one entry — the same shape the source + * registry uses. + * + * `run` takes no arguments on purpose: each platform resolves its own flags, + * environment variables and prompts, because what Auth0 needs (a tenant domain + * and M2M credentials) has nothing in common with what a database export needs. + */ + +import { exportAuth0 } from "./auth0.ts"; +import { exportAuthJs } from "./authjs.ts"; +import { exportBetterAuth } from "./betterauth.ts"; +import { exportClerk } from "./clerk.ts"; +import { exportFirebase } from "./firebase.ts"; +import { exportSupabase } from "./supabase.ts"; +import { exportWorkOs } from "./workos.ts"; + +export type ExportRegistryEntry = { + key: string; + label: string; + description: string; + /** Which source reads the file this export writes. */ + sourceKey: string; + run: (options: Record) => Promise; +}; + +export const exportPlatforms: ExportRegistryEntry[] = [ + { + key: "clerk", + label: "Clerk", + description: "Another Clerk instance, e.g. development → production", + sourceKey: "clerk", + run: async (options) => exportClerk(options), + }, + { + key: "auth0", + label: "Auth0", + description: "An Auth0 tenant, via the Management API", + sourceKey: "auth0", + run: async (options) => exportAuth0(options), + }, + { + key: "supabase", + label: "Supabase", + description: "A Supabase Postgres database — includes password hashes", + sourceKey: "supabase", + run: async (options) => exportSupabase(options), + }, + { + key: "authjs", + label: "Auth.js (NextAuth)", + description: "An Auth.js database — Postgres, MySQL or SQLite", + sourceKey: "authjs", + run: async (options) => exportAuthJs(options), + }, + { + key: "firebase", + label: "Firebase", + description: "A Firebase project, via Identity Toolkit", + sourceKey: "firebase", + run: async (options) => exportFirebase(options), + }, + { + key: "betterauth", + label: "Better Auth", + description: "A Better Auth database — plugin columns detected automatically", + sourceKey: "betterauth", + run: async (options) => exportBetterAuth(options), + }, + { + key: "workos", + label: "WorkOS", + description: "A WorkOS tenant, via the User Management API — no password hashes", + sourceKey: "workos", + run: async (options) => exportWorkOs(options), + }, +]; + +export function exportPlatformKeys(): string[] { + return exportPlatforms.map((entry) => entry.key); +} + +export function getExportPlatform(key: string): ExportRegistryEntry | undefined { + return exportPlatforms.find((entry) => entry.key === key); +} diff --git a/packages/cli-core/src/commands/migrate/export/shared.test.ts b/packages/cli-core/src/commands/migrate/export/shared.test.ts new file mode 100644 index 000000000..09ac64b89 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.test.ts @@ -0,0 +1,161 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { readEnvelope } from "../lib/export-file.ts"; +import { latestUserLines, readRun } from "../lib/run-store.ts"; +import { finishExport, formatImportCommand, startExportRun } from "./shared.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; +let originalCwd: string; + +beforeEach(() => { + originalCwd = process.cwd(); + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-export-shared-"))); + process.chdir(runsDir); +}); + +afterEach(() => { + process.chdir(originalCwd); + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +const users = [{ id: "u1", email: "a@x.dev" }]; +const coverage = [{ label: "have an email address", count: 1 }]; + +describe("finishExport", () => { + test("writes the envelope into the run folder by default", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + run.append({ sourceId: "u1", status: "exported" }); + + const { record, outputPath } = finishExport({ run, options: {}, users, coverage }); + + expect(outputPath).toBe(path.join(runsDir, record.id, "export.json")); + expect(readEnvelope(outputPath)).toMatchObject({ + clerkMigrate: 1, + source: "supabase", + runId: record.id, + users, + }); + expect(readRun(runsDir, record.id)).toMatchObject({ + kind: "export", + status: "complete", + file: { path: outputPath }, + }); + expect(latestUserLines(runsDir, record.id).get("u1")?.status).toBe("exported"); + }); + + // The file holds password hashes and PII, so no other local account reads it. + test("writes the file owner-only, in an owner-only run folder", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + + const { outputPath } = finishExport({ run, options: {}, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + expect(fs.statSync(path.dirname(outputPath)).mode & 0o777).toBe(0o700); + }); + + test("tightens an existing --output file to owner-only", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "existing.json"); + fs.writeFileSync(output, "", { mode: 0o644 }); + + const { outputPath } = finishExport({ run, options: { output }, users, coverage }); + + expect(fs.statSync(outputPath).mode & 0o777).toBe(0o600); + }); + + // Written to a new file and renamed into place: a failed write leaves an + // existing export as it was, and no half-written file behind. + test("a failed write keeps an existing --output whole", async () => { + const run = await startExportRun({ runsDir }, { platform: "supabase" }); + const output = path.join(runsDir, "existing-before.json"); + fs.writeFileSync(output, "old", { mode: 0o600 }); + const write = fs.writeFileSync; + const spy = spyOn(fs, "writeFileSync").mockImplementation((( + target: unknown, + ...rest: unknown[] + ) => { + if (String(target).endsWith(".tmp")) { + (write as (...args: unknown[]) => void)(target, "{ half", ...rest.slice(1)); + throw new Error("ENOSPC: no space left on device"); + } + return (write as (...args: unknown[]) => void)(target, ...rest); + }) as typeof fs.writeFileSync); + try { + expect(() => finishExport({ run, options: { output }, users, coverage })).toThrow(/ENOSPC/); + } finally { + spy.mockRestore(); + } + + expect(fs.readFileSync(output, "utf-8")).toBe("old"); + expect(fs.readdirSync(runsDir).filter((name) => name.endsWith(".tmp"))).toEqual([]); + }); + + test("--output writes somewhere else, and the run still records where", async () => { + const run = await startExportRun({ runsDir }, { platform: "auth0" }); + + const { record, outputPath } = finishExport({ + run, + options: { output: "mine/users.json" }, + users, + coverage, + }); + + expect(outputPath).toBe(path.join(runsDir, "mine", "users.json")); + expect(readRun(runsDir, record.id)?.file?.path).toBe(outputPath); + }); + + test("carries Firebase's hash parameters to the import", async () => { + const run = await startExportRun({ runsDir }, { platform: "firebase" }); + const firebase = { + base64_signer_key: "k", + base64_salt_separator: "s", + rounds: 8, + mem_cost: 14, + }; + + const { outputPath } = finishExport({ run, options: {}, users, coverage, firebase }); + + expect(readEnvelope(outputPath)?.firebase).toEqual(firebase); + }); + + test("prints the import command by run ID", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); + + const { record } = finishExport({ run, options: {}, users, coverage }); + + expect(captured.err).toContain(`clerk migrate import ${record.id}`); + }); + + test("--json returns the result on stdout instead", async () => { + const run = await startExportRun({ runsDir }, { platform: "clerk" }); + + const { record, outputPath } = finishExport({ run, options: { json: true }, users, coverage }); + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { id: record.id, kind: "export" }, + output: outputPath, + users: 1, + next: `clerk migrate import ${record.id}`, + }); + expect(captured.err).not.toContain("Field coverage"); + }); +}); + +describe("formatImportCommand", () => { + const render = () => Bun.stripANSI(formatImportCommand("20260929-141502-a1b2").join("\n")); + + test("names the export run, which carries its own source", () => { + expect(render()).toContain("clerk migrate import 20260929-141502-a1b2"); + }); + + // The instance comes from the resolved key, so there is no flag whose + // absence means development — the note says what actually decides. + test("says how to reach production", () => { + expect(render()).toContain("--instance prod"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/shared.ts b/packages/cli-core/src/commands/migrate/export/shared.ts new file mode 100644 index 000000000..d329a6d25 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/shared.ts @@ -0,0 +1,209 @@ +/** + * Shared plumbing for the export modules: the run that records each user, + * where the file lands, and what the user is told about it. + * + * Every export is a run. The file lands in that run's folder as + * `export.json` unless `--output` names somewhere else, and `--output` + * resolves against the **current working directory**, the way every other + * path flag in this CLI does. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { dim, green, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { ENVELOPE_VERSION, type ExportEnvelope } from "../lib/export-file.ts"; +import { ACCOUNT_LINKING_NOTE } from "../sources/registry.ts"; +import { + resolveRunsDir, + sha256File, + startRun, + type Run, + type RunRecord, + type RunTarget, +} from "../lib/run-store.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** What every export command takes on top of its own credentials. */ +export type ExportCommonOptions = { + /** Where to write the file, instead of the run folder. */ + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + json?: boolean; +}; + +/** + * Starts the export run that records each user as it is exported. + * + * Started once the users are in hand, so a rejected credential leaves no run + * behind. + */ +export async function startExportRun( + options: ExportCommonOptions, + target: RunTarget, +): Promise { + const runsDir = await resolveRunsDir(options.runsDir, { write: true }); + return startRun(runsDir, { kind: "export", target, source: target.platform }); +} + +/** Where the file lands: `--output`, or `export.json` in the run folder. */ +export function exportPath(run: Run, output: string | undefined): string { + return output ? path.resolve(process.cwd(), output) : path.join(run.dir, "export.json"); +} + +/** + * Writes the envelope, creating any missing parent directories. + * + * @returns The absolute path written. + */ +export function writeExportFile(file: string, envelope: ExportEnvelope): string { + // Password hashes, PII and a Firebase signer key: owner-only. + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + // Written whole to a new owner-only file, then renamed over the output: an + // existing `--output` keeps its old contents if the write fails, and the + // data is never at that file's own mode (`mode` applies only on create; + // the rename keeps the new file's). + const temp = `${file}.${process.pid}.tmp`; + try { + fs.writeFileSync(temp, JSON.stringify(envelope, null, 2), { mode: 0o600, flag: "wx" }); + fs.renameSync(temp, file); + } catch (error) { + fs.rmSync(temp, { force: true }); + throw error; + } + return file; +} + +export type CoverageField = { label: string; count: number }; + +/** + * How complete an export is, per field. + * + * ✓ every user, ! some, dim ✗ none. The point is to see *before* importing + * that, say, only 3 of 400 users have a password — which changes what the + * migration means. + */ +export function formatFieldCoverage(fields: CoverageField[], total: number): string[] { + return fields.map(({ label, count }) => { + const icon = count === total ? green("✓") : count > 0 ? yellow("!") : dim("✗"); + return ` ${icon} ${dim(`${count}/${total} ${label}`)}`; + }); +} + +/** + * An extra block printed under the coverage table. + * + * For a breakdown that is not "how many users have this field" — WorkOS's + * OAuth providers, where one user can appear in two rows and the denominator + * is not the user count. Folding that into the coverage table would put rows + * of two different kinds under one heading. + */ +export type ExportSection = { title: string; rows: string[] }; + +/** + * The import command for an export run, and what it will target. + * + * One command rather than a development and a production variant, because + * there is no flag whose absence means "development": the key decides, through + * `--secret-key`, `--app`, `CLERK_SECRET_KEY`, the keyless project and the + * linked profile in that order. A line labelled "development" would be wrong + * for anyone holding `CLERK_SECRET_KEY=sk_live_…`, which is the reader who can + * least afford it. So the note names what picks the instance instead. + */ +export function formatImportCommand(runId: string): string[] { + return [ + "Import them with:", + dim(` clerk migrate import ${runId}`), + "", + dim(" Imports into whichever instance the resolved secret key belongs to."), + dim(" For production, add `--instance prod` or use a production secret key."), + ]; +} + +export type FinishExportInput = { + run: Run; + options: ExportCommonOptions; + users: Record[]; + coverage: CoverageField[]; + /** Extra blocks, printed under the coverage table in order. */ + sections?: ExportSection[]; + /** Firebase's hash parameters, carried to the import in the envelope. */ + firebase?: FirebaseHashConfig; + /** The platform stopped short of every user; `--json` says so. */ + truncated?: boolean; +}; + +export type FinishedExport = { record: RunRecord; outputPath: string }; + +/** + * Writes the envelope, finishes the run, and reports it. + * + * The import command prints through `log.info` rather than the gutter's + * next-steps outro, which is human-only: this is the one line that says what + * to do with the file, and an agent that cannot see it has to guess. `--json` + * returns the same facts on stdout instead. + */ +export function finishExport(input: FinishExportInput): FinishedExport { + const { run, options, users, coverage } = input; + const platform = run.record.target.platform ?? run.record.source ?? ""; + const outputPath = writeExportFile(exportPath(run, options.output), { + clerkMigrate: ENVELOPE_VERSION, + source: run.record.source ?? platform, + exportedAt: new Date().toISOString(), + runId: run.record.id, + ...(input.firebase ? { firebase: input.firebase } : {}), + users, + }); + run.update({ file: { path: outputPath, sha256: sha256File(outputPath) } }); + const record = run.finish(); + const next = `clerk migrate import ${record.id}`; + + if (options.json) { + log.data( + JSON.stringify( + { + target: record.target, + run: record, + output: outputPath, + users: users.length, + coverage, + ...(input.sections?.length ? { sections: input.sections } : {}), + ...(input.truncated ? { truncated: true } : {}), + next, + }, + null, + 2, + ), + ); + return { record, outputPath }; + } + + log.blank(); + if (users.length === 0) { + log.warn(`No users found to export. Wrote an empty file to ${outputPath}.`); + log.info(dim(`Run ${record.id}`)); + return { record, outputPath }; + } + + log.info("Field coverage"); + for (const line of formatFieldCoverage(coverage, users.length)) log.info(line); + + for (const section of input.sections ?? []) { + log.blank(); + log.info(section.title); + for (const row of section.rows) log.info(row); + } + + log.blank(); + log.success(`Exported ${users.length} user${users.length === 1 ? "" : "s"} to ${outputPath}`); + log.info(dim(`Run ${record.id}. See each user with \`clerk migrate runs ${record.id}\`.`)); + + log.blank(); + log.info(dim(ACCOUNT_LINKING_NOTE)); + + log.blank(); + for (const line of formatImportCommand(record.id)) log.info(line); + + return { record, outputPath }; +} diff --git a/packages/cli-core/src/commands/migrate/export/supabase.ts b/packages/cli-core/src/commands/migrate/export/supabase.ts new file mode 100644 index 000000000..321902519 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/supabase.ts @@ -0,0 +1,158 @@ +/** + * `clerk migrate export supabase` — read users straight out of `auth.users`. + * + * Ported from the standalone migration-tool's `src/export/supabase.ts`, on + * `Bun.sql` instead of `pg`. + * + * The database rather than the Admin API because **`encrypted_password` only + * exists here**. Supabase's API does not return password hashes, so an + * API-based export forces every user to reset their password; this one carries + * the bcrypt digests across. + */ + +import { throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { withGutter, withSpinner } from "../../../lib/spinner.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { detectDbType, type DbClient } from "../lib/db.ts"; +import { finishExport, startExportRun } from "./shared.ts"; +import { + resolveDbUrl, + withDbConnection, + type DbExportOptions, + type ResolveConfig, +} from "./db-options.ts"; + +/** + * Only the metadata's own `first_name` and `last_name` are pulled out as + * columns. A `display_name`, `full_name` or OAuth `name` stays in + * `raw_user_meta_data` for the Supabase source to split, which it does only + * when `first_name` is empty: coalescing them in here would keep "Jane Doe" + * as one first name, and let a display name outrank a real first name. + */ +const EXPORT_QUERY = ` + SELECT + id, + email, + email_confirmed_at, + encrypted_password, + phone, + phone_confirmed_at, + raw_user_meta_data->>'first_name' AS first_name, + raw_user_meta_data->>'last_name' AS last_name, + raw_user_meta_data, + raw_app_meta_data, + banned_until, + -- Through to_jsonb so an older auth.users without the column still reads. + to_jsonb(u)->>'deleted_at' AS deleted_at, + created_at + FROM auth.users u + ORDER BY created_at +`; + +type SupabaseRow = Record & { + id?: unknown; + email?: string | null; + encrypted_password?: string | null; + raw_app_meta_data?: unknown; +}; + +/** Serializes values the JSON export cannot carry as-is. */ +function normalizeRow(row: SupabaseRow): Record { + const normalized: Record = {}; + + for (const [key, value] of Object.entries(row)) { + if (value === null || value === undefined) continue; + // Postgres returns timestamps as Date objects; the transformer parses + // strings, and JSON.stringify would otherwise bury the format difference. + normalized[key] = value instanceof Date ? value.toISOString() : value; + } + + return normalized; +} + +/** + * Every `auth.users` row, at once. + * + * ponytail: the whole table in memory, then the whole envelope. The import + * reads the file whole too, so streaming this side alone would buy nothing; + * page both (keyset on `created_at, id`) if a project's users outgrow memory. + */ +export async function fetchSupabaseUsers(client: DbClient): Promise { + return client.query(EXPORT_QUERY); +} + +export function buildSupabaseExport( + rows: SupabaseRow[], + record: (line: UserLine) => void = () => {}, +) { + const users: Record[] = []; + const counts = { email: 0, emailConfirmed: 0, password: 0, phone: 0, firstName: 0, lastName: 0 }; + + for (const row of rows) { + const userId = String(row.id ?? ""); + try { + users.push(normalizeRow(row)); + + if (row.email) counts.email++; + if (row.email_confirmed_at) counts.emailConfirmed++; + if (row.encrypted_password) counts.password++; + if (row.phone) counts.phone++; + if (row.first_name) counts.firstName++; + if (row.last_name) counts.lastName++; + + record({ sourceId: userId, status: "exported" }); + } catch (error) { + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); + } + } + + return { + users, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a confirmed email", count: counts.emailConfirmed }, + { label: "have a password hash", count: counts.password }, + { label: "have a phone number", count: counts.phone }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + ], + }; +} + +const SUPABASE_DB = { + platform: "supabase", + envVar: "SUPABASE_DB_URL", + prompt: "Supabase Postgres connection string", + hint: "Dashboard → Connect → Session pooler. Direct connections need the IPv4 add-on.", +} as const satisfies ResolveConfig; + +export async function exportSupabase(options: DbExportOptions): Promise { + const dbUrl = await resolveDbUrl(options, SUPABASE_DB); + // The query reads `auth.users` with Postgres syntax: a MySQL or SQLite URL + // would connect, then fail with nothing exported. + if (detectDbType(dbUrl) !== "postgres") { + throwUsageError( + "Supabase's database is Postgres: pass a postgres:// or postgresql:// connection string. Nothing was exported.\n" + + `Find it under ${SUPABASE_DB.hint}`, + ); + } + + await withGutter("Exporting users from Supabase", async () => { + if (!options.json) printTarget({ platform: "supabase" }); + const rows = await withDbConnection(dbUrl, SUPABASE_DB, options, async (client) => + withSpinner("Reading auth.users...", async () => fetchSupabaseUsers(client)), + ); + + const run = await startExportRun(options, { platform: "supabase" }); + const { users, coverage } = buildSupabaseExport(rows, run.append); + finishExport({ run, options, users, coverage }); + + if (users.length > 0) { + log.info( + "Password hashes are included — this is why the export reads the database rather than the Admin API.", + ); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/export/workos.test.ts b/packages/cli-core/src/commands/migrate/export/workos.test.ts new file mode 100644 index 000000000..d4bd51ea7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.test.ts @@ -0,0 +1,537 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode } from "../../../mode.ts"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { UserLine } from "../lib/run-store.ts"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { setAssumeYes } from "../lib/assume-yes.ts"; +import { + buildIdentityReport, + buildWorkOsExport, + exportWorkOs, + fetchAllWorkOsIdentities, + fetchAllWorkOsUsers, + fetchWorkOsIdentities, + fetchWorkOsPage, + mapWorkOsUserToExport, + resolveWithIdentities, + resolveWorkOsApiKey, + type WorkOsIdentity, +} from "./workos.ts"; + +const captured = useCaptureLog(); + +const API_KEY = "sk_test"; + +/** Colour is on or off depending on the runner, so rows are compared bare. */ +const stripAnsi = (value: string): string => value.replace(/\u001b\[[0-9;]*m/g, ""); + +let workDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: string[]; + +beforeAll(() => { + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-expworkos-"))); + process.chdir(workDir); +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + // Tests that need a prompt set human mode themselves; without this a leaked + // "human" from an earlier test stops a later one on the destination prompt. + setMode("agent"); + requests = []; + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(workDir, "exports"), { recursive: true, force: true }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const workosUser = (i: number, overrides: Record = {}) => ({ + id: `user_0${i}`, + email: `a${i}@x.dev`, + email_verified: true, + first_name: `Given${i}`, + last_name: `Family${i}`, + ...overrides, +}); + +/** + * Stubs one users page per entry in `pages`, chaining the cursor, plus an + * identities response per user id in `identities`. + */ +function stubWorkOs( + pages: Record[][], + identities: Record = {}, +) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = input.toString(); + requests.push(url); + + const identityMatch = /\/users\/([^/]+)\/identities/.exec(url); + if (identityMatch) { + const entry = identities[identityMatch[1] as string]; + if (entry === "fail") return new Response("nope", { status: 500 }); + return Response.json(entry ?? []); + } + + const after = new URL(url).searchParams.get("after"); + const index = after ? Number(after.replace("cursor", "")) : 0; + const isLast = index >= pages.length - 1; + return Response.json({ + data: pages[index] ?? [], + list_metadata: { after: isLast ? null : `cursor${index + 1}` }, + }); + }) as unknown as typeof fetch; +} + +describe("resolveWorkOsApiKey", () => { + test("prefers the flag", async () => { + expect(await resolveWorkOsApiKey({ apiKey: "sk_flag" }, {})).toBe("sk_flag"); + }); + + test("falls back to the environment", async () => { + expect(await resolveWorkOsApiKey({}, { WORKOS_API_KEY: "sk_env" })).toBe("sk_env"); + }); + + // Tests run non-TTY, the same signal an agent gives. + test("names the flag and the variable when neither supplied one", async () => { + await expect(resolveWorkOsApiKey({}, {})).rejects.toThrow( + /Missing: --api-key \(or WORKOS_API_KEY\)\./, + ); + }); + + test("does not prompt under --json, even with a human at the TTY", async () => { + const originalMode = getMode(); + setMode("human"); + try { + await expect(resolveWorkOsApiKey({ json: true }, {})).rejects.toThrow(/cannot prompt here/); + } finally { + setMode(originalMode); + } + }); +}); + +describe("fetchWorkOsPage", () => { + test("asks for the documented page size", async () => { + stubWorkOs([[]]); + await fetchWorkOsPage(API_KEY); + expect(requests[0]).toContain("limit=100"); + }); + + test("explains a rejection instead of surfacing a raw status", async () => { + globalThis.fetch = (async () => + Response.json({ message: "Unauthorized" }, { status: 401 })) as unknown as typeof fetch; + + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow( + /WorkOS returned 401 listing users: Unauthorized/, + ); + }); + + // The usual cause is a publishable key, or a key from the other environment. + test("points at the key itself", async () => { + globalThis.fetch = (async () => new Response("{}", { status: 401 })) as unknown as typeof fetch; + await expect(fetchWorkOsPage(API_KEY)).rejects.toThrow(/secret key/); + }); +}); + +describe("fetchAllWorkOsUsers", () => { + test("follows the cursor until it comes back null", async () => { + stubWorkOs([ + Array.from({ length: 100 }, (_, i) => workosUser(i)), + Array.from({ length: 4 }, (_, i) => workosUser(100 + i)), + ]); + + const all = await fetchAllWorkOsUsers({ apiKey: API_KEY }); + + expect(all).toHaveLength(104); + expect(requests).toHaveLength(2); + expect(requests[1]).toContain("after=cursor1"); + }); + + // Cursor pagination has no record ceiling, so a full page that happens to be + // the last one must not read as "there is more". + test("stops on a full final page", async () => { + stubWorkOs([Array.from({ length: 100 }, (_, i) => workosUser(i))]); + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY })).toHaveLength(100); + expect(requests).toHaveLength(1); + }); + + test("stops when WorkOS hands back the cursor it was given", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json({ data: [workosUser(0)], list_metadata: { after: "stuck" } }); + }) as unknown as typeof fetch; + + await expect(fetchAllWorkOsUsers({ apiKey: API_KEY })).rejects.toThrow( + /same pagination cursor twice \(stuck\)/, + ); + expect(requests).toHaveLength(2); + }); + + test("reuses a page already fetched rather than asking twice", async () => { + stubWorkOs([[workosUser(0)]]); + const firstPage = await fetchWorkOsPage(API_KEY); + requests = []; + + expect(await fetchAllWorkOsUsers({ apiKey: API_KEY, firstPage })).toHaveLength(1); + expect(requests).toHaveLength(0); + }); +}); + +describe("resolveWithIdentities", () => { + test("is on when the flag asked for it", async () => { + expect(await resolveWithIdentities({ withIdentities: true }, 10)).toBe(true); + }); + + // The fan-out is one request per user and nothing it returns can be + // imported, so it is never the default. + test("is off without the flag when there is nobody to ask", async () => { + expect(await resolveWithIdentities({}, 10)).toBe(false); + }); + + test("is off when `--no-with-identities` said so, even under -y", async () => { + setAssumeYes(true); + try { + expect(await resolveWithIdentities({ withIdentities: false }, 10)).toBe(false); + } finally { + setAssumeYes(false); + } + }); + + // `-y` is "answer the prompts yes", and the prompt is "also fetch providers?". + test("is on under -y, which answers the question rather than asking it", async () => { + const originalMode = getMode(); + setMode("human"); + setAssumeYes(true); + try { + expect(await resolveWithIdentities({}, 10)).toBe(true); + } finally { + setAssumeYes(false); + setMode(originalMode); + } + }); + + test("is off under --json, even with a human at the TTY", async () => { + const originalMode = getMode(); + setMode("human"); + try { + expect(await resolveWithIdentities({ json: true }, 10)).toBe(false); + } finally { + setMode(originalMode); + } + }); + + test("does not ask when there are no users to ask about", async () => { + const originalMode = getMode(); + setMode("human"); + try { + expect(await resolveWithIdentities({}, 0)).toBe(false); + } finally { + setMode(originalMode); + } + }); +}); + +describe("fetchAllWorkOsIdentities", () => { + test("collects each user's providers", async () => { + stubWorkOs([[]], { + user_00: [{ provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }], + user_01: [], + }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.get("user_00")).toEqual([ + { provider: "GoogleOAuth", idp_id: "g1", type: "OAuth" }, + ]); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(0); + }); + + // "Lookup failed" and "has no providers" are different facts, and flattening + // the first into the second would put a wrong number in the report. + test("leaves a failed lookup absent rather than empty, and counts it", async () => { + stubWorkOs([[]], { user_00: "fail", user_01: [] }); + + const { identities, failed } = await fetchAllWorkOsIdentities({ + apiKey: API_KEY, + users: [workosUser(0), workosUser(1)], + }); + + expect(identities.has("user_00")).toBe(false); + expect(identities.get("user_01")).toEqual([]); + expect(failed).toBe(1); + }); +}); + +test("fetchWorkOsIdentities keeps the user ID inside its path segment", async () => { + stubWorkOs([[]]); + await fetchWorkOsIdentities(API_KEY, "user_01/../x?y#z"); + expect(requests[0]).toBe( + "https://api.workos.com/user_management/users/user_01%2F..%2Fx%3Fy%23z/identities", + ); +}); + +// A key revoked partway would otherwise mark every later user unreadable and +// let the export finish as if it had worked. +test("fetchAllWorkOsIdentities stops on a rejected key rather than counting it", async () => { + globalThis.fetch = (async (input: string | URL | Request) => { + requests.push(input.toString()); + return Response.json({ message: "Unauthorized" }, { status: 401 }); + }) as unknown as typeof fetch; + const users = Array.from({ length: 50 }, (_, i) => workosUser(i)); + + const error = (await fetchAllWorkOsIdentities({ apiKey: API_KEY, users }).catch( + (e: unknown) => e, + )) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests.length).toBeLessThan(users.length); +}); + +describe("buildIdentityReport", () => { + const rowsOf = (section: { rows: string[] }) => + section.rows.map((row) => stripAnsi(row).trimEnd()); + + test("ranks providers by use, and counts users with none", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1), workosUser(2), workosUser(3)], + new Map([ + ["user_00", [{ provider: "GoogleOAuth" }]], + ["user_01", [{ provider: "GoogleOAuth" }, { provider: "MicrosoftOAuth" }]], + ["user_02", []], + ["user_03", []], + ]), + 0, + ); + + expect(section.title).toBe("OAuth providers"); + expect(rowsOf(section)).toEqual([ + " GoogleOAuth 2 users", + " MicrosoftOAuth 1 user", + " no OAuth provider 2 users", + ]); + }); + + // Counting a failed lookup as "no provider" would understate social sign-in. + test("reports unreadable lookups on their own row, with the caveat", () => { + const section = buildIdentityReport( + [workosUser(0), workosUser(1)], + new Map([["user_01", []]]), + 1, + ); + + const rows = rowsOf(section); + expect(rows).toContain(" not readable 1 user"); + expect(rows).toContain(" no OAuth provider 1 user"); + expect(rows.at(-1)).toContain("no `identities` field in the export, rather than an empty one"); + }); + + test("says nothing about unreadable lookups when there were none", () => { + const section = buildIdentityReport([workosUser(0)], new Map([["user_00", []]]), 0); + expect(rowsOf(section)).toEqual([" no OAuth provider 1 user"]); + }); +}); + +describe("mapWorkOsUserToExport", () => { + test("keeps the fields the workos transformer maps from", () => { + expect(mapWorkOsUserToExport(workosUser(0, { created_at: "2025-01-01" }))).toEqual({ + id: "user_00", + email: "a0@x.dev", + first_name: "Given0", + last_name: "Family0", + created_at: "2025-01-01", + email_verified: true, + }); + }); + + // Dropping a false flag would import an unconfirmed address as verified. + test("keeps email_verified when it is false", () => { + expect(mapWorkOsUserToExport(workosUser(0, { email_verified: false })).email_verified).toBe( + false, + ); + }); + + test("drops tenant fields the import has no use for", () => { + const mapped = mapWorkOsUserToExport( + workosUser(0, { + locale: "en-GB", + profile_picture_url: "https://x.dev/a.png", + last_sign_in_at: "2026-01-01", + updated_at: "2026-01-01", + }), + ); + for (const noise of ["locale", "profile_picture_url", "last_sign_in_at", "updated_at"]) { + expect(noise in mapped).toBe(false); + } + }); + + test("keeps the tenant's own external_id", () => { + expect(mapWorkOsUserToExport(workosUser(0, { external_id: "cust_1" })).external_id).toBe( + "cust_1", + ); + }); + + test("omits empty metadata", () => { + expect("metadata" in mapWorkOsUserToExport(workosUser(0, { metadata: {} }))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0, { metadata: { plan: "pro" } })).metadata).toEqual({ + plan: "pro", + }); + }); + + test("carries identities only when they were fetched", () => { + expect("identities" in mapWorkOsUserToExport(workosUser(0))).toBe(false); + expect(mapWorkOsUserToExport(workosUser(0), [{ provider: "GoogleOAuth" }]).identities).toEqual([ + { provider: "GoogleOAuth" }, + ]); + }); +}); + +describe("buildWorkOsExport", () => { + test("counts coverage and records each user", () => { + const lines: UserLine[] = []; + const { users, coverage } = buildWorkOsExport( + [workosUser(0), workosUser(1, { first_name: undefined })], + (line) => lines.push(line), + ); + + expect(users).toHaveLength(2); + const byLabel = Object.fromEntries(coverage.map((c) => [c.label, c.count])); + expect(byLabel["have an email address"]).toBe(2); + expect(byLabel["have a first name"]).toBe(1); + + expect(lines.map((line) => line.status)).toEqual(["exported", "exported"]); + }); + + // Always shown, always zero: seeing it before the import is the point. + test("reports the password row even though it can only ever be zero", () => { + const { coverage } = buildWorkOsExport([workosUser(0)]); + expect(coverage.at(-1)).toEqual({ + label: "have a password (WorkOS returns none)", + count: 0, + }); + }); + + // Providers get their own block: a coverage row means "N of M users have + // this field", and a provider count can exceed M. + test("keeps providers out of the coverage table", () => { + const { coverage } = buildWorkOsExport( + [workosUser(0)], + undefined, + new Map([["user_00", [{ provider: "GoogleOAuth" }]]]), + ); + expect(coverage.some((row) => row.label.toLowerCase().includes("oauth"))).toBe(false); + }); +}); + +/** The envelope the one export run in this project wrote. */ +function onlyExportFile(): string { + const dir = path.join(workDir, ".clerk", "migrate"); + const entries = fs.readdirSync(dir); + expect(entries).toHaveLength(1); + return path.join(dir, entries[0] as string, "export.json"); +} + +/** The users inside that envelope. */ +function exportedUsers(): Record[] { + return ( + JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8")) as { users: Record[] } + ).users; +} + +describe("exportWorkOs", () => { + test("writes the default path and reports coverage", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY }); + expect(JSON.parse(fs.readFileSync(onlyExportFile(), "utf-8"))).toMatchObject({ + source: "workos", + }); + const written = exportedUsers(); + expect(written[0]?.id).toBe("user_00"); + expect(captured.err).toContain("Field coverage"); + }); + + test("names the command that consumes the file", async () => { + stubWorkOs([[workosUser(0)]]); + // The suggestion rides the gutter's Next steps block, which only renders + // in human mode. + const originalMode = getMode(); + setMode("human"); + try { + // --output answers the destination prompt, and --with-identities answers + // the providers question, so human mode stops on neither. + await exportWorkOs({ apiKey: API_KEY, output: "exports/mine.json", withIdentities: true }); + } finally { + setMode(originalMode); + } + expect(captured.err).toMatch(/clerk migrate import \d{8}-\d{6}-[0-9a-f]{4}/); + }); + + test("--output controls the destination", async () => { + stubWorkOs([[workosUser(0)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "tenant.json" }); + + expect(fs.existsSync(path.join(workDir, "tenant.json"))).toBe(true); + }); + + test("skips the per-user identity fan-out unless asked", async () => { + stubWorkOs([[workosUser(0), workosUser(1)]]); + + await exportWorkOs({ apiKey: API_KEY, output: "plain.json" }); + + expect(requests.filter((url) => url.includes("/identities"))).toHaveLength(0); + }); + + test("--with-identities records each user's providers in the file", async () => { + stubWorkOs([[workosUser(0)]], { user_00: [{ provider: "GoogleOAuth", idp_id: "g1" }] }); + + await exportWorkOs({ apiKey: API_KEY, output: "rich.json", withIdentities: true }); + + const written = ( + JSON.parse(fs.readFileSync(path.join(workDir, "rich.json"), "utf-8")) as { + users: Record[]; + } + ).users; + expect(written[0]?.identities).toEqual([{ provider: "GoogleOAuth", idp_id: "g1" }]); + }); + + // Finding this out after the import means nobody can sign in. + test("says plainly that no credentials are in the file", async () => { + stubWorkOs([[workosUser(0)]]); + await exportWorkOs({ apiKey: API_KEY, output: "warned.json" }); + expect(captured.err).toContain("does not return password hashes or TOTP secrets"); + }); +}); + +describe("fetchWorkOsIdentities", () => { + test("accepts the bare array the endpoint returns", async () => { + stubWorkOs([[]], { user_00: [{ provider: "GoogleOAuth" }] }); + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "GoogleOAuth" }]); + }); + + // A move to WorkOS's usual envelope must not read as "no providers". + test("accepts a { data } envelope too", async () => { + globalThis.fetch = (async () => + Response.json({ data: [{ provider: "AppleOAuth" }] })) as unknown as typeof fetch; + expect(await fetchWorkOsIdentities(API_KEY, "user_00")).toEqual([{ provider: "AppleOAuth" }]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/export/workos.ts b/packages/cli-core/src/commands/migrate/export/workos.ts new file mode 100644 index 000000000..572b6633e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/export/workos.ts @@ -0,0 +1,520 @@ +/** + * `clerk migrate export workos` — pull users out of a WorkOS tenant. + * + * Structurally the Auth0 case, and written against the same shape: two REST + * calls through `loggedFetch` rather than the `@workos-inc/node` SDK, so + * everything the command sends shows up under `--verbose`. See the header of + * `auth0.ts` and `.claude/rules/debug-logging.md`. + * + * **WorkOS is API-only, and no credential leaves it.** There is no + * bring-your-own-database option — apps mirror WorkOS users into their own + * store through webhooks, but that mirror is a derived copy holding no secrets, + * which is why this has no `--db-url` sibling. Password hashes are accepted on + * import and never returned; TOTP secrets come back on enrol only, never on + * list or get. So a WorkOS migration moves identities, and every password user + * signs in again by reset. The run says so rather than leaving it to be + * discovered when nobody can sign in. + */ + +import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { loggedFetch } from "../../../lib/fetch.ts"; +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { isCancelled } from "../../../lib/signals.ts"; +import { confirm, password as passwordPrompt } from "../../../lib/prompts.ts"; +import { withGutter, withSpinner, type SpinnerControls } from "../../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import type { UserLine } from "../lib/run-store.ts"; +import { printTarget } from "../lib/target.ts"; +import { isAssumeYes } from "../lib/assume-yes.ts"; +import { throwApiFailure, withInputRetry } from "../lib/input-retry.ts"; +import { createApiScheduler } from "../lib/scheduler.ts"; +import { finishExport, startExportRun, type ExportSection } from "./shared.ts"; + +const API_BASE = "https://api.workos.com/user_management"; + +/** WorkOS caps `limit` at 100. */ +const PAGE_SIZE = 100; + +/** + * Pacing for the per-user identity fan-out. + * + * WorkOS allows 6,000 requests a minute, so this is nowhere near the ceiling — + * it is here so a 50,000-user tenant does not open 50,000 sockets at once. + */ +const IDENTITY_CONCURRENCY = 10; +const IDENTITY_RATE_PER_SECOND = 20; + +/** + * How often a non-interactive run says where it has got to. + * + * `withSpinner` hands a no-op `update` to anything that is not a TTY, so an + * agent exporting 50,000 users with `--with-identities` would otherwise see + * nothing at all for the ten minutes the fan-out takes. These two print + * through `log.info` instead, which a non-TTY does get. + */ +const IDENTITY_PROGRESS_EVERY = 500; +const USER_PROGRESS_EVERY_PAGES = 10; + +const NO_PROVIDER_LABEL = "no OAuth provider"; +const NOT_READABLE_LABEL = "not readable"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +export type ExportWorkOsOptions = { + apiKey?: string; + /** Unset means "ask"; `--no-with-identities` sets it to false. */ + withIdentities?: boolean; + output?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; + /** Print the result as JSON on stdout; never prompts. */ + json?: boolean; +}; + +export type WorkOsUser = Record & { id?: string }; + +export type WorkOsIdentity = { idp_id?: string; type?: string; provider?: string }; + +/** + * Resolves the API key: flag, then environment, then a prompt. + * + * One value rather than Auth0's three, so there is no "name everything that is + * missing" pass — there is only ever the one thing. + * + * @throws CliError in agent mode when nothing supplied it. + */ +export async function resolveWorkOsApiKey( + options: ExportWorkOsOptions, + env: Record = process.env, +): Promise { + const resolved = options.apiKey ?? env.WORKOS_API_KEY; + + if (resolved) return resolved.trim(); + + if (options.json || isAgent() || !isHuman()) { + throwUsageError( + "`clerk migrate export workos` needs a WorkOS API key and cannot prompt here.\n" + + "Missing: --api-key (or WORKOS_API_KEY).", + DOCS_URL, + undefined, + [ + { + command: "clerk migrate export workos --api-key sk_…", + description: "Export with an explicit API key", + }, + ], + ); + } + + log.info( + "WorkOS needs a secret API key, the one starting `sk_`. Find it in the WorkOS dashboard under API Keys.", + ); + + return promptWorkOsApiKey(); +} + +export async function promptWorkOsApiKey(): Promise { + const key = await passwordPrompt({ + message: "WorkOS secret API key (sk_…)", + validate: (value) => (value?.trim() ? undefined : "An API key is required"), + }); + return key.trim(); +} + +/** + * Whatever WorkOS put in an error body, in one string. + * + * Read as text and parsed from that, rather than `response.json()` with a text + * fallback: the failed parse disturbs the stream, so the fallback could never + * actually run. A non-JSON body — a proxy's HTML error page — is the case worth + * surfacing verbatim. + */ +async function describeFailure(response: Response): Promise { + const raw = (await response.text().catch(() => "")).trim(); + if (!raw) return "no detail returned"; + + try { + const body = JSON.parse(raw) as { + message?: string; + error_description?: string; + error?: string; + }; + return body.message ?? body.error_description ?? body.error ?? raw; + } catch { + return raw; + } +} + +export type WorkOsPage = { users: WorkOsUser[]; after?: string }; + +/** + * Fetches one page of users. + * + * Cursor pagination, so unlike Auth0's offset endpoint there is no record + * ceiling to warn about — `after` runs to the end of the tenant. + */ +export async function fetchWorkOsPage(apiKey: string, after?: string): Promise { + const url = new URL(`${API_BASE}/users`); + url.searchParams.set("limit", String(PAGE_SIZE)); + url.searchParams.set("order", "asc"); + if (after) url.searchParams.set("after", after); + + const response = await loggedFetch(url, { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }); + + if (!response.ok) { + throwApiFailure( + response.status, + `WorkOS returned ${response.status} listing users: ${await describeFailure(response)}\n` + + "Check that the key is a secret key (`sk_…`) for the right environment, and that it has not been revoked.", + DOCS_URL, + ); + } + + const body = (await response.json()) as { + data?: WorkOsUser[]; + list_metadata?: { after?: string | null }; + }; + + return { users: body.data ?? [], after: body.list_metadata?.after ?? undefined }; +} + +/** + * Pages through the tenant's users. + * + * @param firstPage - A page already fetched, so the request that proved the API + * key is not sent twice. + */ +export async function fetchAllWorkOsUsers(options: { + apiKey: string; + firstPage?: WorkOsPage; + spinner?: SpinnerControls; +}): Promise { + let page = options.firstPage ?? (await fetchWorkOsPage(options.apiKey)); + const all = [...page.users]; + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + + // Counted in pages rather than users: a short page would knock a + // `users % N` check off its multiple and silence every later one. + for (let pages = 1; page.after; pages++) { + const cursor = page.after; + page = await fetchWorkOsPage(options.apiKey, cursor); + // A cursor that does not move would refetch the same page forever. + if (page.after === cursor) { + throw new CliError(`WorkOS returned the same pagination cursor twice (${cursor}).`); + } + all.push(...page.users); + options.spinner?.update(`Fetching users from WorkOS: ${all.length} so far...`); + if (!isHuman() && pages % USER_PROGRESS_EVERY_PAGES === 0) { + log.info(`Fetched ${all.length} users from WorkOS so far...`); + } + } + + return all; +} + +/** + * Whether to spend one request per user on OAuth providers. + * + * Off unless asked for, both times. WorkOS has no bulk identities endpoint, so + * this is the difference between ten requests and one per user — and nothing it + * returns can be imported, because `POST /v1/users` has no external-accounts + * field. It is a line in the coverage report, and a record kept in the file. + * + * Agent mode and `--json` get the flag's answer and no question: there is + * nobody to ask. + * `-y` answers the question the way a `yes` would, so `--no-with-identities` + * is the way to say no without being asked. + */ +export async function resolveWithIdentities( + options: ExportWorkOsOptions, + userCount: number, +): Promise { + if (options.withIdentities !== undefined) return options.withIdentities; + if (userCount === 0) return false; + if (isAssumeYes()) return true; + if (options.json || isAgent() || !isHuman()) return false; + + return confirm({ + message: `Also fetch each user's OAuth providers? That is ${userCount} extra request${userCount === 1 ? "" : "s"}, and the result is report-only — Clerk's import cannot take external accounts.`, + default: false, + }); +} + +/** Fetches one user's OAuth identities. */ +export async function fetchWorkOsIdentities( + apiKey: string, + userId: string, +): Promise { + const response = await loggedFetch( + new URL(`${API_BASE}/users/${encodeURIComponent(userId)}/identities`), + { + tag: "workos", + method: "GET", + headers: { Authorization: `Bearer ${apiKey}`, Accept: "application/json" }, + }, + ); + + if (!response.ok) { + throwApiFailure( + response.status, + `WorkOS returned ${response.status} listing identities for ${userId}: ${await describeFailure(response)}`, + DOCS_URL, + ); + } + + // The endpoint returns a bare array; the envelope is handled too so a future + // move to WorkOS's usual `{ data }` shape does not read as "no providers". + const body = (await response.json()) as WorkOsIdentity[] | { data?: WorkOsIdentity[] }; + return Array.isArray(body) ? body : (body.data ?? []); +} + +/** + * Fetches identities for every user. + * + * A user missing from the returned map is one whose lookup **failed**, which is + * not the same as one with no providers — so failures are counted and returned + * separately rather than flattened into an empty list. + * + * A Ctrl-C, or a key WorkOS stops accepting partway, is not a failed lookup: + * every later one would fail the same way. It stops the fan-out and is thrown. + */ +export async function fetchAllWorkOsIdentities(options: { + apiKey: string; + users: WorkOsUser[]; + spinner?: SpinnerControls; +}): Promise<{ identities: Map; failed: number }> { + const schedule = createApiScheduler(IDENTITY_CONCURRENCY, IDENTITY_RATE_PER_SECOND); + const total = options.users.length; + const identities = new Map(); + let failed = 0; + let done = 0; + let fatal: unknown; + + await Promise.all( + options.users.map(async (user) => + schedule(async () => { + if (fatal) return; + const userId = String(user.id ?? ""); + try { + if (userId) identities.set(userId, await fetchWorkOsIdentities(options.apiKey, userId)); + } catch (error) { + const stops = + isCancelled(error) || (error instanceof CliError && error.exitCode === EXIT_CODE.USAGE); + if (stops) { + fatal ??= error; + return; + } + failed++; + } + done++; + options.spinner?.update(`Fetching OAuth providers: ${done}/${total}...`); + if (!isHuman() && done % IDENTITY_PROGRESS_EVERY === 0) { + log.info(`Fetched OAuth providers for ${done}/${total} users...`); + } + }), + ), + ); + + if (fatal) throw fatal; + return { identities, failed }; +} + +/** + * The OAuth provider breakdown, as its own block under the coverage table. + * + * Kept out of coverage on purpose: a coverage row means "N of the M users have + * this field", and these rows do not. One user can hold two providers, so the + * counts can sum past the user count, and "not readable" is not a property of + * the user at all. Two kinds of row under one heading would make both harder + * to read. + */ +export function buildIdentityReport( + users: WorkOsUser[], + identities: Map, + failed: number, +): ExportSection { + const byProvider = new Map(); + let none = 0; + + for (const user of users) { + const found = identities.get(String(user.id ?? "")); + // Absent means the lookup failed; `failed` already counts it. + if (!found) continue; + if (found.length === 0) { + none++; + continue; + } + for (const identity of found) { + const provider = identity.provider ?? "unknown"; + byProvider.set(provider, (byProvider.get(provider) ?? 0) + 1); + } + } + + // Busiest provider first; alphabetical within a tie so two runs of the same + // tenant print the same order. + const entries = [...byProvider].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])); + + const labels = [ + ...entries.map(([provider]) => provider), + NO_PROVIDER_LABEL, + ...(failed > 0 ? [NOT_READABLE_LABEL] : []), + ]; + const width = Math.max(...labels.map((label) => label.length)); + const row = (label: string, count: number) => + ` ${label.padEnd(width)} ${dim(`${count} user${count === 1 ? "" : "s"}`)}`; + + const rows = [ + ...entries.map(([provider, count]) => row(provider, count)), + row(NO_PROVIDER_LABEL, none), + ]; + + if (failed > 0) { + rows.push(row(NOT_READABLE_LABEL, failed)); + rows.push( + dim(" Those users have no `identities` field in the export, rather than an empty one."), + ); + } + + return { title: "OAuth providers", rows }; +} + +/** + * Keeps the fields the `workos` transformer maps from, plus `identities` when + * they were fetched. + * + * A copy rather than the raw record: WorkOS also returns `locale`, + * `profile_picture_url`, `last_sign_in_at` and `updated_at`, which the import + * does not carry. `identities` has no target field either and is + * dropped at validation, so it rides along purely as a record for whoever runs + * the migration. + */ +export function mapWorkOsUserToExport( + user: WorkOsUser, + identities?: WorkOsIdentity[], +): Record { + const exported: Record = {}; + + for (const field of [ + "id", + "email", + "first_name", + "last_name", + "external_id", + "created_at", + ] as const) { + if (user[field]) exported[field] = user[field]; + } + + // Meaningful when false: dropping it would import an address WorkOS never + // confirmed as a verified one. + if (user.email_verified !== undefined) exported.email_verified = user.email_verified; + + const metadata = user.metadata; + if (metadata && typeof metadata === "object" && Object.keys(metadata).length > 0) { + exported.metadata = metadata; + } + + if (identities && identities.length > 0) exported.identities = identities; + + return exported; +} + +export type WorkOsExportResult = { + users: Record[]; + coverage: { label: string; count: number }[]; +}; + +export function buildWorkOsExport( + users: WorkOsUser[], + record: (line: UserLine) => void = () => {}, + identities?: Map, +): WorkOsExportResult { + const exported: Record[] = []; + const counts = { email: 0, firstName: 0, lastName: 0, metadata: 0 }; + + for (const user of users) { + const userId = String(user.id ?? ""); + try { + const mapped = mapWorkOsUserToExport(user, identities?.get(userId)); + exported.push(mapped); + + if (mapped.email) counts.email++; + if (mapped.first_name) counts.firstName++; + if (mapped.last_name) counts.lastName++; + if (mapped.metadata) counts.metadata++; + + record({ sourceId: userId, status: "exported" }); + } catch (error) { + record({ sourceId: userId, status: "skipped", error: (error as Error).message }); + } + } + + return { + users: exported, + coverage: [ + { label: "have an email address", count: counts.email }, + { label: "have a first name", count: counts.firstName }, + { label: "have a last name", count: counts.lastName }, + { label: "have metadata", count: counts.metadata }, + // Always present, always zero. WorkOS returns no digest for anyone, and + // seeing that before the import is the whole reason the row is here. + { label: "have a password (WorkOS returns none)", count: 0 }, + ], + }; +} + +export async function exportWorkOs(options: ExportWorkOsOptions): Promise { + const resolved = await resolveWorkOsApiKey(options); + + await withGutter("Exporting users from WorkOS", async () => { + if (!options.json) printTarget({ platform: "workos" }); + // Only WorkOS can say whether the key is live, for the right environment, + // and not revoked — so a rejected key is asked for again here. The page it + // fetches is kept and reused, so proving the key costs no extra request. + const { value: firstPage, input: apiKey } = await withInputRetry( + resolved, + async () => promptWorkOsApiKey(), + async (candidate) => + withSpinner("Authenticating with WorkOS...", async () => fetchWorkOsPage(candidate)), + options, + ); + + const users = await withSpinner("Fetching users from WorkOS...", async (spinner) => + fetchAllWorkOsUsers({ apiKey, firstPage, spinner }), + ); + + const providers = (await resolveWithIdentities(options, users.length)) + ? await withSpinner("Fetching OAuth providers...", async (spinner) => + fetchAllWorkOsIdentities({ apiKey, users, spinner }), + ) + : undefined; + + const run = await startExportRun(options, { platform: "workos" }); + const { users: exported, coverage } = buildWorkOsExport( + users, + run.append, + providers?.identities, + ); + finishExport({ + run, + options, + users: exported, + coverage, + sections: providers + ? [buildIdentityReport(users, providers.identities, providers.failed)] + : [], + }); + + if (exported.length > 0) { + log.warn( + "WorkOS does not return password hashes or TOTP secrets, and there is no export that does. Imported users who signed in with a password must reset it on their first Clerk sign-in, and anyone using an authenticator app has to re-enrol. Users on SSO or social sign-in are unaffected.", + ); + log.info(dim(`See ${DOCS_URL}`)); + } + }); +} diff --git a/packages/cli-core/src/commands/migrate/import-users.test.ts b/packages/cli-core/src/commands/migrate/import-users.test.ts new file mode 100644 index 000000000..829c4c7f3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.test.ts @@ -0,0 +1,956 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { BapiError } from "../../lib/errors.ts"; +import { _resetInterruptState, abortInFlight } from "../../lib/signals.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { + buildCreateUserBody, + importUsers, + normalizeErrorMessage, + readRetryAfter, + splitIdentifiers, +} from "./import-users.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import type { UserLine } from "./lib/run-store.ts"; +import type { User } from "./types.ts"; + +const LIMITS: ResolvedLimits = { instanceType: "dev", rateLimit: 10_000, concurrencyLimit: 8 }; + +const user = (overrides: Partial = {}): User => + ({ userId: "u1", email: "a@x.dev", ...overrides }) as User; + +describe("splitIdentifiers", () => { + test("promotes the first verified email and phone to primary", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev", "b@x.dev"], phone: ["+15555550100", "+15555550101"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + expect(result.primaryPhone).toBe("+15555550100"); + expect(result.additionalPhones).toEqual(["+15555550101"]); + }); + + test("merges the email and emailAddresses fields, deduping", () => { + const result = splitIdentifiers( + user({ email: "a@x.dev", emailAddresses: ["a@x.dev", "b@x.dev"] }), + ); + expect(result.primaryEmail).toBe("a@x.dev"); + expect(result.additionalEmails).toEqual(["b@x.dev"]); + }); + + test("drops an unverified identifier that is already verified", () => { + const result = splitIdentifiers( + user({ email: ["a@x.dev"], unverifiedEmailAddresses: ["a@x.dev", "c@x.dev"] }), + ); + expect(result.unverifiedEmails).toEqual(["c@x.dev"]); + }); + + test("copes with a user identified only by username", () => { + const result = splitIdentifiers({ userId: "u1", username: "alice" } as User); + expect(result.primaryEmail).toBeUndefined(); + expect(result.additionalEmails).toEqual([]); + }); +}); + +describe("buildCreateUserBody", () => { + test("maps the schema onto BAPI's snake_case body", () => { + const target = user({ + firstName: "Alice", + lastName: "Smith", + username: "alice", + createdAt: "2024-01-01T00:00:00.000Z", + publicMetadata: { plan: "pro" }, + createOrganizationsLimit: 3, + banned: true, + }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + + expect(body).toMatchObject({ + external_id: "u1", + email_address: ["a@x.dev"], + first_name: "Alice", + last_name: "Smith", + username: "alice", + created_at: "2024-01-01T00:00:00.000Z", + public_metadata: { plan: "pro" }, + create_organizations_limit: 3, + banned: true, + }); + }); + + test("sends only the primary identifier; the rest are attached separately", () => { + const target = user({ email: ["a@x.dev", "b@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true).email_address).toEqual([ + "a@x.dev", + ]); + }); + + // Only the create can make a reserved identifier, so it goes here, after the + // verified primary, with a status per address in the same order. + test("sends unverified identifiers reserved when asked, in order", () => { + const target = user({ + email: "a@x.dev", + unverifiedEmailAddresses: ["c@x.dev"], + unverifiedPhoneNumbers: ["+15555550101"], + }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true, true)).toMatchObject({ + email_address: ["a@x.dev", "c@x.dev"], + email_address_identification_status: ["verified", "reserved"], + phone_number: ["+15555550101"], + phone_number_identification_status: ["reserved"], + }); + }); + + test("makes a reserved email the primary when there is no verified one", () => { + const target = user({ email: undefined, unverifiedEmailAddresses: ["c@x.dev"] }); + expect(buildCreateUserBody(target, splitIdentifiers(target), true, true)).toMatchObject({ + email_address: ["c@x.dev"], + email_address_identification_status: ["reserved"], + }); + }); + + test("sends no status arrays by default, or with nothing to reserve", () => { + const unverified = user({ unverifiedEmailAddresses: ["c@x.dev"] }); + const body = buildCreateUserBody(unverified, splitIdentifiers(unverified), true); + expect(body.email_address).toEqual(["a@x.dev"]); + expect(body).not.toHaveProperty("email_address_identification_status"); + + const verified = user(); + expect( + buildCreateUserBody(verified, splitIdentifiers(verified), true, true), + ).not.toHaveProperty("email_address_identification_status"); + }); + + // Allowlists and blocklists police sign-ups; these users already signed up. + test("skips the instance's sign-up restrictions", () => { + expect(buildCreateUserBody(user(), splitIdentifiers(user()), true)).toMatchObject({ + skip_restriction_checks: true, + }); + }); + + test("omits fields the source platform never recorded", () => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), true); + expect("first_name" in body).toBe(false); + expect("banned" in body).toBe(false); + expect("created_at" in body).toBe(false); + }); + + test("sends the password digest and hasher together", () => { + const target = user({ password: "digest", passwordHasher: "bcrypt" }); + const body = buildCreateUserBody(target, splitIdentifiers(target), true); + expect(body).toMatchObject({ password_digest: "digest", password_hasher: "bcrypt" }); + expect("skip_password_requirement" in body).toBe(false); + }); + + test.each([ + [true, true], + [false, false], + ])("skipPasswordRequirement=%p on a passwordless user -> flag present: %p", (skip, present) => { + const body = buildCreateUserBody(user(), splitIdentifiers(user()), skip); + expect("skip_password_requirement" in body).toBe(present); + }); +}); + +describe("readRetryAfter", () => { + const withHeader = (value: string) => + new BapiError(429, "{}", new Headers({ "retry-after": value })); + + test.each([ + ["12", 12], + ["0", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(withHeader(header))).toBe(expected as number | undefined); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ + errors: [{ code: "rate_limit", message: "slow down", meta: { retryAfter: 7 } }], + }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("returns undefined when neither source carries a value", () => { + expect(readRetryAfter(new BapiError(429, "{}", new Headers()))).toBeUndefined(); + }); +}); + +describe("normalizeErrorMessage", () => { + test("sorts field arrays so equivalent errors group together", () => { + const a = normalizeErrorMessage('["last_name" "first_name"] data does not match'); + const b = normalizeErrorMessage('["first_name" "last_name"] data does not match'); + expect(a).toBe(b); + expect(a).toBe('["first_name" "last_name"] data does not match'); + }); + + test("leaves messages without field arrays untouched", () => { + expect(normalizeErrorMessage("that email is taken")).toBe("that email is taken"); + }); +}); + +describe("importUsers", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + let lines: UserLine[]; + let allLines: UserLine[]; + // `lines` leaves out the `creating` marker every user gets first; one test + // below covers it through `allLines`. + const record = (line: UserLine) => { + allLines.push(line); + if (line.status !== "creating") lines.push(line); + }; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + beforeEach(() => { + requests = []; + lines = []; + allLines = []; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + /** Installs a fetch that records every request and replies per `respond`. */ + function stub(respond: (url: string, attempt: number) => Response): void { + const attempts = new Map(); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = input.toString(); + requests.push({ + method: init?.method ?? "GET", + url, + body: init?.body ? JSON.parse(init.body as string) : null, + }); + const attempt = (attempts.get(url) ?? 0) + 1; + attempts.set(url, attempt); + return respond(url, attempt); + }) as typeof fetch; + } + + const ok = (id: string) => new Response(JSON.stringify({ id }), { status: 200 }); + + const clerkError = (status: number, message: string, headers?: Record) => + new Response(JSON.stringify({ errors: [{ code: "err", message, long_message: message }] }), { + status, + headers, + }); + + // How a continue or an undo tells its own cut-off create from a user + // someone else made with the same external_id. + test("marks each create with its run, keeping the source's private metadata", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [user({ userId: "u1", privateMetadata: { plan: "pro" } })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + runId: "20260101-000000-abcd", + }); + + expect(requests.find((r) => r.url.endsWith("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/v1/users"))?.body).toMatchObject({ + private_metadata: { plan: "pro", clerkMigrateRun: "20260101-000000-abcd" }, + }); + }); + + test("creates each user and reports them as successful", async () => { + stub(() => ok("user_created")); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ totalProcessed: 2, successful: 2, failed: 0 }); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + expect(lines.filter((line) => line.status === "created")).toHaveLength(2); + }); + + test("neither fails nor records a create a Ctrl-C stopped before it was sent", async () => { + stub(() => ok("user_created")); + abortInFlight(); + try { + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 0, notSent: 2 }); + expect(summary.errorBreakdown.size).toBe(0); + expect(requests).toHaveLength(0); + expect(allLines).toHaveLength(0); + } finally { + _resetInterruptState(); + } + }); + + test("attaches additional and unverified identifiers after the user exists", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ + email: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["c@x.dev"], + phone: ["+15555550100", "+15555550101"], + }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + const emails = requests.filter((r) => r.url.endsWith("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/v1/email_addresses")); + expect(emails.map((r) => r.body)).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + { user_id: "user_created", email_address: "c@x.dev", primary: false, verified: false }, + ]); + expect(requests.filter((r) => r.url.endsWith("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/v1/phone_numbers"))).toHaveLength(1); + }); + + test("creates unverified identifiers reserved rather than attaching them", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"], unverifiedEmailAddresses: ["c@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests.find((r) => r.url.endsWith("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/v1/users"))?.body).toMatchObject({ + email_address: ["a@x.dev", "c@x.dev"], + email_address_identification_status: ["verified", "reserved"], + }); + expect( + requests.filter((r) => r.url.endsWith("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/v1/email_addresses")).map((r) => r.body), + ).toEqual([ + { user_id: "user_created", email_address: "b@x.dev", primary: false, verified: true }, + ]); + expect(lines.at(-1)).not.toHaveProperty("pending"); + }); + + test("marks a user whose password the source dropped", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [user({ passwordDropped: true })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(lines[0]).toMatchObject({ status: "created", passwordDropped: true }); + // Never sent: it is the CLI's own bookkeeping, not a Clerk field. + expect(JSON.stringify(requests[0]?.body)).not.toContain("passwordDropped"); + }); + + test("notes a failed additional identifier without failing the user", async () => { + stub((url) => + url.endsWith("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/v1/email_addresses") + ? clerkError(422, "that email is taken") + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + // On record once created, then with its attach pending, then with what + // the attach added. A refused attach is not retried, so nothing is pending. + expect(lines).toHaveLength(3); + expect(lines[0]).toEqual({ sourceId: "u1", clerkId: "user_created", status: "created" }); + expect(lines[1]?.pending).toEqual([{ kind: "email", value: "b@x.dev", verified: true }]); + expect(lines[2]).toMatchObject({ status: "created", clerkId: "user_created" }); + expect(lines[2]?.error).toContain("Failed to add additional email b@x.dev"); + expect(lines[2]).not.toHaveProperty("pending"); + }); + + test("retries an attach that hits a 429", async () => { + stub((url, attempt) => + url.endsWith("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/v1/email_addresses") && attempt === 1 + ? clerkError(429, "slow down", { "retry-after": "1" }) + : ok("user_created"), + ); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/email_addresses"))).toHaveLength(2); + expect(lines.at(-1)).not.toHaveProperty("error"); + expect(lines.at(-1)).not.toHaveProperty("pending"); + }); + + test("keeps an attach with no answer pending, for a continued run", async () => { + stub((url) => + url.endsWith("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/v1/email_addresses") ? clerkError(503, "unavailable") : ok("user_created"), + ); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(lines.at(-1)?.pending).toEqual([{ kind: "email", value: "b@x.dev", verified: true }]); + }); + + // One slot: a user's attaches go ahead of the next queued create, so a run + // stopped midway leaves few users without their extra identifiers. + test("attaches a user's identifiers before the next queued create", async () => { + stub((url) => ok(url.endsWith("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/v1/users") ? "user_created" : "idn_1")); + + await importUsers({ + users: [ + user({ userId: "u1", email: ["a@x.dev", "b@x.dev"] }), + user({ userId: "u2", email: ["c@x.dev", "d@x.dev"] }), + user({ userId: "u3", email: ["e@x.dev", "f@x.dev"] }), + ], + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(requests.map((r) => new URL(r.url).pathname)).toEqual([ + "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/v1/users", + "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/v1/email_addresses", + "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/v1/users", + "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/v1/email_addresses", + "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/v1/users", + "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/v1/email_addresses", + ]); + }); + + test("attachOnly sends just the pending attaches, and clears them", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [], + attachOnly: [ + { + sourceId: "u1", + clerkId: "user_1", + status: "created", + pending: [{ kind: "phone", value: "+15555550100", verified: false }], + }, + ], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests.map((r) => [new URL(r.url).pathname, r.body])).toEqual([ + [ + "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/v1/phone_numbers", + { user_id: "user_1", phone_number: "+15555550100", primary: false, verified: false }, + ], + ]); + expect(lines.at(-1)).toEqual({ sourceId: "u1", clerkId: "user_1", status: "created" }); + }); + + test("an adopted user is not created again; only its extras attach", async () => { + stub(() => ok("idn_1")); + + const summary = await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests.map((r) => new URL(r.url).pathname)).toEqual(["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/v1/email_addresses"]); + expect(summary.successful).toBe(1); + expect(lines.at(-1)).toMatchObject({ clerkId: "user_found", status: "created" }); + }); + + // The `creating` line says what the create did, so a continued run that + // adopts the user knows what is left to attach. + test("marks a creating line reserved only when the create reserved something", async () => { + stub(() => ok("user_created")); + + await importUsers({ + users: [ + user({ userId: "u1", unverifiedEmailAddresses: ["c@x.dev"] }), + user({ userId: "u2", email: "b@x.dev" }), + ], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(allLines.filter((line) => line.status === "creating")).toEqual([ + { sourceId: "u1", status: "creating", reserved: true }, + { sourceId: "u2", status: "creating" }, + ]); + }); + + // Adopted, the user is not created again, so its create's mode decides what + // is still missing, not this run's flag. + test("attaches an adopted user's unverified email its create left out, even with the flag", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [user({ unverifiedEmailAddresses: ["c@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests.map((r) => r.body)).toEqual([ + { user_id: "user_found", email_address: "c@x.dev", primary: false, verified: false }, + ]); + }); + + test("attaches nothing for an adopted user whose create reserved them, even without the flag", async () => { + stub(() => ok("idn_1")); + + await importUsers({ + users: [user({ unverifiedEmailAddresses: ["c@x.dev"] })], + adopted: new Map([["u1", "user_found"]]), + adoptedReserved: new Set(["u1"]), + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(requests).toEqual([]); + }); + + // Shapes from clerk_go's apierror: the country error carries its own code + // and no param_name; the E.164 error is a form error on phone_number. + test.each([ + [ + 403, + { + code: "unsupported_country_code", + message: "Unsupported country code", + long_message: "Phone numbers from this country (Netherlands) are currently not supported.", + meta: { alpha2: "NL", country_code: "31" }, + }, + ], + [ + 422, + { + code: "form_param_format_invalid", + message: "is invalid", + long_message: + "Phone number must be a valid phone number according to E.164 international standard.", + meta: { param_name: "phone_number" }, + }, + ], + ])("retries without a phone Clerk refuses (%i), and notes it", async (status, clerkErr) => { + stub((url, attempt) => + url.endsWith("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/v1/users") && attempt === 1 + ? new Response(JSON.stringify({ errors: [clerkErr] }), { status }) + : ok("user_created"), + ); + + const summary = await importUsers({ + users: [user({ phone: "+31612345678" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + // Imported, but the summary has to say the phone was lost. + expect([...summary.droppedPhones]).toEqual([[clerkErr.long_message, 1]]); + expect(requests).toHaveLength(2); + expect(requests[1]?.body).not.toHaveProperty("phone_number"); + expect(lines.at(-1)?.error).toContain( + `Failed to add phone +31612345678: ${clerkErr.long_message}`, + ); + }); + + test("drops a refused phone's statuses with it", async () => { + stub((url, attempt) => + url.endsWith("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/v1/users") && attempt === 1 + ? new Response( + JSON.stringify({ + errors: [{ code: "x", message: "bad phone", meta: { param_name: "phone_number" } }], + }), + { status: 422 }, + ) + : ok("user_created"), + ); + + await importUsers({ + users: [user({ unverifiedPhoneNumbers: ["+31612345678"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests[0]?.body).toHaveProperty("phone_number_identification_status"); + expect(requests[1]?.body).not.toHaveProperty("phone_number"); + expect(requests[1]?.body).not.toHaveProperty("phone_number_identification_status"); + expect(lines.at(-1)?.error).toContain("Failed to add phone +31612345678"); + }); + + describe("a refused phone among reserved ones", () => { + const refusePhone = (refusals: number) => + stub((url, attempt) => + url.endsWith("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/v1/users") && attempt <= refusals + ? new Response( + JSON.stringify({ + errors: [{ code: "x", message: "bad phone", meta: { param_name: "phone_number" } }], + }), + { status: 422 }, + ) + : ok("user_created"), + ); + const withPhones = (fields: Partial = {}) => + user({ phone: "+15555550100", unverifiedPhoneNumbers: ["+31612345678"], ...fields }); + + // Any of the phones may be the refused one; the verified one goes alone first. + test("keeps the verified phone, dropping only the reserved ones", async () => { + refusePhone(1); + await importUsers({ + users: [withPhones()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests[1]?.body).toMatchObject({ phone_number: ["+15555550100"] }); + expect(requests[1]?.body).not.toHaveProperty("phone_number_identification_status"); + expect(lines.at(-1)?.error).toContain("Failed to add phone +31612345678:"); + expect(lines.at(-1)?.error).not.toContain("+15555550100"); + }); + + test("creates a user with no email on its verified phone", async () => { + refusePhone(1); + const summary = await importUsers({ + users: [withPhones({ email: undefined })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + }); + + test("drops every phone only when the verified one is refused too", async () => { + refusePhone(2); + await importUsers({ + users: [withPhones()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + reserveUnverified: true, + }); + + expect(requests).toHaveLength(3); + expect(requests[2]?.body).not.toHaveProperty("phone_number"); + expect(lines.at(-1)?.error).toContain("Failed to add phone +15555550100, +31612345678:"); + }); + }); + + test("does not retry without the phone when it is the only identifier", async () => { + stub( + () => + new Response( + JSON.stringify({ + errors: [{ code: "x", message: "bad phone", meta: { param_name: "phone_number" } }], + }), + { status: 422 }, + ), + ); + + const summary = await importUsers({ + users: [user({ email: undefined, phone: "+31612345678" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary.failed).toBe(1); + expect(requests).toHaveLength(1); + }); + + // The marker `undo` uses to find a user whose create was in flight when the + // run stopped. + test("marks each user as creating before its POST /v1/users", async () => { + let recordedBeforePost = false; + stub((url) => { + if (url.endsWith("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/v1/users")) { + recordedBeforePost = allLines.some((line) => line.status === "creating"); + } + return ok("user_created"); + }); + + await importUsers({ users: [user()], secretKey: "sk_test_x", limits: LIMITS, record }); + + expect(recordedBeforePost).toBe(true); + expect(allLines.map((line) => line.status)).toEqual(["creating", "created"]); + }); + + // A run stopped while attaches wait on the scheduler must still have the + // user on record, or `undo` leaves it behind. + test("records the user before its additional identifiers attach", async () => { + let recordedBeforeAttach = false; + stub((url) => { + if (url.endsWith("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/v1/email_addresses")) { + recordedBeforeAttach = lines.some( + (line) => line.status === "created" && line.clerkId === "user_created", + ); + } + return ok("user_created"); + }); + + await importUsers({ + users: [user({ email: ["a@x.dev", "b@x.dev"] })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(recordedBeforeAttach).toBe(true); + }); + + // A user is on record only once its create may land, so `undo` never + // looks up users that were still queued when the run stopped. + test("writes creating only as each POST /v1/users goes out", async () => { + const creatingAtFirstPost: number[] = []; + stub((url) => { + if (url.endsWith("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/v1/users") && creatingAtFirstPost.length === 0) { + creatingAtFirstPost.push(allLines.filter((line) => line.status === "creating").length); + } + return ok("user_created"); + }); + + await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2" }), user({ userId: "u3" })], + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(creatingAtFirstPost).toEqual([1]); + }); + + test.each([ + ["a 5xx", () => clerkError(502, "bad gateway")], + [ + "a network error", + () => { + throw new TypeError("fetch failed"); + }, + ], + ])("a create that gets %s keeps creating, not failed", async (_label, respond) => { + stub(respond); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary.failed).toBe(1); + expect(allLines.map((line) => line.status)).toEqual(["creating"]); + expect([...summary.errorBreakdown.keys()][0]).toContain("a re-run checks"); + }); + + test("leaves a create answered without a user ID as unknown, never created", async () => { + stub(() => new Response(JSON.stringify({}), { status: 200 })); + + const summary = await importUsers({ + users: [user({ userId: "u1" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + expect(allLines).toEqual([{ sourceId: "u1", status: "creating" }]); + }); + + // Every create after the first refusal would be refused too, so none is + // sent: they stay unrecorded, and a re-run picks them up. + test("stops sending creates once the instance's user quota is reached", async () => { + stub((url) => + url.endsWith("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/v1/users") + ? new Response( + JSON.stringify({ + errors: [ + { + code: "user_quota_exceeded", + message: "user quota exceeded", + long_message: "You have reached your limit of 100 users.", + }, + ], + }), + { status: 403 }, + ) + : ok("unused"), + ); + + const summary = await importUsers({ + users: ["u1", "u2", "u3", "u4"].map((userId) => user({ userId, email: `${userId}@x.dev` })), + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(1); + expect(summary).toMatchObject({ successful: 0, failed: 1, notSent: 3 }); + expect(allLines.filter((line) => line.status !== "creating")).toMatchObject([ + { sourceId: "u1", status: "failed", code: "403" }, + ]); + expect(allLines.map((line) => line.sourceId)).not.toContain("u2"); + // Returned for the caller to print after the progress bar, not logged + // under it, where the bar would redraw over it. + expect(summary.stopReason).toBe("You have reached your limit of 100 users."); + expect(captured.err).not.toContain("limit of 100 users"); + }); + + // On dev, at 10 a second, thousands of queued users would otherwise spend + // minutes taking paced turns only to be skipped. + test("after the quota stop, queued creates are skipped without waiting their turn", async () => { + stub((url) => + url.endsWith("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/v1/users") + ? new Response( + JSON.stringify({ + errors: [{ code: "user_quota_exceeded", message: "quota", long_message: "Full." }], + }), + { status: 403 }, + ) + : ok("unused"), + ); + + const started = performance.now(); + const summary = await importUsers({ + users: Array.from({ length: 6 }, (_, i) => user({ userId: `u${i}`, email: `u${i}@x.dev` })), + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1, rateLimit: 2 }, + record, + }); + + expect(summary).toMatchObject({ failed: 1, notSent: 5 }); + // Paced, the five would take about 2.5s; skipped, they take none. + expect(performance.now() - started).toBeLessThan(1000); + }); + + test("records a failed user and keeps going", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(422, "that email is taken") : ok("user_ok"), + ); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary.successful + summary.failed).toBe(2); + expect(summary.failed).toBe(1); + expect([...summary.errorBreakdown.values()]).toEqual([1]); + expect(lines.some((line) => line.status === "failed" && line.code === "422")).toBe(true); + }); + + test("retries a 429 after the interval the server asked for", async () => { + stub((_url, attempt) => + attempt === 1 ? clerkError(429, "slow down", { "retry-after": "1" }) : ok("user_ok"), + ); + + const started = performance.now(); + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 1, failed: 0 }); + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + expect(lines).toHaveLength(2); + expect(lines[1]).toMatchObject({ status: "created", clerkId: "user_ok" }); + expect(lines[1]?.error).toContain("Rate limit hit (429)"); + }); + + // The instance is over its limit for every user, so the others wait too. + test("holds every other user's create through a 429's wait", async () => { + const started = performance.now(); + const sentAt: number[] = []; + stub((_url, attempt) => { + sentAt.push(performance.now() - started); + return attempt === 1 ? clerkError(429, "slow down", { "retry-after": "1" }) : ok("user_ok"); + }); + + const summary = await importUsers({ + users: [user({ userId: "u1" }), user({ userId: "u2", email: "b@x.dev" })], + secretKey: "sk_test_x", + limits: { ...LIMITS, concurrencyLimit: 1 }, + record, + }); + + expect(summary).toMatchObject({ successful: 2, failed: 0 }); + expect(sentAt).toHaveLength(3); + for (const at of sentAt.slice(1)) expect(at).toBeGreaterThanOrEqual(900); + }); + + test("gives up after the retry ceiling and records the user as failed", async () => { + stub(() => clerkError(429, "slow down", { "retry-after": "1" })); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + }); + + expect(summary).toMatchObject({ successful: 0, failed: 1 }); + // One initial attempt plus MAX_RETRIES retries. + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(6); + expect(lines).toMatchObject([{ status: "failed", code: "429" }]); + }, 20_000); + + test("carries the validation failure count into the summary", async () => { + stub(() => ok("user_ok")); + + const summary = await importUsers({ + users: [user()], + secretKey: "sk_test_x", + limits: LIMITS, + record, + validationFailed: 4, + }); + + expect(summary.validationFailed).toBe(4); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/import-users.ts b/packages/cli-core/src/commands/migrate/import-users.ts new file mode 100644 index 000000000..f4d53ab82 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/import-users.ts @@ -0,0 +1,645 @@ +/** + * Creates users in Clerk from a validated batch. + * + * Ported from the standalone migration-tool's `src/migrate/import-users.ts`, + * rewritten onto `bapiRequest` instead of `@clerk/backend`. Two things fall out + * of that move: + * + * - The request body is BAPI's snake_case shape directly, so `created_at` and + * `legal_accepted_at` stay RFC3339 strings rather than round-tripping + * through `Date`. + * - `banned`, `delete_self_enabled` and the organization limits are all + * accepted by `POST /v1/users`, so the follow-up `updateUser`/`banUser` + * calls the SDK version needed are gone. + * + * Run state is local to {@link importUsers} rather than module-level, so two + * runs in one process (or one test file) cannot see each other's counters. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { BapiError } from "../../lib/errors.ts"; +import { interruptSignal } from "../../lib/signals.ts"; +import type { ResolvedLimits } from "./lib/instance.ts"; +import type { ProgressUpdate } from "./lib/progress.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import type { PendingIdentifier, UserLine } from "./lib/run-store.ts"; +import { RUN_MARKER_KEY } from "./lib/user-lookup.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import type { ImportSummary, User } from "./types.ts"; + +// Re-exported for the tests and callers that grew up against this module. +export { readRetryAfter } from "./lib/retry.ts"; + +/** + * Groups error messages that differ only in field ordering, so the summary + * reports "12 users: [\"first_name\" \"last_name\"] ..." once instead of twice. + */ +export function normalizeErrorMessage(errorMessage: string): string { + let normalized = ""; + let lastCopiedIndex = 0; + let arrayStartIndex = -1; + + for (let i = 0; i < errorMessage.length; i++) { + const char = errorMessage[i]; + + if (arrayStartIndex === -1) { + if (char === "[") arrayStartIndex = i; + continue; + } + if (char !== "]") continue; + + normalized += errorMessage.slice(lastCopiedIndex, arrayStartIndex); + normalized += normalizeFieldArray(errorMessage.slice(arrayStartIndex + 1, i)); + lastCopiedIndex = i + 1; + arrayStartIndex = -1; + } + + return normalized + errorMessage.slice(lastCopiedIndex); +} + +function normalizeFieldArray(fields: string): string { + const fieldNames: string[] = []; + let current = ""; + + for (const char of fields) { + if (char === '"' || char === "'" || char.trim() === "") { + if (current.length > 0) { + fieldNames.push(current); + current = ""; + } + continue; + } + current += char; + } + if (current.length > 0) fieldNames.push(current); + + fieldNames.sort(); + return `[${fieldNames.map((name) => `"${name}"`).join(" ")}]`; +} + +function toArray(value: string | string[] | undefined): string[] { + if (!value) return []; + return Array.isArray(value) ? value : [value]; +} + +function dedupe(values: string[]): string[] { + const seen: string[] = []; + for (const value of values) { + if (value && !seen.includes(value)) seen.push(value); + } + return seen; +} + +type Identifiers = { + primaryEmail: string | undefined; + additionalEmails: string[]; + unverifiedEmails: string[]; + primaryPhone: string | undefined; + additionalPhones: string[]; + unverifiedPhones: string[]; +}; + +/** + * Splits a user's identifiers into the one that goes on `POST /v1/users` and + * the rest, which are attached afterwards. + */ +export function splitIdentifiers(user: User): Identifiers { + const verifiedEmails = dedupe([...toArray(user.email), ...toArray(user.emailAddresses)]); + const verifiedPhones = dedupe([...toArray(user.phone), ...toArray(user.phoneNumbers)]); + + return { + primaryEmail: verifiedEmails[0], + additionalEmails: verifiedEmails.slice(1), + unverifiedEmails: dedupe( + toArray(user.unverifiedEmailAddresses).filter((email) => !verifiedEmails.includes(email)), + ), + primaryPhone: verifiedPhones[0], + additionalPhones: verifiedPhones.slice(1), + unverifiedPhones: dedupe( + toArray(user.unverifiedPhoneNumbers).filter((phone) => !verifiedPhones.includes(phone)), + ), + }; +} + +/** + * The values and parallel statuses for one identifier kind on `POST /v1/users`. + * + * A reserved identifier is unverified, but usable for sign-in and locked to + * this user. Only the create can make one, so with `reserve` the unverified + * identifiers go here instead of being attached afterwards. + */ +function createIdentifiers( + verified: string | undefined, + unverified: string[], + reserve: boolean, +): { values: string[]; statuses?: string[] } { + const values = [...(verified ? [verified] : []), ...(reserve ? unverified : [])]; + if (!reserve || unverified.length === 0) return { values }; + return { + values, + statuses: values.map((_, index) => (verified && index === 0 ? "verified" : "reserved")), + }; +} + +/** + * Builds the `POST /v1/users` request body. + * + * Optional fields are omitted rather than sent as null so Clerk applies its own + * defaults for anything the source platform did not record. + * + * @param reserveUnverified - Create the identifiers the source never verified + * as reserved, in this request, rather than attaching them unverified. + */ +export function buildCreateUserBody( + user: User, + identifiers: Identifiers, + skipPasswordRequirement: boolean, + reserveUnverified = false, +): Record { + // The instance's allowlist, blocklist, disposable-email and subaddress rules + // police sign-ups. These users already signed up, on the source platform. + const body: Record = { external_id: user.userId, skip_restriction_checks: true }; + + const emails = createIdentifiers( + identifiers.primaryEmail, + identifiers.unverifiedEmails, + reserveUnverified, + ); + if (emails.values.length > 0) body.email_address = emails.values; + if (emails.statuses) body.email_address_identification_status = emails.statuses; + const phones = createIdentifiers( + identifiers.primaryPhone, + identifiers.unverifiedPhones, + reserveUnverified, + ); + if (phones.values.length > 0) body.phone_number = phones.values; + if (phones.statuses) body.phone_number_identification_status = phones.statuses; + if (user.firstName) body.first_name = user.firstName; + if (user.lastName) body.last_name = user.lastName; + if (user.username) body.username = user.username; + if (user.totpSecret) body.totp_secret = user.totpSecret; + if (user.backupCodes) body.backup_codes = user.backupCodes; + if (user.unsafeMetadata) body.unsafe_metadata = user.unsafeMetadata; + if (user.privateMetadata) body.private_metadata = user.privateMetadata; + if (user.publicMetadata) body.public_metadata = user.publicMetadata; + if (user.createdAt) body.created_at = user.createdAt; + if (user.legalAcceptedAt) body.legal_accepted_at = user.legalAcceptedAt; + if (user.skipLegalChecks !== undefined) body.skip_legal_checks = user.skipLegalChecks; + if (user.skipPasswordChecks !== undefined) body.skip_password_checks = user.skipPasswordChecks; + if (user.banned !== undefined) body.banned = user.banned; + if (user.bypassClientTrust !== undefined) body.bypass_client_trust = user.bypassClientTrust; + if (user.deleteSelfEnabled !== undefined) body.delete_self_enabled = user.deleteSelfEnabled; + if (user.createOrganizationEnabled !== undefined) { + body.create_organization_enabled = user.createOrganizationEnabled; + } + if (user.createOrganizationsLimit !== undefined) { + body.create_organizations_limit = user.createOrganizationsLimit; + } + + if (user.password && user.passwordHasher) { + body.password_digest = user.password; + body.password_hasher = user.passwordHasher; + } else if (skipPasswordRequirement) { + body.skip_password_requirement = true; + } + // Without a password and without skipPasswordRequirement, Clerk rejects the + // user — which is exactly what --require-password is asking for. + + return body; +} + +type CreateContext = { + secretKey: string; + schedule: ApiScheduler; + /** Unverified identifiers go on the create as reserved, not attached after. */ + reserveUnverified: boolean; + /** The run sending these creates; each carries it as its marker. */ + runId?: string; + /** Set by the first `user_quota_exceeded`: every later create would be refused too. */ + quotaReached: boolean; +}; + +/** + * True when Clerk refused the create for a phone. The country error names no + * parameter, only its own code. + */ +function isPhoneRefusal(error: unknown): error is BapiError { + return ( + error instanceof BapiError && + (error.code === "unsupported_country_code" || error.meta?.param_name === "phone_number") + ); +} + +/** A create a Ctrl-C stopped before it went out: neither a failure nor unknown. */ +class NotSentError extends Error {} + +/** + * True when no answer says whether a request landed: an abort, a network + * error, or a 5xx after which Clerk may still have committed it. A 4xx, and a + * 429 that ran out of retries, are definite refusals. + */ +export function outcomeUnknown(error: unknown): boolean { + if (error instanceof RateLimitExceededError) return false; + if (error instanceof BapiError) return error.status >= 500; + return true; +} + +/** + * The extra identifiers a user carries, in the order they are attached. + * + * @param reserveUnverified - The unverified ones went on the create as + * reserved, so there is nothing to attach for them. + */ +export function pendingIdentifiers( + identifiers: Identifiers, + reserveUnverified = false, +): PendingIdentifier[] { + if (reserveUnverified) { + identifiers = { ...identifiers, unverifiedEmails: [], unverifiedPhones: [] }; + } + return [ + ...identifiers.additionalEmails.map((value) => ({ + kind: "email" as const, + value, + verified: true, + })), + ...identifiers.unverifiedEmails.map((value) => ({ + kind: "email" as const, + value, + verified: false, + })), + ...identifiers.additionalPhones.map((value) => ({ + kind: "phone" as const, + value, + verified: true, + })), + ...identifiers.unverifiedPhones.map((value) => ({ + kind: "phone" as const, + value, + verified: false, + })), + ]; +} + +/** + * Attaches one extra identifier, backing off on a 429. + * + * @returns A note when Clerk refused it, `pending` when nothing says whether + * it attached. Never throws: the user itself was already created. + */ +async function attachIdentifier( + ctx: CreateContext, + clerkUserId: string, + { kind, value, verified }: PendingIdentifier, +): Promise<{ note?: string; pending?: boolean }> { + const path = kind === "email" ? "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/v1/email_addresses" : "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/v1/phone_numbers"; + const body = + kind === "email" + ? { user_id: clerkUserId, email_address: value, primary: false, verified } + : { user_id: clerkUserId, phone_number: value, primary: false, verified }; + + try { + await retryOn429( + async () => + ctx.schedule( + async () => + bapiRequest({ + method: "POST", + path, + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }), + { first: true }, + ), + { onRetry: ({ delaySeconds }) => ctx.schedule.pause(delaySeconds * 1000) }, + ); + return {}; + } catch (error) { + if (outcomeUnknown(error)) return { pending: true }; + const label = `${verified ? "additional" : "unverified"} ${kind} ${value}`; + return { note: `Failed to add ${label}: ${(error as Error).message}` }; + } +} + +/** + * Attaches each identifier. Extra identifiers are best-effort: a duplicate + * secondary email should not undo a user who was otherwise imported. + * + * @returns A note per identifier Clerk refused, and those still pending. + */ +async function attachAll( + ctx: CreateContext, + clerkUserId: string, + identifiers: PendingIdentifier[], +): Promise<{ notes: string[]; pending: PendingIdentifier[] }> { + const results = await Promise.all( + identifiers.map(async (identifier) => attachIdentifier(ctx, clerkUserId, identifier)), + ); + return { + notes: results.flatMap((result) => (result.note ? [result.note] : [])), + pending: identifiers.filter((_, index) => results[index]?.pending), + }; +} + +/** + * Creates one user, retrying without a phone Clerk refuses. + * + * @param sending - Called as each `POST /v1/users` goes out, so the run + * records the user only once a create may actually land. + * @returns The Clerk ID, a note when the phone was dropped, and Clerk's + * reason for refusing it. + */ +async function createUser( + ctx: CreateContext, + user: User, + identifiers: Identifiers, + skipPasswordRequirement: boolean, + sending: () => void, +): Promise<{ clerkUserId: string; notes: string[]; phoneRefusal?: string }> { + // A Ctrl-C or a full instance hands the slot on to queued creates; none + // of them is sent, and none waits for a paced turn to find that out. + const stopped = () => interruptSignal().aborted || ctx.quotaReached; + const create = async (body: Record) => + ctx.schedule( + async () => { + if (stopped()) throw new NotSentError(); + sending(); + return bapiRequest({ + method: "POST", + path: "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/v1/users", + secretKey: ctx.secretKey, + body: JSON.stringify(body), + }); + }, + { skipWait: stopped }, + ); + + const body = buildCreateUserBody( + user, + identifiers, + skipPasswordRequirement, + ctx.reserveUnverified, + ); + // The run's marker, with whatever private metadata the source brought: + // without it a create cut off mid-flight could never be told from a user + // someone else made with the same external_id. + if (ctx.runId) { + body.private_metadata = { + ...(body.private_metadata as Record | undefined), + [RUN_MARKER_KEY]: ctx.runId, + }; + } + const notes: string[] = []; + let phoneRefusal: string | undefined; + let response; + try { + response = await create(body); + } catch (error) { + // A phone Clerk refuses (a country it does not support, a number that is + // not E.164) should not cost a user who has an email to be created under. + if (!isPhoneRefusal(error)) throw error; + const { + phone_number: sent, + phone_number_identification_status: statuses, + ...withoutPhone + } = body; + let refusal = error; + let dropped = sent as string[]; + // With reserved phones on the create, any one of them may be the refused + // one: the verified phone alone goes first, so it is not lost with them. + if (statuses && identifiers.primaryPhone) { + try { + response = await create({ ...withoutPhone, phone_number: [identifiers.primaryPhone] }); + dropped = dropped.filter((phone) => phone !== identifiers.primaryPhone); + } catch (retryError) { + if (!isPhoneRefusal(retryError)) throw retryError; + refusal = retryError; + } + } + if (!response) { + if (!body.email_address) throw refusal; + response = await create(withoutPhone); + } + phoneRefusal = refusal.longMessage ?? refusal.message; + notes.push(`Failed to add phone ${dropped.join(", ")}: ${phoneRefusal}`); + } + + // Untracked, the user could never be undone. Thrown, the outcome is unknown, + // so `creating` stays the latest line and a re-run looks the user up. + const clerkUserId = (response.body as { id?: unknown })?.id; + if (typeof clerkUserId !== "string" || !clerkUserId) { + throw new Error("Clerk answered POST /v1/users without a user ID"); + } + return { clerkUserId, notes, ...(phoneRefusal ? { phoneRefusal } : {}) }; +} + +export type ImportUsersOptions = { + users: User[]; + secretKey: string; + limits: ResolvedLimits; + /** Receives each user's lines as they happen. */ + record: (line: UserLine) => void; + /** + * Users a continued run created whose extra identifiers never attached: their + * latest `created` line, with `pending`. Only the attaches are sent. + */ + attachOnly?: UserLine[]; + /** The run these creates belong to, sent on each as its marker. */ + runId?: string; + /** + * Source ID → Clerk ID for users whose create a stopped run sent with no + * answer, and which a continued run then found in the instance. They are + * not created again; only their extra identifiers are sent. + */ + adopted?: Map; + /** + * The adopted users whose create reserved their unverified identifiers, from + * its `creating` line. The rest had them attached unverified, whatever this + * run's `reserveUnverified` says. + */ + adoptedReserved?: Set; + /** Allow users that carry no password. */ + skipPasswordRequirement?: boolean; + /** Create the identifiers the source never verified as reserved. */ + reserveUnverified?: boolean; + /** Carried into the summary so the report covers the whole file. */ + validationFailed?: number; + /** Receives the counts as each user finishes. */ + progress?: ProgressUpdate; +}; + +/** + * Imports every user, concurrently and within the instance's rate limit. + * + * A failed user is recorded and the run continues; a 429 backs off (honouring + * `Retry-After`) and retries up to {@link MAX_RETRIES} times. A create with no + * answer keeps its `creating` line, for a continued run or `undo` to resolve. + */ +export async function importUsers(options: ImportUsersOptions): Promise { + const { + users, + secretKey, + limits, + record, + attachOnly = [], + runId, + adopted = new Map(), + adoptedReserved = new Set(), + skipPasswordRequirement = true, + reserveUnverified = false, + validationFailed = 0, + progress: report, + } = options; + + const total = users.length; + const errorBreakdown = new Map(); + let processed = 0; + let successful = 0; + let failed = 0; + let notSent = 0; + let stopReason: string | undefined; + const droppedPhones = new Map(); + + const ctx: CreateContext = { + secretKey, + schedule: createApiScheduler(limits.concurrencyLimit, limits.rateLimit), + reserveUnverified, + quotaReached: false, + ...(runId ? { runId } : {}), + }; + + const progress = () => report?.({ done: processed, ok: successful, failed }); + + const recordFailure = ( + userId: string, + message: string, + code: string, + notes: string[], + unknown: boolean, + ) => { + failed++; + processed++; + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + // With no answer, the `creating` line stays the latest: Clerk may hold + // the user, and a re-run looks it up before creating it again. + if (!unknown) { + record({ sourceId: userId, status: "failed", error: [message, ...notes].join("; "), code }); + } + progress(); + }; + + /** + * Attaches a created user's extra identifiers. The user goes on record with + * them `pending` first, so a run stopped before they attach can finish them. + */ + const finishUser = async (line: UserLine, toAttach: PendingIdentifier[], notes: string[]) => { + const { error: _error, pending: _pending, ...base } = line; + if (toAttach.length > 0) record({ ...base, pending: toAttach }); + const attached = await attachAll(ctx, base.clerkId ?? "", toAttach); + const error = [...notes, ...attached.notes].join("; "); + // A second line, which wins as the latest, adds what happened on the way. + if (toAttach.length > 0 || error) { + record({ + ...base, + ...(error ? { error } : {}), + ...(attached.pending.length > 0 ? { pending: attached.pending } : {}), + }); + } + }; + + const processUser = async (user: User): Promise => { + const retries: string[] = []; + const identifiers = splitIdentifiers(user); + let created: { clerkUserId: string; notes: string[]; phoneRefusal?: string }; + const adoptedId = adopted.get(user.userId); + // What the create did with the unverified identifiers decides what is left + // to attach: an adopted user's create may have run in the other mode. + const reserved = adoptedId + ? adoptedReserved.has(user.userId) + : reserveUnverified && + (identifiers.unverifiedEmails.length > 0 || identifiers.unverifiedPhones.length > 0); + try { + created = adoptedId + ? { clerkUserId: adoptedId, notes: [] } + : await retryOn429( + async () => + createUser(ctx, user, identifiers, skipPasswordRequirement, () => + record({ + sourceId: user.userId, + status: "creating", + ...(reserved ? { reserved: true } : {}), + }), + ), + { + onRetry: ({ message, delaySeconds }) => { + retries.push(message); + ctx.schedule.pause(delaySeconds * 1000); + }, + }, + ); + } catch (error) { + // Unrecorded, so a re-run picks the user up like any other. + if (error instanceof NotSentError) { + notSent++; + return; + } + if (error instanceof RateLimitExceededError) { + recordFailure(user.userId, error.message, "429", retries, false); + return; + } + const apiError = error as BapiError; + if (apiError.code === "user_quota_exceeded" && !ctx.quotaReached) { + ctx.quotaReached = true; + // Returned, not logged: a progress bar would redraw over a warning. + stopReason = apiError.longMessage ?? apiError.message; + } + const unknown = outcomeUnknown(error); + const message = apiError.longMessage ?? apiError.message ?? "Unknown error"; + recordFailure( + user.userId, + unknown ? `${message} (Clerk may have created the user; a re-run checks)` : message, + String(apiError.status ?? "unknown"), + retries, + unknown, + ); + return; + } + + const line: UserLine = { + sourceId: user.userId, + clerkId: created.clerkUserId, + status: "created", + ...(user.passwordDropped ? { passwordDropped: true } : {}), + }; + record(line); + if (created.phoneRefusal) { + const reason = normalizeErrorMessage(created.phoneRefusal); + droppedPhones.set(reason, (droppedPhones.get(reason) ?? 0) + 1); + } + await finishUser(line, pendingIdentifiers(identifiers, reserved), [ + ...created.notes, + ...retries, + ]); + successful++; + processed++; + progress(); + }; + + progress(); + await Promise.all([ + ...users.map(async (user) => processUser(user)), + ...attachOnly.map(async (line) => finishUser(line, line.pending ?? [], [])), + ]); + + return { + totalProcessed: total, + successful, + failed, + notSent, + ...(stopReason ? { stopReason } : {}), + droppedPhones, + validationFailed, + errorBreakdown, + }; +} diff --git a/packages/cli-core/src/commands/migrate/index.test.ts b/packages/cli-core/src/commands/migrate/index.test.ts new file mode 100644 index 000000000..657fa0e55 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { getMode, setMode } from "../../mode.ts"; +import { createProgram } from "../../cli-program.ts"; +import { exportPlatformKeys } from "./export/registry.ts"; +import { isAssumeYes, setAssumeYes } from "./lib/assume-yes.ts"; + +function findCommand(names: string[]) { + let current = createProgram().commands.find((cmd) => cmd.name() === names[0]); + for (const name of names.slice(1)) { + current = current?.commands.find((cmd) => cmd.name() === name); + } + return current; +} + +describe("registerMigrate", () => { + test("registers migrate as a top-level command group", () => { + const migrate = findCommand(["migrate"]); + expect(migrate).toBeDefined(); + expect(migrate?.description()).toContain("Migrate users"); + }); + + test("registers the import subcommand", () => { + expect(findCommand(["migrate", "import"])).toBeDefined(); + }); + + // The direction is never implied: `import` and `export` are siblings, so a + // default would make one of them the meaning of the bare group name. + test("leaves migrate with no default subcommand", () => { + const migrate = findCommand(["migrate"]) as unknown as { _defaultCommandName?: string }; + expect(migrate._defaultCommandName).toBeFalsy(); + }); + + test.each([ + "--source", + "--dry-run", + "--allow-partial", + "--new-run", + "--require-password", + "--reserve-unverified", + "--json", + "--firebase-signer-key", + "--firebase-salt-separator", + "--firebase-rounds", + "--firebase-mem-cost", + "--yes", + "--secret-key", + "--app", + "--instance", + ])("migrate import accepts %s", (flag) => { + const flags = findCommand(["migrate", "import"])?.options.map((option) => option.long); + expect(flags).toContain(flag); + }); + + test("registers sources with an optional source and --json", () => { + const sources = findCommand(["migrate", "sources"]); + expect(sources?.registeredArguments[0]?.required).toBe(false); + expect(sources?.options.map((option) => option.long)).toEqual(["--json"]); + }); + + test.each([[["transformers"]], [["transformers", "list"]]])("no longer registers %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeUndefined(); + }); + + test.each([ + "--transformer", + "--transformer-file", + "--file", + "--resume-after", + "--skip-unsupported-providers", + ])("migrate import drops %s", (flag) => { + expect(findCommand(["migrate", "import"])?.options.map((o) => o.long)).not.toContain(flag); + }); + + test.each([ + [["export"]], + [["export", "clerk"]], + [["export", "auth0"]], + [["export", "supabase"]], + [["export", "authjs"]], + [["export", "betterauth"]], + [["export", "firebase"]], + ])("registers migrate %p", (names) => { + expect(findCommand(["migrate", ...names])).toBeDefined(); + }); + + // Bare `migrate export` runs the picker rather than defaulting to a + // platform, so nobody exports from the wrong place by pressing enter. + test("leaves export with no default subcommand", () => { + const group = findCommand(["migrate", "export"]) as unknown as { + _defaultCommandName?: string; + }; + expect(group._defaultCommandName).toBeFalsy(); + }); + + test("registers an export subcommand per registered platform", () => { + const registered = findCommand(["migrate", "export"])?.commands.map((c) => c.name()); + for (const key of exportPlatformKeys()) expect(registered).toContain(key); + }); + + test.each(["--output", "--secret-key", "--app", "--instance"])( + "export clerk accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "clerk"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["--domain", "--client-id", "--client-secret", "--output"])( + "export auth0 accepts %s", + (flag) => { + expect(findCommand(["migrate", "export", "auth0"])?.options.map((o) => o.long)).toContain( + flag, + ); + }, + ); + + test.each(["supabase", "authjs", "betterauth"])("export %s accepts --db-url", (platform) => { + expect(findCommand(["migrate", "export", platform])?.options.map((o) => o.long)).toContain( + "--db-url", + ); + }); + + test("export firebase accepts --service-account", () => { + expect(findCommand(["migrate", "export", "firebase"])?.options.map((o) => o.long)).toContain( + "--service-account", + ); + }); + + test.each(exportPlatformKeys())( + "migrate export %s names the run folder as the default output", + (platform) => { + const output = findCommand(["migrate", "export", platform])?.options.find( + (option) => option.long === "--output", + ); + expect(output?.description).toContain("instead of the run folder"); + }, + ); + + test.each(exportPlatformKeys())("migrate export %s accepts --json", (platform) => { + const flags = findCommand(["migrate", "export", platform])?.options.map((o) => o.long); + expect(flags).toContain("--json"); + }); + + test("registers runs with an optional run ID", () => { + const runs = findCommand(["migrate", "runs"]); + expect(runs?.registeredArguments[0]?.required).toBe(false); + expect(runs?.options.map((option) => option.long)).toEqual(["--json", "--runs-dir"]); + }); + + test("registers undo with a required run ID and its flags", () => { + const undo = findCommand(["migrate", "undo"]); + expect(undo?.registeredArguments[0]?.required).toBe(true); + expect(undo?.options.map((option) => option.long)).toEqual([ + "--dry-run", + "--yes", + "--json", + "--secret-key", + "--app", + "--instance", + "--runs-dir", + ]); + }); + + test("the logs group is gone: runs replaces it", () => { + expect(findCommand(["migrate", "logs"])).toBeUndefined(); + }); + + // Every command reads or writes the run store, so each one can be pointed + // somewhere else. + test.each([ + [["import"]], + [["runs"]], + [["undo"]], + [["export"]], + ...exportPlatformKeys().map((platform) => [["export", platform]]), + ])("migrate %p accepts --runs-dir", (names) => { + expect(findCommand(["migrate", ...names])?.options.map((option) => option.long)).toContain( + "--runs-dir", + ); + }); + + // It also takes a path, so it cannot use `.choices()`: completion offers the + // built-in keys through `KNOWN_OPTION_VALUES` instead. + test("--source accepts any value, so a path to a source you wrote gets through", () => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === "--source"); + expect(option?.argChoices).toBeUndefined(); + }); + + test("takes the file, or the export run that wrote it, as an optional argument", () => { + const [argument] = findCommand(["migrate", "import"])?.registeredArguments ?? []; + expect(argument?.name()).toBe("file|export-run-id"); + expect(argument?.required).toBe(false); + }); + + test.each([["-y", "--yes"]])("exposes %s as the short form of %s", (short, long) => { + const option = findCommand(["migrate", "import"])?.options.find((o) => o.long === long); + expect(option?.short).toBe(short); + }); + + // Every export takes `-y`: it is what turns the credential-retry loop off, + // and the loop is on every one of them. + test.each(exportPlatformKeys())("migrate export %s accepts --yes", (platform) => { + const flags = findCommand(["migrate", "export", platform])?.options.map((o) => o.long); + expect(flags).toContain("--yes"); + }); +}); + +/** + * The hook is the only link between the parsed flag and the two places that + * read it, three layers down. If it stopped firing — a Commander upgrade that + * dropped hook inheritance, an action registered outside the group — both + * behaviours would silently revert and every unit test around them would still + * pass, because they set the flag directly. + */ +describe("the migrate group's -y hook", () => { + async function parse(argv: string[]) { + const program = createProgram(); + // `exitOverride` so a usage error inside the action throws here instead of + // taking the test runner down with it; the hook has already run by then. + program.exitOverride(); + try { + await program.parseAsync(["node", "clerk", ...argv]); + } catch { + // The action is allowed to fail — only the hook's effect is under test. + } + return isAssumeYes(); + } + + test("records -y on an export", async () => { + expect(await parse(["migrate", "export", "supabase", "-y", "--db-url", "./none.sqlite"])).toBe( + true, + ); + }); + + // `--json` means nobody reads a prompt, and agent mode is how every prompt in + // this tree already knows to stand down. + test("--json runs the command in agent mode", async () => { + const original = getMode(); + const runsDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-json-")); + try { + setMode("human"); + await parse(["migrate", "runs", "--json", "--runs-dir", runsDir]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + fs.rmSync(runsDir, { recursive: true, force: true }); + } + }); + + // The export group declares --json too, and takes the flag; the hook has to + // read the group's options as well as the subcommand's. + test("--json on an export subcommand runs it in agent mode", async () => { + const original = getMode(); + try { + setMode("human"); + await parse(["migrate", "export", "supabase", "--json", "--db-url", "./none.sqlite"]); + expect(getMode()).toBe("agent"); + } finally { + setMode(original); + } + }); + + test("records its absence, so a previous run cannot leak into this one", async () => { + setAssumeYes(true); + expect(await parse(["migrate", "export", "supabase", "--db-url", "./none.sqlite"])).toBe(false); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/index.ts b/packages/cli-core/src/commands/migrate/index.ts new file mode 100644 index 000000000..383f06cec --- /dev/null +++ b/packages/cli-core/src/commands/migrate/index.ts @@ -0,0 +1,187 @@ +import type { Program } from "../../cli-program.ts"; +import { parseIntegerOption } from "../../lib/option-parsers.ts"; +import { setMode } from "../../mode.ts"; +import { setAssumeYes } from "./lib/assume-yes.ts"; +import { registerMigrateExport } from "./export/index.ts"; +import { RUNS_DIR_DESCRIPTION, RUNS_DIR_FLAG } from "./lib/run-store.ts"; +import { run } from "./run.ts"; +import { runs } from "./runs.ts"; +import { undo } from "./undo.ts"; +import { list as sources } from "./sources/list.ts"; + +const migrate = { run, runs, undo, sources }; + +export function registerMigrate(program: Program): void { + const migrateCommand = program + .command("migrate") + .description("Migrate users into Clerk from another auth provider or another Clerk instance") + .setExamples([ + { + command: "clerk migrate export supabase", + description: "Export users from Supabase into a new run", + }, + { + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check an export against the instance, and write nothing", + }, + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import it", + }, + { command: "clerk migrate runs", description: "List every migration run" }, + { + command: "clerk migrate undo 20260929-141502-a1b2", + description: "Delete the users an import created", + }, + { command: "clerk migrate sources", description: "What each source brings across" }, + ]); + + // `-y` is read several layers down — by the credential-retry loop and the + // export commands — so it is resolved once here rather than threaded + // through every export handler. Hooks are inherited, so this fires for every + // subcommand under `migrate`; one that declares no `-y` resolves to false. + // + // `--json` means nobody is reading a prompt, so it runs the command in agent + // mode: every prompt in this tree already stands down for an agent, with the + // usage error naming what to pass instead. + migrateCommand.hook("preAction", (_thisCommand, actionCommand) => { + // With globals: `export auth0 --json` lands on the export group's own + // --json, which the subcommand's opts() never sees. + const opts = actionCommand.optsWithGlobals(); + setAssumeYes(Boolean(opts.yes)); + if (opts.json) setMode("agent"); + }); + + // Named, not `isDefault`. `import` and `export` are the two directions this + // group moves users in, and neither is implied by the bare group name — a + // default would make `clerk migrate --file users.json` mean "import" while + // its sibling has to be spelled out. Bare `clerk migrate` prints help. + // + // The flags stay here rather than on `migrate`, matching how `config` keeps + // its own on `pull`/`patch`/`put` — a group's help is a list of subcommands + // and examples, not a merge of everything underneath it. + migrateCommand + .command("import") + .description("Import users from an exported JSON or CSV file") + .argument("[file|export-run-id]", "The export file, or the ID of the export run that wrote it") + .option( + "--source ", + "Where the file came from: a built-in source, or a source you wrote. Not needed for a file from `clerk migrate export`", + ) + .option("--dry-run", "Check the file against the instance, report, and write nothing") + .option("--allow-partial", "Import the users that pass the checks, and skip the rest") + .option("--new-run", "Start a new run instead of continuing an earlier one of this file") + .option("--require-password", "Import only users that have a password") + .option( + "--skip-legal-checks", + "Import users with no legal acceptance on record into an instance that requires it", + ) + .option( + "--reserve-unverified", + "Create emails and phones the source never verified as reserved (usable for sign-in, locked to the user) instead of unverified", + ) + .option("--firebase-signer-key ", "Firebase base64 signer key (overrides the export file)") + .option("--firebase-salt-separator ", "Firebase base64 salt separator") + .option("--firebase-rounds ", "Firebase scrypt rounds", (value: string) => + parseIntegerOption(value, "--firebase-rounds", { min: 1 }), + ) + .option("--firebase-mem-cost ", "Firebase scrypt memory cost", (value: string) => + parseIntegerOption(value, "--firebase-mem-cost", { min: 1 }), + ) + .option("-y, --yes", "Import without prompting") + .option("--json", "Output as JSON; never prompts, so pair it with --yes to import") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { + command: "clerk migrate import 20260929-141502-a1b2 --dry-run", + description: "Check what an export run wrote, and write nothing", + }, + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import it. Run it again to continue after a failure", + }, + { + command: "clerk migrate import users.json --source clerk --allow-partial --yes", + description: "Import a Clerk Dashboard export, skipping users that would be rejected", + }, + { + command: "clerk migrate import users.json --source ./my-source.ts --yes", + description: "Import with a source you wrote", + }, + ]) + .action(async (input, _opts, cmd) => + migrate.run({ + ...(cmd.optsWithGlobals() as Parameters[0]), + ...(input ? { input } : {}), + }), + ); + + registerMigrateExport(migrateCommand); + + // Flat, not under a noun group: this is the one command in the tree that + // destroys data in Clerk, and it is worth keeping short and prominent. + migrateCommand + .command("undo") + .description("Delete the users an import run created") + .argument("", "The import run to undo (see `clerk migrate runs`)") + .option("--dry-run", "Show what would be deleted, and delete nothing") + .option("-y, --yes", "Delete without prompting") + .option("--json", "Output as JSON; never prompts, so pair it with --yes to delete") + .option("--secret-key ", "Backend API secret key to use") + .option("--app ", "Application ID to target (works from any directory)") + .option("--instance ", "Instance to target (dev, prod, or a full instance ID)") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { + command: "clerk migrate undo 20260929-141502-a1b2 --dry-run", + description: "Preview what would be deleted", + }, + { + command: "clerk migrate undo 20260929-141502-a1b2 --yes", + description: "Delete without prompting", + }, + ]) + .action(async (runId, _opts, cmd) => + migrate.undo(runId, cmd.optsWithGlobals() as Parameters[1]), + ); + + migrateCommand + .command("runs") + .description("List migration runs, or show one") + .argument("[run-id]", "A run to show in full") + .option("--json", "Output as JSON") + .option(RUNS_DIR_FLAG, RUNS_DIR_DESCRIPTION) + .setExamples([ + { command: "clerk migrate runs", description: "List every run, newest first" }, + { + command: "clerk migrate runs 20260929-141502-a1b2", + description: "Show one run: counts, errors and the users that did not make it", + }, + { command: "clerk migrate runs --json", description: "Machine-readable listing" }, + ]) + .action(async (runId, _opts, cmd) => + migrate.runs(runId, cmd.optsWithGlobals() as Parameters[1]), + ); + + // A compiled binary has no source tree to grep, so the available mappings + // need a command rather than only appearing in the interactive picker. + migrateCommand + .command("sources") + .description("List the sources an import can read, or show one in full") + .argument("[source]", "A built-in source, or the path to a source you wrote") + .option("--json", "Output as JSON") + .setExamples([ + { command: "clerk migrate sources", description: "What each source brings across" }, + { + command: "clerk migrate sources betterauth", + description: "Where each field lands, how to export, and caveats", + }, + { command: "clerk migrate sources ./my-source.ts", description: "Check a source you wrote" }, + ]) + .action(async (source, _opts, cmd) => + migrate.sources(source, cmd.optsWithGlobals() as Parameters[1]), + ); +} diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.test.ts b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts new file mode 100644 index 000000000..322c2d753 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { analyzeFields, hasValue } from "./analysis.ts"; + +describe("hasValue", () => { + test.each([ + ["a string", "x", true], + ["zero", 0, true], + ["false", false, true], + ["a populated array", ["a"], true], + ["an object", {}, true], + ["an empty string", "", false], + ["an empty array", [], false], + ["null", null, false], + ["undefined", undefined, false], + ])("treats %s as present: %p", (_label, value, expected) => { + expect(hasValue(value)).toBe(expected); + }); +}); + +describe("analyzeFields", () => { + test("returns zeroed counts for an empty file", () => { + const result = analyzeFields([]); + expect(result.totalUsers).toBe(0); + expect(result.identifiers.hasAnyIdentifier).toBe(0); + expect(result.fieldCounts).toEqual({}); + }); + + test("counts each identifier kind separately", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", unverifiedEmailAddresses: ["b@x.dev"] }, + { userId: "3", phone: "+15555550100" }, + { userId: "4", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "5", username: "carol" }, + ]); + + expect(result.identifiers).toMatchObject({ + verifiedEmails: 1, + unverifiedEmails: 1, + verifiedPhones: 1, + unverifiedPhones: 1, + username: 1, + hasAnyIdentifier: 5, + }); + }); + + test("counts emailAddresses towards verified emails", () => { + const result = analyzeFields([{ userId: "1", emailAddresses: ["a@x.dev"] }]); + expect(result.identifiers.verifiedEmails).toBe(1); + }); + + test("counts a user with several identifiers once", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", phone: "+15555550100", username: "ada" }, + ]); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + expect(result.identifiers.verifiedEmails).toBe(1); + expect(result.identifiers.verifiedPhones).toBe(1); + }); + + // These users cannot be imported under any instance configuration, which is + // what makes the count worth surfacing separately. + test("counts users carrying no identifier at all", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev" }, + { userId: "2", firstName: "Nobody" }, + { userId: "3" }, + ]); + expect(result.totalUsers).toBe(3); + expect(result.identifiers.hasAnyIdentifier).toBe(1); + }); + + test("counts the analyzed non-identifier fields", () => { + const result = analyzeFields([ + { userId: "1", email: "a@x.dev", firstName: "Ada", password: "d", totpSecret: "s" }, + { userId: "2", email: "b@x.dev", firstName: "Grace" }, + { userId: "3", email: "c@x.dev", lastName: "Hopper" }, + ]); + expect(result.fieldCounts).toEqual({ + firstName: 2, + lastName: 1, + password: 1, + totpSecret: 1, + }); + }); + + test("omits fields no user carries, rather than reporting them as zero", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev" }]); + expect("password" in result.fieldCounts).toBe(false); + }); + + test("does not count an empty value as present", () => { + const result = analyzeFields([{ userId: "1", email: "a@x.dev", firstName: "", lastName: [] }]); + expect(result.fieldCounts.firstName).toBeUndefined(); + expect(result.fieldCounts.lastName).toBeUndefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/analysis.ts b/packages/cli-core/src/commands/migrate/lib/analysis.ts new file mode 100644 index 000000000..be371127e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/analysis.ts @@ -0,0 +1,88 @@ +/** + * Counts what an import file actually contains, per field. + * + * Ported from the standalone migration-tool's `src/lib/analysis.ts`. Runs on + * transformed-but-unvalidated users so the report describes the whole file, + * including the rows that will be skipped. + */ + +import type { User } from "../types.ts"; + +/** Non-identifier fields the readiness rows report coverage for. */ +export const ANALYZED_FIELDS = [ + { key: "firstName", label: "First name" }, + { key: "lastName", label: "Last name" }, + { key: "password", label: "Password" }, + { key: "totpSecret", label: "TOTP secret" }, +] as const; + +export type IdentifierCounts = { + verifiedEmails: number; + unverifiedEmails: number; + verifiedPhones: number; + unverifiedPhones: number; + username: number; + /** Users with any email, verified or not: each user counted once. */ + anyEmail: number; + /** Users with any phone, verified or not: each user counted once. */ + anyPhone: number; + /** Users with at least one identifier — the rest cannot be imported at all. */ + hasAnyIdentifier: number; +}; + +export type FieldAnalysis = { + identifiers: IdentifierCounts; + totalUsers: number; + fieldCounts: Record; +}; + +/** True for anything with real content — `0` and `false` count, `""` and `[]` do not. */ +export function hasValue(value: unknown): boolean { + if (value === undefined || value === null || value === "") return false; + if (Array.isArray(value)) return value.length > 0; + return true; +} + +export function analyzeFields(users: (User | Record)[]): FieldAnalysis { + const identifiers: IdentifierCounts = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + anyEmail: 0, + anyPhone: 0, + hasAnyIdentifier: 0, + }; + const fieldCounts: Record = {}; + + for (const entry of users) { + const user = entry as Record; + + for (const field of ANALYZED_FIELDS) { + if (hasValue(user[field.key])) { + fieldCounts[field.key] = (fieldCounts[field.key] ?? 0) + 1; + } + } + + const verifiedEmail = hasValue(user.email) || hasValue(user.emailAddresses); + const unverifiedEmail = hasValue(user.unverifiedEmailAddresses); + const verifiedPhone = hasValue(user.phone) || hasValue(user.phoneNumbers); + const unverifiedPhone = hasValue(user.unverifiedPhoneNumbers); + const username = hasValue(user.username); + + if (verifiedEmail) identifiers.verifiedEmails++; + if (unverifiedEmail) identifiers.unverifiedEmails++; + if (verifiedPhone) identifiers.verifiedPhones++; + if (unverifiedPhone) identifiers.unverifiedPhones++; + if (username) identifiers.username++; + if (verifiedEmail || unverifiedEmail) identifiers.anyEmail++; + if (verifiedPhone || unverifiedPhone) identifiers.anyPhone++; + + if (verifiedEmail || unverifiedEmail || verifiedPhone || unverifiedPhone || username) { + identifiers.hasAnyIdentifier++; + } + } + + return { identifiers, totalUsers: users.length, fieldCounts }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/assume-yes.ts b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts new file mode 100644 index 000000000..dd277dad0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/assume-yes.ts @@ -0,0 +1,32 @@ +/** + * Whether this run was given `-y`. + * + * `-y` is not the same question as {@link isAgent}. Agent mode says the CLI + * *cannot* prompt; `-y` says the operator does not want it to. Most prompts + * only care about the first — a confirm is skipped by either — but the two + * places that take a default instead of asking need to know a human chose it, + * so the two cannot be collapsed into one flag. + * + * Held per-run rather than threaded through, because the readers are three + * layers below the command that parses it: `withInputRetry` sits under every + * credential prompt, and the export commands read it to decide what to print. + * Passing it down would put a `yes` parameter on every export handler + * signature on the way. This mirrors `mode.ts`, which resolves + * `--mode` once in a `preAction` hook and is read the same way. + * + * Set by the `migrate` group's `preAction` hook, so every subcommand under it + * is covered whether or not it declares the flag — one that does not simply + * resolves to `false`. + */ + +let assumeYes = false; + +/** Records this run's `-y`. Called once per invocation, before the action. */ +export function setAssumeYes(value: boolean): void { + assumeYes = value; +} + +/** Whether `-y` was passed to the command now running. */ +export function isAssumeYes(): boolean { + return assumeYes; +} diff --git a/packages/cli-core/src/commands/migrate/lib/checks.test.ts b/packages/cli-core/src/commands/migrate/lib/checks.test.ts new file mode 100644 index 000000000..ba18b77ef --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/checks.test.ts @@ -0,0 +1,1047 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { User } from "../types.ts"; +import { checkImport, hashShapeProblem, passwordIsOnlySignIn, type CheckInput } from "./checks.ts"; + +const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; + +/** Settings as given, with no way to sign in but what they list. */ +const bare = (attributes: object, social: object = {}) => + ({ attributes, social }) as unknown as UserSettingsJSON; + +/** + * Settings with a way to sign in besides a password (SSO), so users without + * one import: Clerk refuses them on an instance with no other way in. + */ +const settings = (attributes: object, social: object = {}) => + ({ ...bare(attributes, social), enterprise_sso: { enabled: true } }) as UserSettingsJSON; + +const EMAIL_REQUIRED = settings({ email_address: { enabled: true, required: true } }); + +const user = (userId: string, fields: Partial = {}): User => + ({ userId, email: `${userId}@x.dev`, ...fields }) as User; + +let originalFetch: typeof globalThis.fetch; +/** Users `GET /v1/users` finds, whatever the filter. */ +let existing: Record[]; + +beforeAll(() => { + originalFetch = globalThis.fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +beforeEach(() => { + existing = []; + globalThis.fetch = (async (input: string | URL | Request) => { + const url = new URL(input.toString()); + // BAPI strips the `+` the lookup puts on each external_id. + const wanted = new Set([...url.searchParams.values()].map((value) => value.replace(/^\+/, ""))); + return Response.json( + existing.filter((candidate) => + [ + candidate.external_id, + candidate.username, + ...((candidate.email_addresses as { email_address: string }[] | undefined) ?? []).map( + (email) => email.email_address, + ), + ].some((value) => wanted.has(value as string)), + ), + ); + }) as typeof fetch; +}); + +function input(overrides: Partial = {}): CheckInput { + return { + users: [], + failures: [], + settings: null, + instanceType: "prod", + target: { + env: "production", + instanceId: "ins_1", + instanceType: "prod", + keySource: "--secret-key", + }, + secretKey: "sk_live_x", + schedule: Object.assign(async (fn: () => Promise) => fn(), { pause: () => {} }), + ...overrides, + }; +} + +const reasonsOf = async (overrides: Partial) => + Object.fromEntries( + (await checkImport(input(overrides))).rejects.map((reject) => [reject.sourceId, reject.reason]), + ); + +describe("rejects", () => { + // Each one is a user POST /v1/users refuses (clerk_go create_service.go). + describe("what the create refuses", () => { + const EMAIL_ON = { email_address: { enabled: true } }; + + test.each([ + ["first_name", { lastName: "L" }, "no first name, which this instance requires"], + ["last_name", { firstName: "F" }, "no last name, which this instance requires"], + ] as const)("a missing required %s", async (attribute, fields, reason) => { + const reasons = await reasonsOf({ + users: [user("a", fields), user("b", { firstName: "F", lastName: "L" })], + settings: settings({ ...EMAIL_ON, [attribute]: { enabled: true, required: true } }), + }); + expect(reasons).toEqual({ a: reason }); + }); + + test.each([ + [ + "totpSecret", + "authenticator_app", + "has an authenticator app (TOTP) secret, and this instance has authenticator apps off", + ], + ["backupCodes", "backup_code", "has backup codes, and this instance has backup codes off"], + ] as const)("%s with %s off, offering to turn it on", async (field, attribute, reason) => { + const value = field === "backupCodes" ? ["code1"] : "SECRET"; + const checks = await checkImport( + input({ + users: [user("a", { [field]: value })], + settings: settings({ ...EMAIL_ON, [attribute]: { enabled: false } }), + }), + ); + expect(checks.rejects).toEqual([{ sourceId: "a", reason }]); + expect(checks.fixes.map((fix) => fix.command).join("\n")).toContain( + `"auth_multi_factor":{"${attribute}":{"enabled":true}}`, + ); + }); + + test("no password where password is the only way to sign in", async () => { + const passwordOnly = bare({ + email_address: { enabled: true, used_for_first_factor: false, first_factors: [] }, + // Clerk's real shape: a password is never itself a listed first factor. + password: { enabled: true, used_for_first_factor: false, first_factors: [] }, + }); + const reasons = await reasonsOf({ + users: [user("a"), user("b", { password: BCRYPT, passwordHasher: "bcrypt" })], + settings: passwordOnly, + }); + expect(reasons).toEqual({ + a: "no password, and password is this instance's only way to sign in", + }); + }); + + // Firebase or Supabase phone-auth users, into an instance with phone off. + test("a user left with no identifier once disabled ones are stripped", async () => { + const reasons = await reasonsOf({ + users: [user("phone-only", { email: undefined, phone: "+15555550100" }), user("b")], + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + }); + expect(reasons).toEqual({ + "phone-only": + "has no identifier this instance accepts (its email or phone is turned off, or its username is one Clerk refuses)", + }); + }); + + describe("legal consent", () => { + const LEGAL = { + ...settings(EMAIL_ON), + sign_up: { legal_consent_enabled: true }, + } as unknown as UserSettingsJSON; + const users = [user("a"), user("b", { legalAcceptedAt: "2024-01-01T00:00:00.000Z" })]; + + test("rejects a user with no acceptance on record, naming the flag", async () => { + expect((await reasonsOf({ users, settings: LEGAL })).a).toContain("--skip-legal-checks"); + }); + + test("with skipLegalChecks, imports them with skip_legal_checks and a warning", async () => { + const checks = await checkImport(input({ users, settings: LEGAL, skipLegalChecks: true })); + expect(checks.rejects).toEqual([]); + expect(checks.importable.map((u) => u.skipLegalChecks)).toEqual([true, undefined]); + expect(checks.warnings.join("\n")).toContain("1 user has no legal acceptance on record"); + }); + }); + }); + + test("a user that failed validation", async () => { + const checks = await checkImport( + input({ failures: [{ userId: "bad", row: 0, error: "Invalid email", path: ["email"] }] }), + ); + expect(checks.rejects).toEqual([{ sourceId: "bad", reason: "invalid: Invalid email" }]); + expect(checks.total).toBe(1); + }); + + test("a duplicate source ID, email, phone or username within the file", async () => { + expect( + await reasonsOf({ + users: [ + user("a"), + user("a", { email: "other@x.dev" }), + user("b", { email: "A@x.dev" }), + user("c", { phone: "+15555550100" }), + user("d", { phone: "+15555550100" }), + // The same number, punctuated. + user("f", { phone: "+1 555-555-0100" }), + user("g", { username: "Ada" }), + // Clerk lowercases usernames. + user("h", { username: "ada" }), + ], + }), + ).toEqual({ + a: "duplicate source ID in the file", + b: "email is also used by an earlier user in the file, which is kept", + d: "phone number is also used by an earlier user in the file, which is kept", + f: "phone number is also used by an earlier user in the file, which is kept", + h: "username is also used by an earlier user in the file, which is kept", + }); + }); + + // Only what the create carries can clash: an extra email is attached after + // it, and a clash there is a note on the user, not a failure. + test("an extra email or phone shared with an earlier user is no duplicate", async () => { + expect( + await reasonsOf({ + users: [ + user("a", { + emailAddresses: ["shared@x.dev"], + phone: ["+15555550101", "+15555550100"], + }), + user("b", { + emailAddresses: ["shared@x.dev"], + phone: ["+15555550102", "+15555550100"], + }), + ], + }), + ).toEqual({}); + }); + + // A stripped email never goes out, so it cannot clash with anything. + test("an email the instance would strip is no duplicate", async () => { + existing = [{ id: "user_1", email_addresses: [{ email_address: "taken@x.dev" }] }]; + const users = [ + user("a", { email: "same@x.dev", username: "a" }), + user("b", { email: "same@x.dev", username: "b" }), + user("c", { email: "taken@x.dev", username: "c" }), + ]; + const checks = await checkImport( + input({ users, settings: settings({ username: { enabled: true } }) }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable.map((u) => [u.userId, u.email])).toEqual([ + ["a", undefined], + ["b", undefined], + ["c", undefined], + ]); + // The warning still describes the file: the emails are dropped. + expect(checks.warnings.join("\n")).toContain( + "3 users have an email, which this instance is not set up to store", + ); + }); + + // --reserve-unverified puts unverified identifiers on the create, so they + // clash there as a primary would; without it they are attached after. + test.each([ + [ + true, + { + b: "email is also used by an earlier user in the file, which is kept", + c: "email is already used by a user in the instance", + }, + ], + [false, {}], + ])( + "with reserveUnverified %p, unverified emails that clash are duplicates", + async (reserveUnverified, expected) => { + existing = [{ id: "user_1", email_addresses: [{ email_address: "taken@x.dev" }] }]; + expect( + await reasonsOf({ + reserveUnverified, + users: [ + user("a", { unverifiedEmailAddresses: ["shared@x.dev"] }), + user("b", { unverifiedEmailAddresses: ["shared@x.dev"] }), + user("c", { unverifiedEmailAddresses: ["taken@x.dev"] }), + ], + }), + ).toEqual(expected); + }, + ); + + // An adopted user's create already ran without reserving, so its unverified + // email is attached later, whatever this run's flag says: no clash. + test("an adopted user's unverified email counts by its own create's mode", async () => { + expect( + await reasonsOf({ + reserveUnverified: true, + adoptedSourceIds: new Set(["b"]), + users: [ + user("a", { unverifiedEmailAddresses: ["shared@x.dev"] }), + user("b", { unverifiedEmailAddresses: ["shared@x.dev"] }), + ], + }), + ).toEqual({}); + }); + + // The source's order decides which duplicate survives, so the dry run says. + test("a duplicate names the earlier user kept in its place", async () => { + const checks = await checkImport( + input({ + users: [user("email|1", { email: "a@x.dev" }), user("auth0|1", { email: "a@x.dev" })], + }), + ); + expect(checks.rejects).toEqual([ + { + sourceId: "auth0|1", + reason: "email is also used by an earlier user in the file, which is kept", + keptSourceId: "email|1", + }, + ]); + }); + + // G20: an unverified email is attached after the user exists, so it cannot + // satisfy a sign-up requirement. + test("a user with only an unverified email, where email is required", async () => { + expect( + await reasonsOf({ + settings: EMAIL_REQUIRED, + users: [ + user("ok"), + user("unverified", { email: undefined, unverifiedEmailAddresses: ["u@x.dev"] }), + user("none", { email: undefined, username: "none" }), + ], + }), + ).toEqual({ + unverified: "only has an unverified email, and this instance requires an email", + none: "no email, which this instance requires", + }); + }); + + // Reserved is usable for sign-in, so it meets the requirement (checked live + // against the E2E test app, which requires an email). + test("a reserved email meets an email requirement", async () => { + expect( + await reasonsOf({ + settings: EMAIL_REQUIRED, + reserveUnverified: true, + users: [user("unverified", { email: undefined, unverifiedEmailAddresses: ["u@x.dev"] })], + }), + ).toEqual({}); + }); + + test("a password that is not the shape its hasher says", async () => { + expect( + await reasonsOf({ + users: [ + user("good", { password: BCRYPT, passwordHasher: "bcrypt" }), + user("bad", { password: "not-a-hash", passwordHasher: "bcrypt" }), + ], + }), + ).toEqual({ + bad: "password is not a bcrypt hash Clerk can verify ($2a$/$2b$/$2y$, cost 4 to 15, 60 characters)", + }); + }); + + test("a user already in the instance, by source ID, email or username", async () => { + existing = [ + { id: "user_1", external_id: "by-id" }, + { id: "user_2", email_addresses: [{ email_address: "by-email@x.dev" }] }, + { id: "user_3", username: "taken" }, + ]; + + expect( + await reasonsOf({ + users: [ + user("by-id"), + user("by-email"), + user("by-username", { username: "Taken" }), + user("new"), + ], + }), + ).toEqual({ + "by-id": "already in the instance, with this source ID", + "by-email": "email is already used by a user in the instance", + "by-username": "username is already taken in the instance", + }); + }); + + // A repeated record (an export that paged past a sign-up) must not take the + // kept copy down with it. + // Adopted means created by this run, but only for its own record: another + // user that shares its email would still clash at create. + test("an adopted user's identifiers still clash with other users", async () => { + existing = [ + { + id: "user_mine", + external_id: "mine", + email_addresses: [{ email_address: "mine@x.dev" }, { email_address: "shared@x.dev" }], + }, + ]; + expect( + await reasonsOf({ + users: [ + user("mine", { emailAddresses: ["shared@x.dev"] }), + user("other", { email: "shared@x.dev" }), + ], + adoptedClerkIds: new Set(["user_mine"]), + }), + ).toEqual({ other: "email is already used by a user in the instance" }); + }); + + test("a repeated source ID rejects only the later copy", async () => { + const checks = await checkImport(input({ users: [user("a"), user("a")] })); + expect(checks.importable.map((u) => u.userId)).toEqual(["a"]); + expect(checks.rejects).toEqual([{ sourceId: "a", reason: "duplicate source ID in the file" }]); + }); + + test("a rejected user does not claim its email from a later one", async () => { + const checks = await checkImport( + input({ + users: [ + user("skipped", { email: "same@x.dev", skipReason: "anonymous Better Auth user" }), + user("kept", { email: "same@x.dev" }), + ], + }), + ); + expect(checks.importable.map((u) => u.userId)).toEqual(["kept"]); + }); + + // A continued run found this user behind its own in-flight create. + test("not a user the continued run adopted", async () => { + existing = [{ id: "user_1", external_id: "mine" }]; + + expect( + await reasonsOf({ users: [user("mine")], adoptedClerkIds: new Set(["user_1"]) }), + ).toEqual({}); + }); + + test("a supabase user whose only provider is disabled", async () => { + const rows = [ + { id: "only-discord", raw_app_meta_data: { providers: ["discord"] } }, + { id: "has-email", raw_app_meta_data: { providers: ["email", "discord"] } }, + ]; + expect( + await reasonsOf({ + settings: settings( + { email_address: { enabled: true } }, + { oauth_google: { enabled: true } }, + ), + supabaseRows: rows, + users: [user("only-discord"), user("has-email")], + }), + ).toEqual({ "only-discord": "only signs in with Discord, which is not enabled in Clerk" }); + }); + + // A disabled provider strands only a user with no other way in. + test("a supabase user on a disabled provider who can sign in by email or phone code", async () => { + const rows = [ + { id: "email", raw_app_meta_data: { providers: ["discord"] } }, + { id: "phone", raw_app_meta_data: { providers: ["discord"] } }, + { id: "neither", raw_app_meta_data: { providers: ["discord"] } }, + ]; + expect( + await reasonsOf({ + settings: settings({ + email_address: { + enabled: true, + used_for_first_factor: true, + first_factors: ["email_code"], + }, + phone_number: { + enabled: true, + used_for_first_factor: true, + first_factors: ["phone_code"], + }, + username: { enabled: true }, + }), + supabaseRows: rows, + users: [ + user("email"), + user("phone", { email: undefined, phone: "+15555550100" }), + user("neither", { email: undefined, username: "neither" }), + ], + }), + ).toEqual({ neither: "only signs in with Discord, which is not enabled in Clerk" }); + }); + + test("an unverified email is no way in for a supabase user on a disabled provider", async () => { + expect( + await reasonsOf({ + settings: settings({ + email_address: { + enabled: true, + used_for_first_factor: true, + first_factors: ["email_link"], + }, + }), + supabaseRows: [{ id: "u", raw_app_meta_data: { providers: ["discord"] } }], + users: [user("u", { email: undefined, unverifiedEmailAddresses: ["u@x.dev"] })], + }), + ).toEqual({ u: "only signs in with Discord, which is not enabled in Clerk" }); + }); + + // Clerk can't turn on a provider it doesn't offer, so none is suggested. + test("a provider Clerk doesn't offer is named as such, with no fix", async () => { + const rows = [ + { id: "figma", raw_app_meta_data: { providers: ["figma"] } }, + { id: "both", raw_app_meta_data: { providers: ["discord", "figma"] } }, + ]; + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true } }), + supabaseRows: rows, + users: [user("figma"), user("both")], + }), + ); + expect(Object.fromEntries(checks.rejects.map((r) => [r.sourceId, r.reason]))).toEqual({ + figma: "only signs in with Figma, which is not offered by Clerk", + both: "only signs in with Discord (not enabled in Clerk), Figma (not offered by Clerk)", + }); + expect(checks.fixes.map((fix) => fix.label)).not.toContain("Enable Figma sign-in"); + }); + + // Each reject names that user's providers, not every disabled one in the file. + test("a supabase reject names only that user's own providers", async () => { + const rows = [ + { id: "discord", raw_app_meta_data: { providers: ["discord"] } }, + { id: "twitch", raw_app_meta_data: { providers: ["twitch"] } }, + ]; + expect( + await reasonsOf({ + settings: settings({ email_address: { enabled: true } }), + supabaseRows: rows, + users: [user("discord"), user("twitch")], + }), + ).toEqual({ + discord: "only signs in with Discord, which is not enabled in Clerk", + twitch: "only signs in with Twitch, which is not enabled in Clerk", + }); + }); + + test("the users past a development instance's headroom, in file order", async () => { + const checks = await checkImport( + input({ + instanceType: "dev", + existingUsers: 98, + users: [user("a"), user("b"), user("c")], + }), + ); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["a", "b"]); + expect(checks.rejects).toEqual([ + { + sourceId: "c", + reason: + "over the development instance's 100-user limit (raised by Clerk? set CLERK_MIGRATE_DEV_USER_LIMIT)", + }, + ]); + expect(checks.quota).toEqual({ existing: 98, limit: 100, headroom: 2, over: 1 }); + }); + + // A continued run's adopted users exist already and are never created + // again: the live count has them, and only new users take headroom. + test("an adopted user takes no headroom, wherever it sits in the file", async () => { + const checks = await checkImport( + input({ + instanceType: "dev", + existingUsers: 99, + adoptedClerkIds: new Set(["user_c"]), + adoptedSourceIds: new Set(["c"]), + users: [user("a"), user("b"), user("c")], + }), + ); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["a", "c"]); + expect(checks.rejects.map((reject) => reject.sourceId)).toEqual(["b"]); + expect(checks.quota).toEqual({ existing: 99, limit: 100, headroom: 1, over: 1 }); + }); + + // The checks stop creates; an adopted user needs none, so a reject would + // only leave it `creating` for good. + test("an adopted user is never rejected", async () => { + const checks = await checkImport( + input({ + settings: { + ...settings({ email_address: { enabled: true } }), + sign_up: { legal_consent_enabled: true }, + } as never, + adoptedSourceIds: new Set(["a"]), + users: [user("a"), user("b")], + }), + ); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["a"]); + expect(checks.rejects.map((reject) => reject.sourceId)).toEqual(["b"]); + }); + + test("warns that an unreadable user count was checked as empty", async () => { + const checks = await checkImport( + input({ instanceType: "dev", existingUsers: null, users: [user("a")] }), + ); + expect(checks.warnings.join("\n")).toContain("Could not read how many users"); + }); + + test("CLERK_MIGRATE_DEV_USER_LIMIT raises the headroom", async () => { + process.env.CLERK_MIGRATE_DEV_USER_LIMIT = "500"; + try { + const checks = await checkImport( + input({ instanceType: "dev", existingUsers: 98, users: [user("a"), user("b"), user("c")] }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.quota).toEqual({ existing: 98, limit: 500, headroom: 402, over: 0 }); + } finally { + delete process.env.CLERK_MIGRATE_DEV_USER_LIMIT; + } + }); + + test("a user its source asked to skip, with the source's reason", async () => { + const checks = await checkImport( + input({ users: [user("a", { skipReason: "anonymous Better Auth user" }), user("b")] }), + ); + expect(checks.rejects).toEqual([{ sourceId: "a", reason: "anonymous Better Auth user" }]); + expect(checks.importable.map((entry) => entry.userId)).toEqual(["b"]); + }); + + describe("emails Clerk refuses", () => { + test("a user whose only identifier is a placeholder email is rejected", async () => { + const checks = await checkImport( + input({ users: [user("a", { email: "15551234@phone.local" })] }), + ); + // One fixed reason, so these group in the report and the address + // itself stays out of users.ndjson. + expect(checks.rejects).toEqual([ + { + sourceId: "a", + reason: "only has emails Clerk refuses (malformed, or a domain that can't receive mail)", + }, + ]); + }); + + test.each([ + ["a@localhost", "no dotted domain"], + ["not-an-email", "no @"], + ["a b@x.dev", "a space"], + ["ada@corp.internal", "a private TLD"], + ])("drops %p (%s) and keeps the user on its other email", async (bad) => { + const checks = await checkImport( + input({ users: [user("a", { email: "a@x.dev", emailAddresses: [bad] })] }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable[0]?.emailAddresses).toBeUndefined(); + }); + + // clerk_go accepts a non-ASCII local part; Zod's email check did not. + test("keeps a non-ASCII address", async () => { + const checks = await checkImport(input({ users: [user("a", { email: "josé@x.dev" })] })); + expect(checks.importable).toEqual([user("a", { email: "josé@x.dev" })]); + }); + + test("a placeholder email is dropped, and the user imports on what is left", async () => { + const checks = await checkImport( + input({ + users: [ + user("a", { + email: "a@x.dev", + unverifiedEmailAddresses: ["anon-1@anonymous.invalid"], + }), + ], + }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a", { email: "a@x.dev" })]); + expect(checks.warnings).toContain( + "1 user has an email Clerk refuses (malformed, or a domain such as .local or .invalid), which is dropped", + ); + }); + + test("Clerk's own .clerk.test addresses are kept", async () => { + const checks = await checkImport( + input({ users: [user("a", { email: "a@dev.clerk.test" })] }), + ); + expect(checks.importable).toEqual([user("a", { email: "a@dev.clerk.test" })]); + }); + }); + + describe("names Clerk refuses", () => { + test.each([ + ["+447836887904"], + ["4165550123"], + ["https://spam.example"], + ["see x.com/win"], + ["Ada"], + [" "], + ["\u200b"], + ["é".repeat(129)], + ])("%p is dropped, and the user still imports", async (firstName) => { + const checks = await checkImport(input({ users: [user("a", { firstName, lastName: "L" })] })); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a", { lastName: "L" })]); + expect(checks.warnings).toContain( + "1 user has a name Clerk refuses (a phone number, a URL, HTML, blank, or over 256 bytes), which is dropped", + ); + }); + + // Clerk accepts an email as a name, and a URL that is part of one. + test.each([ + ["Ada"], + ["Mary-Jane O'Neil"], + ["Louis XIV"], + ["Agent 007"], + ["redacted.io"], + ["ada@x.dev"], + ["ada@x.dev/x"], + // 256 bytes exactly. + ["é".repeat(128)], + ])("%p is kept", async (firstName) => { + const checks = await checkImport(input({ users: [user("a", { firstName })] })); + expect(checks.importable).toEqual([user("a", { firstName })]); + }); + }); + + describe("usernames", () => { + const withUsernames = (rules: object) => + ({ + attributes: { email_address: { enabled: true }, username: { enabled: true } }, + social: {}, + enterprise_sso: { enabled: true }, + username_settings: { min_length: 4, max_length: 64, ...rules }, + }) as unknown as UserSettingsJSON; + const reasonFor = async (username: string, rules: object = {}) => + ( + await checkImport( + input({ settings: withUsernames(rules), users: [user("a", { username })] }), + ) + ).rejects[0]?.reason; + + test.each([ + ["ada_l-1", {}], + ["ada.l", { allow_extended_special_characters: true }], + ])("%p is accepted", async (username, rules) => { + expect(await reasonFor(username, rules)).toBeUndefined(); + }); + + test("a . needs extended special characters", async () => { + expect(await reasonFor("ada.lovelace")).toContain("turn on extended special characters"); + }); + + test.each([ + ["ada", "4–64 characters"], + ["12345", "no letters"], + ["ada@x", "characters Clerk does not allow"], + ])("%p is rejected: %s", async (username, reason) => { + expect(await reasonFor(username)).toContain(reason); + }); + + // Clerk checks a username with usernames off too, but nothing signs in + // with it there, so it costs the user nothing to lose it. + test("with usernames off, one Clerk would refuse is dropped, not rejected", async () => { + const checks = await checkImport( + input({ settings: EMAIL_REQUIRED, users: [user("a", { username: "a.b" })] }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a")]); + expect(checks.warnings).toContain( + "1 user has a username Clerk refuses, which is dropped: this instance has usernames off", + ); + }); + + // Clerk never checks whether usernames are on (create_service.go). + test("with usernames off, a valid one is kept and said to be stored", async () => { + const checks = await checkImport( + input({ settings: EMAIL_REQUIRED, users: [user("a", { username: "ada" })] }), + ); + expect(checks.importable).toEqual([user("a", { username: "ada" })]); + expect(checks.warnings).toContain( + "1 user has a username, which this instance does not use: it is stored, and works only once usernames are turned on", + ); + }); + }); + + test("groups the rejects by reason", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + users: [ + user("a", { email: undefined, username: "a" }), + user("b", { email: undefined, username: "b" }), + ], + }), + ); + expect(checks.rejectReasons).toEqual([ + { reason: "no email, which this instance requires", count: 2 }, + ]); + }); +}); + +describe("warnings", () => { + test("a field the instance is not set up to store", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + users: [user("a", { phone: "+15555550100" })], + }), + ); + expect(checks.warnings).toContain( + "1 user has a phone number, which this instance is not set up to store", + ); + expect(checks.rejects).toEqual([]); + }); + + // Clerk refuses the whole create when it is sent an identifier the instance + // has off, so the dropped phone has to actually be dropped. + test("strips identifiers the instance has off from the importable users", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, phone_number: { enabled: false } }), + users: [ + user("a", { + phone: "+15555550100", + unverifiedPhoneNumbers: ["+15555550101"], + username: "ada.l", + }), + ], + }), + ); + expect(checks.importable).toEqual([user("a")]); + }); + + // Clerk keeps an email or phone the instance signs in or does MFA with, + // whatever the sign-up setting (`IsEnabledOrFactor` in clerk_go). + test.each([ + ["sign-in", { enabled: false, used_for_first_factor: true, first_factors: ["phone_code"] }], + ["MFA", { enabled: false, used_for_second_factor: true, second_factors: ["phone_code"] }], + ])("keeps a phone used only for %s, with no warning or fix", async (_label, phone_number) => { + const phoneOnly = user("p", { email: undefined, phone: "+15555550100" }); + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, phone_number }), + users: [user("a", { phone: "+15555550101" }), phoneOnly], + }), + ); + expect(checks.rejects).toEqual([]); + expect(checks.importable).toEqual([user("a", { phone: "+15555550101" }), phoneOnly]); + expect(checks.warnings).toEqual([]); + expect(checks.fixes).toEqual([]); + }); + + // Clerk validates a name but never checks its setting (create_service.go). + test("says a name is kept, not dropped, when names are off", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, first_name: { enabled: false } }), + users: [user("a", { firstName: "Ada" })], + }), + ); + expect(checks.importable).toEqual([user("a", { firstName: "Ada" })]); + expect(checks.warnings).toEqual([ + "1 user has a first name, which this instance does not use: it is stored, and shows only once first names are turned on", + ]); + expect(checks.fixes.map((fix) => fix.label)).toEqual(["Enable First name"]); + }); + + test("says a password is kept, not dropped, when passwords are off", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, password: { enabled: false } }), + users: [user("a", { password: BCRYPT, passwordHasher: "bcrypt" })], + }), + ); + expect(checks.warnings).toContain( + "1 user has a password, which this instance does not use: it is stored, and works only once passwords are turned on", + ); + }); + + // Clerk's ban has no end, so a source's temporary ban would outlast itself. + test("a ban that was due to end", async () => { + const checks = await checkImport( + input({ + users: [ + user("a", { banned: true, banEndsAt: "2026-11-01T00:00:00.000Z" }), + user("b", { banned: true, banEndsAt: "2026-12-01T00:00:00.000Z" }), + user("c", { banned: true }), + ], + }), + ); + expect(checks.warnings).toContain( + "2 users have bans that end by 2026-12-01; Clerk's ban has no end, so they stay banned until unbanned in Clerk", + ); + }); + + test("fields Clerk has no place for", async () => { + const checks = await checkImport( + input({ users: [user("a")], unknownFields: { department: 3, role: 1 } }), + ); + expect(checks.warnings).toContain("Clerk won't store: department (3 users), role (1 user)"); + }); + + test("passwords a source had to drop", async () => { + const checks = await checkImport(input({ users: [user("a", { passwordDropped: true })] })); + expect(checks.warnings[0]).toContain("1 password Clerk cannot verify will be dropped"); + }); +}); + +describe("fixes", () => { + test("print a `clerk config patch` per flagged setting, targeting the instance", async () => { + const checks = await checkImport( + input({ + settings: settings({ + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }), + target: { + env: "production", + appId: "app_1", + instanceId: "ins_1", + instanceType: "prod", + keySource: "linked profile", + }, + users: [user("a"), user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes).toEqual([ + { + label: "Make Email optional at sign-up", + command: `clerk config patch --app app_1 --instance ins_1 --json '{"auth_email":{"required_for_sign_up":false}}'`, + }, + ]); + }); + + // Without --instance, `clerk config patch` changes the linked profile's + // development instance, not the one a --secret-key import targets; without + // --app, the linked profile's app. A key names neither app nor profile. + test("name the instance, and leave the app to fill in, when the key came from --secret-key", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + users: [user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes[0]?.command).toStartWith( + "clerk config patch --app APP_ID --instance ins_1 --json", + ); + }); + + test("point at the Dashboard when the instance could not be named", async () => { + const checks = await checkImport( + input({ + settings: EMAIL_REQUIRED, + target: { + env: "production", + instanceId: "key_0123", + instanceType: "prod", + keySource: "--secret-key", + }, + users: [user("b", { email: undefined, username: "b" })], + }), + ); + expect(checks.fixes[0]).toEqual({ + label: "Make Email optional at sign-up", + url: "https://dashboard.clerk.com", + }); + }); + + // The rejects name the option; the fix gives the command for it. + test.each([ + [ + "12345", + "allow_numeric_usernames", + "Allow numeric usernames (Clerk then requires phone numbers in E.164 form, on import and at sign-up)", + ], + [ + "ada.l", + "allow_extended_special_characters", + "Allow extended special characters in usernames", + ], + ])("offer the username option %p needs", async (username, rule, label) => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, username: { enabled: true } }), + users: [user("a", { username }), user("b", { username: "ada" })], + }), + ); + expect(checks.fixes).toEqual([ + { + label, + command: `clerk config patch --app APP_ID --instance ins_1 --json '{"auth_username":{"${rule}":true}}'`, + }, + ]); + }); + + // With numeric usernames on, Clerk refuses a phone not in E.164 form, so + // the fix would cost those users their phone. + test("offer no numeric-username fix to a file with a phone not in E.164 form", async () => { + const checks = await checkImport( + input({ + settings: settings({ email_address: { enabled: true }, username: { enabled: true } }), + users: [user("a", { username: "12345" }), user("b", { phone: "(415) 555-0100" })], + }), + ); + expect(checks.fixes.map((fix) => fix.label).join("\n")).not.toContain("numeric usernames"); + }); + + test("offer nothing when the settings could not be read", async () => { + const checks = await checkImport(input({ users: [user("a")] })); + expect(checks.fixes).toEqual([]); + expect(checks.settingsUnavailable).toBe(true); + }); +}); + +describe("passwordIsOnlySignIn", () => { + const withFactors = (factors: Record, social: object = {}) => + bare( + Object.fromEntries( + Object.entries(factors).map(([name, first_factors]) => [ + name, + { enabled: true, used_for_first_factor: first_factors.length > 0, first_factors }, + ]), + ), + social, + ); + + // Clerk's shape: `password` lists no first factors of its own. + test.each([ + ["password alone", { password: [] }, {}, true], + ["password, and an email used only for sign-up", { password: [], email_address: [] }, {}, true], + // Mirrors clerk_go: neither a passkey nor a reset is a way in on its own. + ["password and passkey", { password: [], passkey: ["passkey"] }, {}, true], + [ + "password and a reset code", + { password: [], email_address: ["reset_password_email_code"] }, + {}, + true, + ], + ["password and email codes", { password: [], email_address: ["email_code"] }, {}, false], + [ + "password and Google", + { password: [] }, + { oauth_google: { enabled: true, authenticatable: true } }, + false, + ], + ["email links without passwords", { email_address: ["email_link"] }, {}, false], + ])("%s -> %p", (_label, factors, social, expected) => { + expect(passwordIsOnlySignIn(withFactors(factors, social))).toBe(expected); + }); +}); + +describe("hashShapeProblem", () => { + test.each([ + [BCRYPT, "bcrypt"], + ["hash$salt$signer$sep$8$14", "scrypt_firebase"], + // A project with no salt separator. + ["hash$salt$signer$$8$14", "scrypt_firebase"], + ["$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", "argon2id"], + [`scrypt:16384:16:1$${"a".repeat(32)}$${"b".repeat(128)}`, "scrypt_werkzeug"], + // Not checked: a bad digest there fails only at sign-in. + ["anything", "pbkdf2_sha256"], + ])("accepts %s as %s", (password, hasher) => { + expect(hashShapeProblem(password, hasher)).toBeUndefined(); + }); + + test.each([ + ["$2a$10$short", "bcrypt"], + // Stored by Clerk, but Go's bcrypt refuses it at sign-in. + ["$2a$03$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", "bcrypt"], + ["hash$salt$signer$sep$eight$14", "scrypt_firebase"], + ["hash$salt", "scrypt_firebase"], + ["hash$$signer$sep$8$14", "scrypt_firebase"], + // Clerk decodes every segment as base64. + ["not base64!$salt$signer$sep$8$14", "scrypt_firebase"], + ["hash$sa=lt$signer$sep$8$14", "scrypt_firebase"], + ["argon2id$...", "argon2id"], + ["scrypt:16384:16:1$salt$not-hex!", "scrypt_werkzeug"], + ])("rejects %s as %s", (password, hasher) => { + expect(hashShapeProblem(password, hasher)).toBeDefined(); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/checks.ts b/packages/cli-core/src/commands/migrate/lib/checks.ts new file mode 100644 index 000000000..62f5556d9 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/checks.ts @@ -0,0 +1,1108 @@ +/** + * `checkImport()`: what the instance will do with each user, worked out before + * anything is written. + * + * Every import runs these checks, and `--dry-run` stops after them. They sort + * the file into three piles: + * + * - **Rejects** — users Clerk would refuse, or that the run would refuse on + * Clerk's behalf. Any reject stops the import unless `--allow-partial` is + * passed, and then each one is recorded as `skipped` with its reason. + * - **Warnings** — users that import, but lose something on the way: a field + * the instance is not set up to store, a field Clerk has no place for, a + * password it cannot verify. + * - **Importable** — everyone else, in file order. + * + * Each rejected user gets the first reason that applies to them, so the counts + * add up to the file. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { SpinnerControls } from "../../../lib/spinner.ts"; +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import { splitIdentifiers } from "../import-users.ts"; +import type { User } from "../types.ts"; +import { analyzeFields, hasValue } from "./analysis.ts"; +import { decodesLikeClerk } from "./firebase-hash.ts"; +import { + clerkOffersProvider, + enabledSocialProviders, + providerLabel, + toClerkStrategy, +} from "./clerk-config.ts"; +import { resolveDevUserLimit } from "./instance.ts"; +import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; +import { acceptsIdentifier, buildReadinessReport } from "./readiness.ts"; +import type { ApiScheduler } from "./scheduler.ts"; +import { + countSocialProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + getUserProviders, +} from "./supabase-providers.ts"; +import type { ClerkTarget } from "./target.ts"; +import type { ValidationFailure } from "./transform.ts"; +import { lookupUsers, type LookedUpUser } from "./user-lookup.ts"; + +export type Reject = { + sourceId: string; + reason: string; + /** For a duplicate: the earlier user in the file that is kept instead. */ + keptSourceId?: string; +}; + +export type ReasonCount = { reason: string; count: number }; + +/** + * A setting change that would stop users being flagged: a `clerk config patch` + * command, or a Dashboard link when the instance can't be named for one. + */ +export type Fix = { label: string; command?: string; url?: string }; + +const DASHBOARD_URL = "https://dashboard.clerk.com"; + +export type Quota = { + /** Users already in the instance, or `null` when the count could not be read. */ + existing: number | null; + limit: number; + headroom: number; + /** Importable users over the headroom. */ + over: number; +}; + +export type ImportChecks = { + /** Users in the file being considered, valid or not. */ + total: number; + /** Users that pass every check, in file order. */ + importable: User[]; + /** Users that do not, each with the first reason that applies to them. */ + rejects: Reject[]; + rejectReasons: ReasonCount[]; + warnings: string[]; + fixes: Fix[]; + quota?: Quota; + /** True when the instance's settings could not be read. */ + settingsUnavailable: boolean; +}; + +export type CheckInput = { + users: User[]; + failures: ValidationFailure[]; + /** Fields the source produced that Clerk has no place for → how many users carry each. */ + unknownFields?: Record; + /** The raw Supabase rows, for the provider checks. Absent for any other source. */ + supabaseRows?: Record[]; + settings: UserSettingsJSON | null; + /** Users already in the instance, for the quota. Development instances only. */ + existingUsers?: number | null; + instanceType: "dev" | "prod"; + target: ClerkTarget; + secretKey: string; + schedule: ApiScheduler; + /** + * Import users with no legal acceptance into an instance that requires it, + * sending `skip_legal_checks`. Without it they are rejected. + */ + skipLegalChecks?: boolean; + /** Unverified identifiers are created reserved, so they meet a requirement. */ + reserveUnverified?: boolean; + /** + * Source IDs of adopted users whose create reserved their unverified + * identifiers: those exist reserved, whatever this run's flag says. + */ + reservedSourceIds?: Set; + /** + * Clerk IDs a continued run found behind its own in-flight creates: finding + * them in the instance is expected. + */ + adoptedClerkIds?: Set; + /** + * Source IDs of those adopted users. Each already exists in Clerk, so none + * is rejected: the checks stop creates, and these need none. They count + * toward no quota either; the instance's live count has them already. Their + * create ran in the mode its `creating` line records, so this run's flag + * does not decide what they reserved. + */ + adoptedSourceIds?: Set; + spinner?: SpinnerControls; +}; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +// --- Hash shapes ----------------------------------------------------------- + +/** + * Why a password digest cannot be what its hasher says it is, for the hashers + * whose shape is cheap and certain to check. + * + * Every other hasher's shape is not checked here: BAPI validates it at + * create, so a bad digest there fails that user mid-import. + */ +export function hashShapeProblem(password: string, hasher: string): string | undefined { + switch (hasher) { + case "bcrypt": + // clerk_go caps the cost at 15 (pkg/hash/bcrypt.go), and Go's bcrypt + // refuses any cost under 4 at sign-in, so that password could never work. + return /^\$2[aby]\$(0[4-9]|1[0-5])\$[./A-Za-z0-9]{53}$/.test(password) + ? undefined + : "password is not a bcrypt hash Clerk can verify ($2a$/$2b$/$2y$, cost 4 to 15, 60 characters)"; + case "scrypt_firebase": { + const parts = password.split("$"); + const numeric = (value: string | undefined) => /^\d+$/.test(value ?? ""); + return parts.length === 6 && + // The salt separator (parts[3]) may be empty. + parts.slice(0, 3).every(Boolean) && + // Clerk decodes all four as base64 (`Validate` in pkg/hash/scrypt.go). + parts.slice(0, 4).every(decodesLikeClerk) && + numeric(parts[4]) && + numeric(parts[5]) + ? undefined + : "password is not a Firebase scrypt digest (hash$salt$signerKey$saltSeparator$rounds$memCost)"; + } + case "argon2i": + case "argon2id": + return password.startsWith("$argon2") ? undefined : "password is not an argon2 hash"; + case "scrypt_werkzeug": + return /^scrypt:\d+:\d+:\d+\$[^$]+\$[0-9a-f]+$/i.test(password) + ? undefined + : "password is not a Werkzeug scrypt hash (scrypt:N:r:p$salt$hex)"; + default: + return undefined; + } +} + +// --- Per-user checks ------------------------------------------------------- + +/** + * Identifiers Clerk creates a user under. A required one the user does not + * carry leaves nothing to create them with. + * + * An email or phone counts only when it is verified: an unverified one is + * attached after the user exists, so it cannot satisfy a sign-up requirement. + * With `reserveUnverified` it goes on the create as reserved, which does. + */ +function missingRequiredIdentifier( + user: User, + settings: UserSettingsJSON | null, + reserveUnverified = false, +): string | undefined { + if (!settings) return undefined; + const required = (attribute: AttributeName) => isRequired(settings, attribute); + const identifiers = splitIdentifiers(user); + const reserved = (unverified: string[]) => reserveUnverified && unverified.length > 0; + + if ( + required("email_address") && + !identifiers.primaryEmail && + !reserved(identifiers.unverifiedEmails) + ) { + return identifiers.unverifiedEmails.length > 0 + ? "only has an unverified email, and this instance requires an email" + : "no email, which this instance requires"; + } + if ( + required("phone_number") && + !identifiers.primaryPhone && + !reserved(identifiers.unverifiedPhones) + ) { + return identifiers.unverifiedPhones.length > 0 + ? "only has an unverified phone number, and this instance requires one" + : "no phone number, which this instance requires"; + } + if (required("username") && !hasValue(user.username)) { + return "no username, which this instance requires"; + } + return undefined; +} + +/** A required first or last name the user lacks: `POST /v1/users` refuses it. */ +function missingRequiredName(user: User, settings: UserSettingsJSON | null): string | undefined { + if (!settings) return undefined; + if (isRequired(settings, "first_name") && !hasValue(user.firstName)) { + return "no first name, which this instance requires"; + } + if (isRequired(settings, "last_name") && !hasValue(user.lastName)) { + return "no last name, which this instance requires"; + } + return undefined; +} + +/** MFA a user can carry, and the setting `POST /v1/users` refuses it without. */ +const MFA_SETTINGS = [ + { + field: "totpSecret", + attribute: "authenticator_app", + reason: "has an authenticator app (TOTP) secret, and this instance has authenticator apps off", + label: "Enable authenticator apps", + path: ["auth_multi_factor", "authenticator_app", "enabled"], + }, + { + field: "backupCodes", + attribute: "backup_code", + reason: "has backup codes, and this instance has backup codes off", + label: "Enable backup codes", + path: ["auth_multi_factor", "backup_code", "enabled"], + }, +] as const; + +/** + * MFA the instance has off. Rejected rather than dropped: importing the user + * without it would quietly take away their second factor. + */ +function mfaProblem(user: User, settings: UserSettingsJSON | null): string | undefined { + if (!settings) return undefined; + return MFA_SETTINGS.find( + ({ field, attribute }) => hasValue(user[field]) && !isEnabled(settings, attribute), + )?.reason; +} + +/** Sign-in strategies that don't count as a way in without a password, per clerk_go. */ +const NOT_ALTERNATIVE_SIGN_IN = new Set([ + "passkey", + "ticket", + "reset_password_email_code", + "reset_password_phone_code", +]); + +/** + * True when the instance offers no way to sign in but a password. Clerk then + * refuses `skip_password_requirement`, so a user without a digest can't be + * created. Mirrors `create_service.go`: `FirstFactors()` minus the strategies + * that are no way in on their own. Clerk never lists `password` itself as a + * first factor, and a password reset doesn't count. + * + * ponytail: Google One Tap counts for Clerk when Google uses custom + * credentials, which FAPI doesn't serve; a Google set up for One Tap alone + * reads as no way in, so those users are rejected rather than created. + */ +export function passwordIsOnlySignIn(settings: UserSettingsJSON): boolean { + const strategies = firstFactorStrategies(settings); + for (const [strategy, social] of Object.entries(settings.social ?? {})) { + if (social?.enabled && social.authenticatable) strategies.add(strategy); + } + if (settings.enterprise_sso?.enabled) strategies.add("enterprise_sso"); + return [...strategies].every((strategy) => NOT_ALTERNATIVE_SIGN_IN.has(strategy)); +} + +/** The first-factor strategies the instance's identifiers offer (`email_code`, `password`, …). */ +function firstFactorStrategies(settings: UserSettingsJSON): Set { + const strategies = new Set(); + for (const attribute of Object.values(settings.attributes ?? {})) { + if (attribute?.used_for_first_factor) { + for (const strategy of attribute.first_factors ?? []) strategies.add(strategy); + } + } + return strategies; +} + +/** True when the instance needs legal acceptance this user has no record of. */ +function lacksLegalAcceptance(user: User, settings: UserSettingsJSON | null): boolean { + return ( + Boolean(settings?.sign_up?.legal_consent_enabled) && + !user.legalAcceptedAt && + !user.skipLegalChecks + ); +} + +/** What FAPI serves under `username_settings`; `@clerk/shared` types only the lengths. */ +type UsernameSettings = { + min_length?: number; + max_length?: number; + allow_extended_special_characters?: boolean; + allow_numeric_usernames?: boolean; +}; + +const USERNAME_DEFAULT = /^[a-zA-Z0-9_-]+$/; +const USERNAME_EXTENDED = /^[a-zA-Z0-9!#$'+.^_`~-]+$/; + +/** The username options a reject asks for, and the config leaf that turns each on. */ +const USERNAME_OPTIONS = [ + { + rule: "allow_numeric_usernames", + // Clerk ties strict E.164 phones to this setting (create_service.go). + label: + "Allow numeric usernames (Clerk then requires phone numbers in E.164 form, on import and at sign-up)", + needs: (username: string) => !/[a-zA-Z]/.test(username), + }, + { + rule: "allow_extended_special_characters", + label: "Allow extended special characters in usernames", + needs: (username: string) => + !USERNAME_DEFAULT.test(username) && USERNAME_EXTENDED.test(username), + }, +] as const; + +/** + * Clerk's username rules, mirrored from `validate.Username` in clerk_go, so a + * username the instance would refuse is a reject here rather than a failed + * create. Clerk checks them with usernames off too. + */ +function usernameProblem(user: User, settings: UserSettingsJSON | null): string | undefined { + const username = user.username; + if (!settings || typeof username !== "string" || !username) return undefined; + + const rules = (settings as { username_settings?: UsernameSettings }).username_settings ?? {}; + const length = [...username].length; + if (rules.min_length !== undefined && rules.max_length !== undefined) { + if (length < rules.min_length || length > rules.max_length) { + return `username is not ${rules.min_length}–${rules.max_length} characters, which this instance requires`; + } + } + if (!rules.allow_numeric_usernames && !/[a-zA-Z]/.test(username)) { + return "username has no letters; turn on numeric usernames to allow it"; + } + if (rules.allow_extended_special_characters) { + if (!USERNAME_EXTENDED.test(username)) return "username has characters Clerk does not allow"; + if (/^\+[1-9]\d{1,14}$/.test(username)) + return "username is a phone number, which Clerk does not allow"; + return undefined; + } + if (!USERNAME_DEFAULT.test(username)) { + return USERNAME_EXTENDED.test(username) + ? "username has special characters this instance does not allow; turn on extended special characters" + : "username has characters Clerk does not allow"; + } + return undefined; +} + +/** + * TLDs Clerk refuses for any email: clerk_go's `emailaddress.nonRoutableTLDs`, + * where placeholder addresses live (`…@phone.local`, `anon-…@anonymous.invalid`), + * plus the common private ones the public suffix list leaves out. + * + * ponytail: Clerk refuses any TLD not on the public suffix list; this names + * the usual ones instead of taking the list as a dependency. + */ +const NON_ROUTABLE_TLDS = new Set([ + "arpa", + "local", + "invalid", + "example", + "test", + "internal", + "lan", + "corp", + "home", + "localdomain", + "intranet", + "private", +]); + +/** + * An address shape Clerk accepts, loosely: one `@`, no spaces, a dotted host, + * a local part of at most 64 bytes and 254 in all. Non-ASCII is fine, as it + * is in clerk_go (`josé@x.dev`). + */ +function isEmailShaped(email: string): boolean { + const at = email.lastIndexOf("@"); + return ( + /^[^\s@]+@[^\s@]+\.[^\s@.]+$/.test(email) && + Buffer.byteLength(email.slice(0, at)) <= 64 && + email.length <= 254 + ); +} +const EMAIL_FIELDS = ["email", "emailAddresses", "unverifiedEmailAddresses"] as const; + +function isRefusedEmail(email: string): boolean { + if (!isEmailShaped(email)) return true; + const host = email.slice(email.lastIndexOf("@") + 1).toLowerCase(); + // Clerk's own dev domains sit under `.test` and are accepted. + if (host.endsWith(".clerk.test")) return false; + return NON_ROUTABLE_TLDS.has(host.slice(host.lastIndexOf(".") + 1)); +} + +/** The user without the emails Clerk would refuse, and those emails. */ +function dropRefusedEmails(user: User): { user: User; refused: string[] } { + const refused: string[] = []; + let kept: User | undefined; + for (const field of EMAIL_FIELDS) { + const value = user[field]; + if (value === undefined) continue; + const list = Array.isArray(value) ? value : [value]; + const bad = list.filter(isRefusedEmail); + if (bad.length === 0) continue; + refused.push(...bad); + kept ??= { ...user }; + const good = list.filter((email) => !isRefusedEmail(email)); + if (good.length > 0) kept[field] = good; + else delete kept[field]; + } + return { user: kept ?? user, refused }; +} + +/** Clerk's cap on a first or last name, in bytes (`firstNameMaxLen` in clerk_go). */ +const NAME_MAX_BYTES = 256; + +/** + * A name Clerk refuses: blank (only whitespace or unprintable characters), + * over {@link NAME_MAX_BYTES}, or what clerk_go's `NameForAbusePreventionLoose` + * refuses, approximated: a phone number (10–15 digits, or fewer behind a + * `+`/`00`), a URL with a scheme or path that isn't part of an email, or an + * HTML tag. Better Auth's phone sign-up stores the number as the name, so this + * is common, not exotic. + */ +function nameProblem(name: string): string | undefined { + // Go's `unicode.IsPrint` refuses control and format characters (\p{C}). + if (/^[\s\p{C}]+$/u.test(name)) return "blank"; + if (Buffer.byteLength(name) > NAME_MAX_BYTES) return "too long"; + for (const candidate of name.match(/(?:\+|00)?\d[\d\s().-]{5,}\d/g) ?? []) { + const digits = candidate.replace(/\D/g, "").length; + const international = /^(\+|00)/.test(candidate.trim()); + if (digits <= 15 && (digits >= 10 || (international && digits >= 7))) return "a phone number"; + } + // Clerk accepts an email as a name (NameForAbusePreventionLoose), and a URL + // that is part of one. + const hasEmail = /\S+@\S+\.\S+/.test(name); + if (!hasEmail && /:\/\/|\b[\w-]+(\.[\w-]+)+[/?#]/.test(name)) return "a URL"; + if (/<\/?[a-z!][^>]*>/i.test(name)) return "HTML"; + return undefined; +} + +/** The user without a first or last name Clerk would refuse. */ +function dropRefusedNames(user: User): { user: User; dropped: boolean } { + let kept: User | undefined; + for (const field of ["firstName", "lastName"] as const) { + const value = user[field]; + if (typeof value === "string" && nameProblem(value)) { + kept ??= { ...user }; + delete kept[field]; + } + } + return { user: kept ?? user, dropped: kept !== undefined }; +} + +/** + * A phone number with its punctuation stripped, so `+1 555-555-0100` and + * `+15555550100` compare equal. + * + * ponytail: punctuation only; a national number without its country code + * still differs from its E.164 form. Full parsing would need a dependency. + */ +const phoneKey = (phone: string) => phone.replace(/[^\d+]/g, ""); + +const hasAnyIdentifier = (user: User) => + [...EMAIL_FIELDS, "phone", "phoneNumbers", "unverifiedPhoneNumbers", "username"].some((field) => + hasValue(user[field as keyof User]), + ); + +/** + * First user in the file to claim each email, phone and source ID. + * + * Keyed by record, not source ID: two records with one source ID must not + * share a verdict, or the one kept would be rejected along with its copy. + * + * Only what `POST /v1/users` carries is matched, as in + * {@link findInstanceDuplicates}: an extra email is attached after the + * create, and a clash there fails only as a note on the user. + * + * @returns Each duplicate's reason, and the earlier user kept in its place. + */ +function findFileDuplicates( + users: User[], + reserves: (user: User) => boolean = () => false, +): { + reasons: Map; + keptBy: Map; +} { + const reasons = new Map(); + const keptBy = new Map(); + const seenIds = new Set(); + const emails = new Map(); + const phones = new Map(); + const usernames = new Map(); + + for (const user of users) { + if (seenIds.has(user.userId)) { + reasons.set(user, "duplicate source ID in the file"); + continue; + } + seenIds.add(user.userId); + + const { emails: ownEmails, phones: ownPhones } = sentIdentifiers(user, reserves(user)); + + const emailOwner = ownEmails.map((email) => emails.get(email.toLowerCase())).find(Boolean); + const phoneOwner = ownPhones.map((phone) => phones.get(phoneKey(phone))).find(Boolean); + // Clerk lowercases usernames, so the second create would fail. + const username = typeof user.username === "string" ? user.username.toLowerCase() : ""; + const usernameOwner = username ? usernames.get(username) : undefined; + // The first record in the file wins, whatever either holds: the source's + // order decides, so the kept ID is named alongside the reject. + if (emailOwner) { + reasons.set(user, "email is also used by an earlier user in the file, which is kept"); + keptBy.set(user, emailOwner); + continue; + } + if (phoneOwner) { + reasons.set(user, "phone number is also used by an earlier user in the file, which is kept"); + keptBy.set(user, phoneOwner); + continue; + } + if (usernameOwner) { + reasons.set(user, "username is also used by an earlier user in the file, which is kept"); + keptBy.set(user, usernameOwner); + continue; + } + for (const email of ownEmails) emails.set(email.toLowerCase(), user.userId); + for (const phone of ownPhones) phones.set(phoneKey(phone), user.userId); + if (username) usernames.set(username, user.userId); + } + return { reasons, keptBy }; +} + +/** + * Whether a user's unverified identifiers are (or will be) created reserved: + * an adopted user's by what its create recorded, everyone else's by this + * run's flag. + */ +function reservesFor(input: CheckInput): (user: User) => boolean { + return (user) => + input.adoptedSourceIds?.has(user.userId) + ? Boolean(input.reservedSourceIds?.has(user.userId)) + : Boolean(input.reserveUnverified); +} + +/** + * The emails and phones `POST /v1/users` sends for a user: the verified + * primary, plus the unverified ones when they are created reserved. + */ +function sentIdentifiers(user: User, reserveUnverified: boolean) { + const { primaryEmail, primaryPhone, unverifiedEmails, unverifiedPhones } = splitIdentifiers(user); + return { + emails: [ + ...(primaryEmail ? [primaryEmail] : []), + ...(reserveUnverified ? unverifiedEmails : []), + ], + phones: [ + ...(primaryPhone ? [primaryPhone] : []), + ...(reserveUnverified ? unverifiedPhones : []), + ], + }; +} + +/** + * Users the instance already holds, found by source ID, the emails and phones + * the create sends, or username. + * + * Only what `POST /v1/users` itself carries is looked up: an extra email that + * collides is attached after the user exists, fails on its own, and is noted + * on the user's line rather than failing them. + */ +async function findInstanceDuplicates( + users: User[], + input: CheckInput, +): Promise> { + const byExternalId = new Map(); + const byEmail = new Map(); + const byPhone = new Map(); + const byUsername = new Map(); + + for (const user of users) { + const sent = sentIdentifiers(user, reservesFor(input)(user)); + byExternalId.set(user.userId, user.userId); + for (const email of sent.emails) byEmail.set(email.toLowerCase(), user.userId); + for (const phone of sent.phones) byPhone.set(phoneKey(phone), user.userId); + if (typeof user.username === "string" && user.username) { + byUsername.set(user.username.toLowerCase(), user.userId); + } + } + + const lookup = async ( + filter: "external_id" | "email_address" | "phone_number" | "username", + values: Iterable, + ) => + lookupUsers({ + filter, + values: [...values], + secretKey: input.secretKey, + schedule: input.schedule, + spinner: input.spinner, + label: "Checking for users already in the instance", + }); + + const found: LookedUpUser[] = ( + await Promise.all([ + lookup("external_id", byExternalId.keys()), + lookup("email_address", byEmail.keys()), + lookup("phone_number", byPhone.keys()), + lookup("username", byUsername.keys()), + ]) + ).flat(); + + const reasons = new Map(); + + for (const existing of found) { + // An adopted user is its own record's create, so it clashes with nothing + // in that record; its identifiers still clash with every other user's. + const own = input.adoptedClerkIds?.has(existing.id) ? existing.external_id : undefined; + const claim = (sourceId: string | undefined, reason: string) => { + if (sourceId && sourceId !== own && !reasons.has(sourceId)) reasons.set(sourceId, reason); + }; + if (existing.external_id) { + claim(byExternalId.get(existing.external_id), "already in the instance, with this source ID"); + } + for (const email of existing.email_addresses ?? []) { + claim( + byEmail.get((email.email_address ?? "").toLowerCase()), + "email is already used by a user in the instance", + ); + } + for (const phone of existing.phone_numbers ?? []) { + claim( + byPhone.get(phoneKey(phone.phone_number ?? "")), + "phone number is already used by a user in the instance", + ); + } + if (existing.username) { + claim( + byUsername.get(existing.username.toLowerCase()), + "username is already taken in the instance", + ); + } + } + return reasons; +} + +/** Supabase users whose every provider is off in Clerk: they could never sign in. */ +function findDisabledProviderRejects(input: CheckInput): Map { + const reasons = new Map(); + if (!input.supabaseRows || !input.settings) return reasons; + + const enabled = enabledSocialProviders(input.settings); + const disabled = findDisabledProviders(input.supabaseRows, enabled, toClerkStrategy); + if (disabled.length === 0) return reasons; + + const { excludedIds } = findUsersWithOnlyDisabledProviders(input.supabaseRows, disabled); + // A disabled provider only strands a user with no other way in: a verified + // email or phone the instance signs in with by code or link still works. + const strategies = firstFactorStrategies(input.settings); + const usersById = new Map(input.users.map((user) => [user.userId, user])); + const canSignInOtherwise = (id: string) => { + const user = usersById.get(id); + if (!user) return false; + const { primaryEmail, primaryPhone } = splitIdentifiers(user); + return ( + (Boolean(primaryEmail) && (strategies.has("email_code") || strategies.has("email_link"))) || + (Boolean(primaryPhone) && strategies.has("phone_code")) + ); + }; + // Each reject names only that user's own providers. + const rowsById = new Map(input.supabaseRows.map((row) => [String(row.id), row])); + for (const id of excludedIds) { + if (canSignInOtherwise(id)) continue; + const own = getUserProviders(rowsById.get(id) ?? {}).filter((p) => disabled.includes(p)); + // Clerk can't turn on a provider it doesn't offer, so say which is which. + const why = (provider: string) => + clerkOffersProvider(provider) ? "not enabled in Clerk" : "not offered by Clerk"; + const kinds = new Set(own.map(why)); + const names = + kinds.size === 1 + ? `${own.map(providerLabel).join(", ")}, which ${own.length === 1 ? "is" : "are"} ${[...kinds][0]}` + : own.map((provider) => `${providerLabel(provider)} (${why(provider)})`).join(", "); + reasons.set(id, `only signs in with ${names}`); + } + return reasons; +} + +// --- Warnings and fixes ---------------------------------------------------- + +/** The Supabase rows for just these users, so provider counts describe them alone. */ +function rowsFor(input: CheckInput, users: User[]): Record[] { + if (!input.supabaseRows) return []; + const ids = new Set(users.map((user) => user.userId)); + return input.supabaseRows.filter((row) => ids.has(String(row.id))); +} + +/** Each readiness row's field, with its article: "an email", not "a email". */ +const FIELD_NOUNS: Record = { + email_address: "an email", + phone_number: "a phone number", + username: "a username", + first_name: "a first name", + last_name: "a last name", +}; + +function buildWarnings(input: CheckInput, importable: User[]): string[] { + const warnings: string[] = []; + + if (input.settings && importable.length > 0) { + const report = buildReadinessReport({ + analysis: analyzeFields(importable), + settings: input.settings, + providerCounts: countSocialProviders(rowsFor(input, importable)), + }); + for (const item of report.blocking) { + if (item.consequence === "stored") { + warnings.push(storedWarning(item.key, item.userCount)); + continue; + } + if (item.consequence !== "drops") continue; + if (item.clerkRequired === true) { + const missing = importable.length - item.userCount; + warnings.push( + item.key === "password" + ? `${plural(missing, "user")} without a password, which this instance requires: they sign in another way, such as a code or a social account` + : `${plural(missing, "user")} without ${FIELD_NOUNS[item.key] ?? item.label.toLowerCase()}, which this instance requires`, + ); + } else { + warnings.push( + item.section === "social" + ? `${plural(item.userCount, "user")} signed in with ${item.label}, which ${clerkOffersProvider(item.key) ? "is not enabled in Clerk" : "Clerk doesn't offer"}` + : `${plural(item.userCount, "user")} ${item.userCount === 1 ? "has" : "have"} ${FIELD_NOUNS[item.key] ?? item.label.toLowerCase()}, which this instance is not set up to store`, + ); + } + } + } + + const dropped = importable.filter((user) => user.passwordDropped).length; + if (dropped > 0) { + warnings.push( + `${plural(dropped, "password")} Clerk cannot verify will be dropped: those users sign in another way, such as a code or a social account`, + ); + } + + const unknown = Object.entries(input.unknownFields ?? {}); + if (unknown.length > 0) { + warnings.push( + `Clerk won't store: ${unknown + .sort((a, b) => b[1] - a[1]) + .map(([field, count]) => `${field} (${plural(count, "user")})`) + .join(", ")}`, + ); + } + + return warnings; +} + +/** Each field Clerk stores with its setting off, and what it does once turned on. */ +const STORED_FIELDS: Record = { + password: { noun: "password", verb: "works" }, + username: { noun: "username", verb: "works" }, + first_name: { noun: "first name", verb: "shows" }, + last_name: { noun: "last name", verb: "shows" }, +}; + +/** + * A field Clerk stores with its setting off, so nothing is lost: it starts + * working, or showing, once the setting is turned on. + */ +function storedWarning(key: string, count: number): string { + const { noun, verb } = STORED_FIELDS[key] ?? { noun: key, verb: "works" }; + return `${plural(count, "user")} ${count === 1 ? "has" : "have"} a ${noun}, which this instance does not use: it is stored, and ${verb} only once ${noun}s are turned on`; +} + +/** Shell-quotes a JSON payload for a single-quoted argument. */ +function quoteJson(payload: unknown): string { + return `'${JSON.stringify(payload).replace(/'/g, "'\\''")}'`; +} + +/** + * One `clerk config patch` per flagged setting, built from the same rows the + * warnings and rejects come from. + * + * These are offers, not corrections: an instance that requires an email is + * configured as its owner intended, and fixing the export may be the answer. + */ +function buildFixes(input: CheckInput, users: User[]): Fix[] { + if (!input.settings || users.length === 0) return []; + + const report = buildReadinessReport({ + analysis: analyzeFields(users), + settings: input.settings, + providerCounts: countSocialProviders(rowsFor(input, users)), + }); + + // An unverified email does not satisfy a required one, so a file of only + // unverified addresses flags the requirement even when every user has one. + // Created reserved, it does, and making email optional fixes nothing. + const unverifiedOnly = + !input.reserveUnverified && + users.some((user) => { + const identifiers = splitIdentifiers(user); + return !identifiers.primaryEmail && identifiers.unverifiedEmails.length > 0; + }); + const flagged = report.blocking.slice(); + if ( + unverifiedOnly && + isRequired(input.settings, "email_address") && + isEnabled(input.settings, "email_address") && + !flagged.some((item) => item.key === "email_address") + ) { + const email = report.items.find((item) => item.key === "email_address"); + if (email) + flagged.unshift({ ...email, clerkRequired: true, blocking: true, consequence: "rejects" }); + } + + // Always name the instance: without it, `clerk config patch` acts on the + // linked profile's development instance, whatever key this import used. A + // `key_` ID is a stand-in for an instance Clerk didn't name, so there is + // nothing to pass; point at the Dashboard instead. A key from --secret-key + // names no app, and without --app the patch goes to the linked one, so the + // command carries a placeholder the user fills in: `APP_ID`, which a shell + // passes as is and Clerk answers "app not found", where `` would + // be read as a redirect. + const { appId, instanceId } = input.target; + const named = instanceId.startsWith("ins_"); + const flags = ` --app ${appId ?? "APP_ID"} --instance ${instanceId}`; + const settings = input.settings; + const mfa = MFA_SETTINGS.filter( + ({ field, attribute }) => + !isEnabled(settings, attribute) && users.some((user) => hasValue(user[field])), + ).map(({ label, path }) => ({ label, writes: [{ path: [...path], value: true }] })); + // Usernames off are not checked by these rejects; see `dropRefusedOffUsername`. + const rules = (settings as { username_settings?: UsernameSettings }).username_settings ?? {}; + const usernames = isEnabled(settings, "username") + ? users.flatMap((user) => (typeof user.username === "string" ? [user.username] : [])) + : []; + // Numeric usernames make Clerk refuse any phone not in E.164 form, so the + // fix is not offered to a file whose phones would then be refused. + const looseFormat = users.some((user) => { + const { primaryPhone, additionalPhones, unverifiedPhones } = splitIdentifiers(user); + return [primaryPhone, ...additionalPhones, ...unverifiedPhones].some( + (phone) => phone !== undefined && !/^\+[1-9]\d{1,14}$/.test(phone), + ); + }); + const username = USERNAME_OPTIONS.filter( + ({ rule, needs }) => + !rules[rule] && usernames.some(needs) && !(rule === "allow_numeric_usernames" && looseFormat), + ).map(({ rule, label }) => ({ label, writes: [{ path: ["auth_username", rule], value: true }] })); + + return [...buildSettingChanges(flagged), ...mfa, ...username].map((change) => + named + ? { + label: change.label, + command: `clerk config patch${flags} --json ${quoteJson(buildChangePayload([change]))}`, + } + : { label: change.label, url: DASHBOARD_URL }, + ); +} + +// --- The whole check ------------------------------------------------------- + +function countReasons(rejects: Reject[]): ReasonCount[] { + const counts = new Map(); + for (const { reason } of rejects) counts.set(reason, (counts.get(reason) ?? 0) + 1); + return [...counts].map(([reason, count]) => ({ reason, count })); +} + +/** + * Removes the emails or phones Clerk would refuse: those of an instance that + * neither has the identifier on nor signs in or does MFA with it. + * + * The warnings already say they are dropped, but Clerk does not drop them: it + * refuses the whole create (`phone_number is not a valid parameter`). A + * username is not among them: Clerk stores it with usernames off. + */ +function dropDisabledIdentifiers(user: User, settings: UserSettingsJSON | null): User { + if (!settings) return user; + const fields = [ + ...(acceptsIdentifier(settings, "email_address") + ? [] + : (["email", "emailAddresses", "unverifiedEmailAddresses"] as const)), + ...(acceptsIdentifier(settings, "phone_number") + ? [] + : (["phone", "phoneNumbers", "unverifiedPhoneNumbers"] as const)), + ]; + if (!fields.some((field) => field in user)) return user; + const kept = { ...user }; + for (const field of fields) delete kept[field]; + return kept; +} + +/** + * The user without a username Clerk would refuse, where usernames are off: + * Clerk still checks it there, and nothing signs in with it, so it is dropped + * rather than costing the user. + */ +function dropRefusedOffUsername( + user: User, + settings: UserSettingsJSON | null, +): { user: User; dropped: boolean } { + if (!settings || isEnabled(settings, "username") || !usernameProblem(user, settings)) { + return { user, dropped: false }; + } + const { username: _dropped, ...kept } = user; + return { user: kept as User, dropped: true }; +} + +function refusedUsernameWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} a username Clerk refuses, which is dropped: this instance has usernames off`, + ]; +} + +function refusedNameWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} a name Clerk refuses (a phone number, a URL, HTML, blank, or over ${NAME_MAX_BYTES} bytes), which is dropped`, + ]; +} + +/** + * Users whose ban was due to end. Clerk's ban has no end, so they stay banned + * until someone lifts it in Clerk; the latest end date says how long to watch. + */ +function temporaryBanWarning(users: User[]): string[] { + const ends = users.flatMap((user) => (user.banned && user.banEndsAt ? [user.banEndsAt] : [])); + if (ends.length === 0) return []; + const latest = ends.sort().at(-1)?.slice(0, 10); + return [ + `${plural(ends.length, "user")} ${ends.length === 1 ? "has a ban that ends" : "have bans that end"} by ${latest}; Clerk's ban has no end, so ${ends.length === 1 ? "it stays" : "they stay"} banned until unbanned in Clerk`, + ]; +} + +function legalWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} no legal acceptance on record, and ${count === 1 ? "is" : "are"} created without it (--skip-legal-checks)`, + ]; +} + +function placeholderWarning(count: number): string[] { + if (count === 0) return []; + return [ + `${plural(count, "user")} ${count === 1 ? "has" : "have"} an email Clerk refuses (malformed, or a domain such as .local or .invalid), which is dropped`, + ]; +} + +export async function checkImport(input: CheckInput): Promise { + const rejects: Reject[] = input.failures.map((failure) => ({ + sourceId: failure.userId, + reason: `invalid: ${failure.error}`, + })); + + const disabledProviders = findDisabledProviderRejects(input); + + // Users that pass every per-user check. Only these claim identifiers in + // the file: a rejected record must not cost a later one its email. Each is + // stripped of what Clerk would refuse, so the duplicate checks see only + // what is sent; `unstripped` keeps what the warnings describe. + const passed: User[] = []; + const unstripped = new Map(); + const placeholderEmails = new Set(); + const refusedNames = new Set(); + const refusedUsernames = new Set(); + for (const original of input.users) { + const named = dropRefusedNames(original); + if (named.dropped) refusedNames.add(original.userId); + const usernamed = dropRefusedOffUsername(named.user, input.settings); + if (usernamed.dropped) refusedUsernames.add(original.userId); + const { user, refused } = dropRefusedEmails(usernamed.user); + const sent = dropDisabledIdentifiers(user, input.settings); + const adopted = input.adoptedSourceIds?.has(original.userId) ?? false; + const reason = adopted + ? undefined + : (original.skipReason ?? + (refused.length > 0 && !hasAnyIdentifier(user) + ? "only has emails Clerk refuses (malformed, or a domain that can't receive mail)" + : undefined) ?? + missingRequiredIdentifier(user, input.settings, reservesFor(input)(user)) ?? + // Stripping the identifiers the instance has off can leave nothing to + // sign in with; Clerk would still create the user. + (!hasAnyIdentifier(sent) + ? "has no identifier this instance accepts (its email or phone is turned off, or its username is one Clerk refuses)" + : undefined) ?? + missingRequiredName(user, input.settings) ?? + mfaProblem(user, input.settings) ?? + (!user.password && input.settings && passwordIsOnlySignIn(input.settings) + ? "no password, and password is this instance's only way to sign in" + : undefined) ?? + (!input.skipLegalChecks && lacksLegalAcceptance(user, input.settings) + ? "no legal acceptance on record, which this instance requires (--skip-legal-checks imports them without it)" + : undefined) ?? + usernameProblem(user, input.settings) ?? + (user.password && user.passwordHasher + ? hashShapeProblem(user.password, user.passwordHasher) + : undefined) ?? + disabledProviders.get(user.userId)); + if (reason) { + rejects.push({ sourceId: user.userId, reason }); + } else { + passed.push(sent); + unstripped.set(sent, user); + if (refused.length > 0) placeholderEmails.add(user.userId); + } + } + + const { reasons: fileDuplicates, keptBy } = findFileDuplicates(passed, reservesFor(input)); + let candidates: User[] = []; + for (const user of passed) { + const reason = input.adoptedSourceIds?.has(user.userId) ? undefined : fileDuplicates.get(user); + const kept = keptBy.get(user); + if (reason) + rejects.push({ sourceId: user.userId, reason, ...(kept ? { keptSourceId: kept } : {}) }); + else candidates.push(user); + } + + const instanceDuplicates = + candidates.length > 0 + ? await findInstanceDuplicates(candidates, input) + : new Map(); + const unique: User[] = []; + for (const user of candidates) { + const reason = input.adoptedSourceIds?.has(user.userId) + ? undefined + : instanceDuplicates.get(user.userId); + if (reason) rejects.push({ sourceId: user.userId, reason }); + else unique.push(user); + } + candidates = unique; + + // The limit is a development-instance default, not a number the API serves, + // so it is checked against the live count and the importable users alone. + // An instance Clerk has raised sets CLERK_MIGRATE_DEV_USER_LIMIT. + let quota: Quota | undefined; + if (input.instanceType === "dev") { + const limit = resolveDevUserLimit(); + // An adopted user is in the instance's count already, and is never + // created: only the new users take headroom, in file order. + const headroom = Math.max(0, limit - (input.existingUsers ?? 0)); + const isNew = (user: User) => !input.adoptedSourceIds?.has(user.userId); + const fresh = candidates.filter(isNew); + const over = Math.max(0, fresh.length - headroom); + quota = { existing: input.existingUsers ?? null, limit, headroom, over }; + if (over > 0) { + const past = new Set(fresh.slice(headroom)); + for (const user of past) { + rejects.push({ + sourceId: user.userId, + reason: `over the development instance's ${limit}-user limit (raised by Clerk? set CLERK_MIGRATE_DEV_USER_LIMIT)`, + }); + } + candidates = candidates.filter((user) => !past.has(user)); + } + } + + return { + total: input.users.length + input.failures.length, + importable: candidates.map((user) => + lacksLegalAcceptance(user, input.settings) ? { ...user, skipLegalChecks: true } : user, + ), + rejects, + rejectReasons: countReasons(rejects), + warnings: [ + ...buildWarnings( + input, + candidates.map((user) => unstripped.get(user) ?? user), + ), + ...placeholderWarning(candidates.filter((user) => placeholderEmails.has(user.userId)).length), + ...refusedNameWarning(candidates.filter((user) => refusedNames.has(user.userId)).length), + ...refusedUsernameWarning( + candidates.filter((user) => refusedUsernames.has(user.userId)).length, + ), + ...temporaryBanWarning(candidates), + ...legalWarning( + candidates.filter((user) => lacksLegalAcceptance(user, input.settings)).length, + ), + // An unknown count is checked as zero; say so rather than imply it fit. + ...(quota && quota.existing === null + ? [ + `Could not read how many users this development instance holds, so the ${quota.limit}-user limit was checked as if it were empty`, + ] + : []), + ], + fixes: buildFixes(input, input.users), + ...(quota ? { quota } : {}), + settingsUnavailable: input.settings === null, + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts new file mode 100644 index 000000000..5fe3842bc --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.test.ts @@ -0,0 +1,148 @@ +import { test, expect, describe, mock, beforeEach, afterAll } from "bun:test"; +import { stubFetch, useCaptureLog } from "../../../test/lib/stubs.ts"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { fetchInstanceSettings, fetchUserCount } from "./clerk-config.ts"; + +const USER_SETTINGS = { + attributes: { email_address: { enabled: true, required: true } }, +} as unknown as UserSettingsJSON; + +function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); +} + +describe("fetchInstanceSettings", () => { + const originalFetch = globalThis.fetch; + useCaptureLog(); + const mockFetch = mock(); + + beforeEach(() => { + mockFetch.mockReset(); + stubFetch(mockFetch); + }); + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + /** Routes the three hops: BAPI domains → FAPI dev browser → FAPI environment. */ + function route(domains: unknown): void { + mockFetch.mockImplementation((input: string | URL) => { + const url = String(input); + if (url.includes("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/v1/domains")) return Promise.resolve(json({ data: domains })); + if (url.includes("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/v1/dev_browser")) return Promise.resolve(json({ token: "jwt" })); + if (url.includes("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/v1/environment")) { + return Promise.resolve(json({ user_settings: USER_SETTINGS })); + } + throw new Error(`unexpected request: ${url}`); + }); + } + + test("reads settings off the primary domain's Frontend API", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("clerk.example.com/v1/dev_browser"))).toBe(true); + expect(urls.some((url) => url.includes("clerk.example.com/v1/environment"))).toBe(true); + }); + + test("prefers the primary domain over a satellite", async () => { + route([ + { is_satellite: true, frontend_api_url: "https://satellite.example.com" }, + { is_satellite: false, frontend_api_url: "https://clerk.example.com" }, + ]); + + await fetchInstanceSettings("sk_test_abc"); + + // Compared as a parsed hostname rather than a substring: the satellite's + // domain can appear anywhere in a URL — in a path or a query parameter — + // so `includes` would pass a request that never went near that host, and + // fail one that did. + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.every((url) => new URL(url).hostname !== "satellite.example.com")).toBe(true); + }); + + test("skips the dev browser bootstrap for a production key", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + + expect(await fetchInstanceSettings("sk_live_abc")).toEqual(USER_SETTINGS); + + const urls = mockFetch.mock.calls.map(([input]) => String(input)); + expect(urls.some((url) => url.includes("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/v1/dev_browser"))).toBe(false); + }); + + // A missing settings read lets the checks pass users the instance refuses. + test("retries a rate-limited domains lookup", async () => { + route([{ is_satellite: false, frontend_api_url: "https://clerk.example.com" }]); + const routed = mockFetch.getMockImplementation()!; + let domainCalls = 0; + mockFetch.mockImplementation((input: string | URL) => { + if (String(input).includes("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/v1/domains") && ++domainCalls === 1) { + return Promise.resolve( + new Response( + JSON.stringify({ errors: [{ code: "too_many_requests", message: "slow" }] }), + { + status: 429, + headers: { "Content-Type": "application/json", "Retry-After": "0.01" }, + }, + ), + ); + } + return routed(input); + }); + + expect(await fetchInstanceSettings("sk_test_abc")).toEqual(USER_SETTINGS); + expect(domainCalls).toBe(2); + }); + + // `null` means "unknown", so callers degrade rather than treating a failed + // lookup as "nothing is enabled". + test("returns null when no domain names a Frontend API URL", async () => { + route([{ is_satellite: false }]); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); + + test("returns null when the domains lookup fails", async () => { + mockFetch.mockResolvedValue(new Response("nope", { status: 401 })); + expect(await fetchInstanceSettings("sk_test_abc")).toBeNull(); + }); +}); + +describe("fetchUserCount", () => { + const originalFetch = globalThis.fetch; + useCaptureLog(); + const mockFetch = mock(); + + beforeEach(() => { + mockFetch.mockReset(); + stubFetch(mockFetch); + }); + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + // A missing count checks a dev instance's quota as if it were empty. + test("retries a rate-limited count", async () => { + let calls = 0; + mockFetch.mockImplementation(() => + Promise.resolve( + ++calls === 1 + ? new Response( + JSON.stringify({ errors: [{ code: "too_many_requests", message: "slow" }] }), + { + status: 429, + headers: { "Content-Type": "application/json", "Retry-After": "0.01" }, + }, + ) + : json({ object: "total_count", total_count: 42 }), + ), + ); + + expect(await fetchUserCount("sk_test_abc")).toBe(42); + expect(calls).toBe(2); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/clerk-config.ts b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts new file mode 100644 index 000000000..4897ae548 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/clerk-config.ts @@ -0,0 +1,202 @@ +/** + * The destination instance's live user settings: which identifiers it accepts + * and requires, and which social providers it has enabled. + * + * Ported from the standalone migration-tool's `src/lib/clerk.ts`, rewritten + * onto the CLI's own primitives: the FAPI host comes from BAPI `/v1/domains` + * — a secret key is all `migrate import` is given — and the settings come from + * `lib/fapi.ts` rather than a bespoke fetch. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import { + bootstrapDevBrowser, + fetchUserSettings, + type UserSettingsJSON, +} from "../../../lib/fapi.ts"; +import { log } from "../../../lib/log.ts"; +import { detectInstanceType } from "./instance.ts"; +import { retryOn429 } from "./retry.ts"; + +/** + * Supabase provider keys whose Clerk strategy is not simply `oauth_`. + * + * Everything not listed here maps by prefix, which covers google, github, + * discord, spotify, twitch, notion, figma, gitlab, bitbucket and the rest. + */ +const CLERK_STRATEGY_ALIASES: Record = { + azure: "oauth_microsoft", + twitter: "oauth_x", + slack_oidc: "oauth_slack", + fly: "oauth_fly", +}; + +/** Supabase's provider key as Clerk's OAuth strategy name. */ +export function toClerkStrategy(provider: string): string { + return CLERK_STRATEGY_ALIASES[provider] ?? `oauth_${provider}`; +} + +/** + * The OAuth strategies Clerk has built in: clerk_go's registration in + * `api/shared/sso/oauth.go`, less the customer-specific ones. Nothing the CLI + * can call lists them, and an instance's settings name only the providers it + * has configured. + * + * ponytail: a copy of that list; check it there when Clerk adds a provider. + */ +const CLERK_OAUTH_STRATEGIES = new Set([ + "oauth_agentid", + "oauth_apple", + "oauth_atlassian", + "oauth_bitbucket", + "oauth_box", + "oauth_coinbase", + "oauth_discord", + "oauth_dropbox", + "oauth_facebook", + "oauth_github", + "oauth_gitlab", + "oauth_google", + "oauth_hubspot", + "oauth_huggingface", + "oauth_instagram", + "oauth_line", + "oauth_linear", + "oauth_linkedin", + "oauth_linkedin_oidc", + "oauth_microsoft", + "oauth_notion", + "oauth_slack", + "oauth_spotify", + "oauth_tiktok", + "oauth_twitch", + "oauth_twitter", + "oauth_vercel", + "oauth_x", + "oauth_xero", +]); + +/** + * True when Clerk offers a Supabase provider at all. One it doesn't (Figma, + * Kakao, Keycloak, WorkOS, Zoom, Fly) can't be turned on, so the checks offer + * no fix for it. + */ +export function clerkOffersProvider(provider: string): boolean { + return CLERK_OAUTH_STRATEGIES.has(toClerkStrategy(provider)); +} + +/** Human label for a provider key, for report output. */ +export function providerLabel(provider: string): string { + const special: Record = { + github: "GitHub", + gitlab: "GitLab", + linkedin_oidc: "LinkedIn (OIDC)", + slack_oidc: "Slack (OIDC)", + twitter: "Twitter (X)", + azure: "Microsoft (Azure)", + workos: "WorkOS", + fly: "Fly.io", + }; + return special[provider] ?? provider.charAt(0).toUpperCase() + provider.slice(1); +} + +/** + * The Frontend API host of the instance a secret key addresses. + * + * `/v1/instance` carries no publishable key — for any instance, linked or not + * — so the primary domain's `frontend_api_url` is the only route from a secret + * key to the host its settings live behind. Every instance has at least one + * domain; satellites share the primary's Frontend API, so ordering only + * matters for tidiness. + */ +async function fetchFapiHost(secretKey: string): Promise { + const response = await bapiRequest({ method: "GET", path: "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/v1/domains", secretKey }); + const domains = (response.body as { data?: unknown })?.data; + if (!Array.isArray(domains)) return null; + + const primary = + domains.find((domain) => !(domain as { is_satellite?: boolean }).is_satellite) ?? domains[0]; + const frontendApiUrl = (primary as { frontend_api_url?: unknown })?.frontend_api_url; + if (typeof frontendApiUrl !== "string" || !frontendApiUrl) return null; + + return new URL(frontendApiUrl).host; +} + +/** + * Fetches the user settings for the instance a secret key addresses. + * + * @returns The settings, or `null` when they could not be read. Callers must + * treat `null` as "unknown" rather than as "nothing is enabled" — the + * import checks say so and skip the checks that need them. + */ +export async function fetchInstanceSettings(secretKey: string): Promise { + try { + // A 429 is retried: under load, a missing settings read would let the + // checks pass users the instance then refuses one create at a time. + const fapiHost = await retryOn429(async () => fetchFapiHost(secretKey)); + if (!fapiHost) { + log.debug("migrate: no domain on this instance named a Frontend API URL"); + return null; + } + + // Development FAPI rejects an environment request without a dev browser JWT. + const jwt = + detectInstanceType(secretKey) === "dev" + ? await retryOn429(async () => bootstrapDevBrowser(fapiHost)) + : undefined; + return await retryOn429(async () => fetchUserSettings(fapiHost, jwt ? { jwt } : {})); + } catch (error) { + log.debug( + `migrate: could not read instance settings: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} + +/** The enabled social strategies (`oauth_google`, …) in a settings payload. */ +export function enabledSocialProviders(settings: UserSettingsJSON): string[] { + return Object.entries(settings.social ?? {}) + .filter(([, value]) => value?.enabled) + .map(([strategy]) => strategy); +} + +/** + * Convenience wrapper for callers that only need the enabled strategies. + * + * @returns `null` when the instance settings could not be read. + */ +export async function fetchEnabledSocialProviders(secretKey: string): Promise { + const settings = await fetchInstanceSettings(secretKey); + return settings ? enabledSocialProviders(settings) : null; +} + +/** + * How many users the destination instance already holds. + * + * The closest thing to a live quota check the CLI has: `max_allowed_users` is + * not exposed by any public API, so headroom on a development instance can + * only be estimated from the count and {@link DEV_USER_LIMIT}. + * + * @returns `null` when the count could not be read — an unknown count must not + * be reported as zero. + */ +export async function fetchUserCount(secretKey: string): Promise { + try { + // A 429 is retried, as the settings read is: a missing count checks a dev + // instance's quota as if it were empty. + const response = await retryOn429(async () => + bapiRequest({ method: "GET", path: "/v1/users/count", secretKey }), + ); + const total = (response.body as { total_count?: unknown })?.total_count; + return typeof total === "number" ? total : null; + } catch (error) { + log.debug( + `migrate: could not read the instance's user count: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return null; + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/db.test.ts b/packages/cli-core/src/commands/migrate/lib/db.test.ts new file mode 100644 index 000000000..7453b4db8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.test.ts @@ -0,0 +1,405 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { + createDbClient, + describeDbError, + detectDbType, + redactConnectionString, + redactDbMessage, + sqlitePath, + withDbClient, +} from "./db.ts"; + +let workDir: string; +let dbPath: string; + +beforeAll(() => { + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-db-"))); + dbPath = path.join(workDir, "test.sqlite"); + + const db = new Database(dbPath, { create: true }); + db.run(`CREATE TABLE "user" (id TEXT PRIMARY KEY, email TEXT, "emailVerified" INTEGER)`); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u1", "a@x.dev", 1]); + db.run(`INSERT INTO "user" VALUES (?, ?, ?)`, ["u2", "b@x.dev", 0]); + db.close(); +}); + +afterAll(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("detectDbType", () => { + test.each([ + ["postgres://u:p@h/db", "postgres"], + ["postgresql://u:p@h/db", "postgres"], + ["POSTGRES://u:p@h/db", "postgres"], + ["mysql://u:p@h/db", "mysql"], + ["mysql2://u:p@h/db", "mysql"], + ["./db.sqlite", "sqlite"], + ["libsql://app-org.turso.io", "sqlite"], + ["file:./db.sqlite", "sqlite"], + ["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/abs/path.db", "sqlite"], + [" postgres://u:p@h/db ", "postgres"], + ])("%s -> %s", (input, expected) => { + expect(detectDbType(input)).toBe(expected as never); + }); +}); + +describe("redactConnectionString", () => { + test.each([ + ["postgres://user:secret@host:5432/db", "postgres://***@host:5432/db"], + ["mysql://root:hunter2@127.0.0.1:3306/app", "mysql://***@127.0.0.1:3306/app"], + ["postgres://host/db", "postgres://host/db"], + ["postgres-x://u:secret@host/db", "postgres-x://***@host/db"], + ["libsql://app.turso.io?authToken=secret", "libsql://app.turso.io?authToken=***"], + ["postgres://u@host/db?password=secret", "postgres://***@host/db?password=***"], + [ + "postgres://host/db?sslmode=require&password=secret", + "postgres://host/db?sslmode=require&password=***", + ], + ])("%s -> %s", (input, expected) => { + expect(redactConnectionString(input)).toBe(expected); + }); + + // An unencoded `@` in the password is the most common mistake, and it is + // exactly when the string ends up in an error message. Matching the first + // `@` would leave the rest of the password visible. + test("redacts a password containing an unencoded @", () => { + const redacted = redactConnectionString("postgres://user:pa@ss@host/db"); + expect(redacted).toBe("postgres://***@host/db"); + expect(redacted).not.toContain("ss"); + }); + + test("redacts a password containing a colon", () => { + expect(redactConnectionString("postgres://user:a:b:c@host/db")).toBe("postgres://***@host/db"); + }); + + test.each([["./db.sqlite"], ["/var/data/app.db"], ["file:./local.sqlite"]])( + "leaves the credential-free path %s alone", + (input) => { + expect(redactConnectionString(input)).toBe(input); + }, + ); +}); + +describe("redactDbMessage", () => { + test.each([ + ["the whole URL", "failed: postgres://u:pw%40x@h/db", "failed: postgres://***@h/db"], + ["the password, raw", 'auth failed for "pw%40x"', 'auth failed for "***"'], + ["the password, decoded", 'auth failed for "pw@x"', 'auth failed for "***"'], + ])("redacts %s", (_label, message, expected) => { + expect(redactDbMessage(message, "postgres://u:pw%40x@h/db")).toBe(expected); + }); + + test("redacts a libsql token from the URL or the environment", () => { + expect(redactDbMessage("token abc is bad", "libsql://h.turso.io?authToken=abc")).toBe( + "token *** is bad", + ); + expect( + redactDbMessage("token envtok is bad", "libsql://h.turso.io", { TURSO_AUTH_TOKEN: "envtok" }), + ).toBe("token *** is bad"); + }); + + test("leaves a message with no credentials alone", () => { + expect(redactDbMessage("no such table: user", "./db.sqlite")).toBe("no such table: user"); + }); +}); + +describe("sqlitePath", () => { + test.each([ + ["./db.sqlite", "./db.sqlite"], + ["file:./db.sqlite", "./db.sqlite"], + ["file:/abs/db.sqlite", "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/abs/db.sqlite"], + ["./db.sqlite?mode=ro", "./db.sqlite"], + [" ./db.sqlite ", "./db.sqlite"], + ])("%s -> %s", (input, expected) => { + expect(sqlitePath(input)).toBe(expected); + }); +}); + +describe("a libsql client", () => { + const originalFetch = globalThis.fetch; + let requests: { url: string; token?: string; body: any; signal?: AbortSignal | null }[] = []; + + function stubFetch(result: unknown) { + requests = []; + globalThis.fetch = (async (url: string, init: RequestInit) => { + requests.push({ + url: String(url), + token: (init.headers as Record).authorization, + body: JSON.parse(String(init.body)), + signal: init.signal, + }); + return new Response(JSON.stringify({ results: [result, { type: "ok" }] }), { + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + const okRows = (cols: string[], rows: unknown[][]) => ({ + type: "ok", + response: { type: "execute", result: { cols: cols.map((name) => ({ name })), rows } }, + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("posts to the pipeline endpoint with the URL's token and decodes rows", async () => { + stubFetch( + okRows( + ["id", "count", "verified", "missing", "hash"], + [ + [ + { type: "text", value: "u1" }, + { type: "integer", value: "12" }, + { type: "float", value: 1.5 }, + { type: "null" }, + { type: "blob", base64: Buffer.from("hash").toString("base64") }, + ], + ], + ), + ); + + const client = await createDbClient("libsql://app-org.turso.io?authToken=t0ken"); + const rows = await client.query('SELECT * FROM "user" WHERE id = ?', ["u1"]); + await client.close(); + + expect(requests[0]?.url).toBe("https://app-org.turso.io/v2/pipeline"); + expect(requests[0]?.token).toBe("Bearer t0ken"); + expect(requests[0]?.signal).toBeInstanceOf(AbortSignal); + expect(requests.at(-1)?.body.requests[0].stmt.args).toEqual([{ type: "text", value: "u1" }]); + expect(rows).toEqual([ + { + id: "u1", + count: 12, + verified: 1.5, + missing: null, + hash: Buffer.from("hash"), + }, + ] as never); + expect(client.dbType).toBe("sqlite"); + }); + + // A server that echoes the token must not get it printed. + test("redacts the token from a server error it echoes", async () => { + stubFetch({ type: "error", error: { message: "bad auth token t0ken for app-org" } }); + + const error = (await createDbClient("libsql://app-org.turso.io?authToken=t0ken").catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).not.toContain("t0ken"); + expect(error.message).toContain("bad auth token *** for app-org"); + }); + + test("redacts the token from a query failure after connecting", async () => { + stubFetch(okRows(["1"], [[{ type: "integer", value: "1" }]])); + + const error = (await withDbClient( + "libsql://app-org.turso.io?authToken=t0ken", + undefined, + async () => { + throw new Error("query rejected for token t0ken"); + }, + ).catch((caught: unknown) => caught)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("query rejected for token ***"); + expect(error.message).not.toContain("t0ken"); + }); + + test("reports a server-side error", async () => { + stubFetch({ type: "error", error: { message: "no such table: user" } }); + + await expect(createDbClient("libsql://app-org.turso.io")).rejects.toThrow(CliError); + }); +}); + +describe("a sqlite client", () => { + test("connects and queries", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ id: string }>(`SELECT id FROM "user" ORDER BY id`); + expect(rows.map((row) => row.id)).toEqual(["u1", "u2"]); + } finally { + await client.close(); + } + }); + + test("binds parameters", async () => { + const client = await createDbClient(dbPath); + try { + const rows = await client.query<{ email: string }>(`SELECT email FROM "user" WHERE id = ?`, [ + "u2", + ]); + expect(rows[0]?.email).toBe("b@x.dev"); + } finally { + await client.close(); + } + }); + + test("reports its dialect's placeholder and quoting", async () => { + const client = await createDbClient(dbPath); + try { + expect(client.dbType).toBe("sqlite"); + expect(client.placeholder(1)).toBe("?"); + expect(client.quote("user")).toBe('"user"'); + } finally { + await client.close(); + } + }); + + test("accepts a file: URL", async () => { + const client = await createDbClient(`file:${dbPath}`); + try { + expect(await client.query(`SELECT 1 AS n`)).toHaveLength(1); + } finally { + await client.close(); + } + }); + + // bun:sqlite opens lazily, so without an explicit probe a missing file would + // surface at the first real query, long after "connecting" finished. + test("fails at connect time when the file is missing, not mid-export", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow(CliError); + }); + + test("names the file in the failure", async () => { + await expect(createDbClient(path.join(workDir, "nope.sqlite"))).rejects.toThrow( + /Could not open the SQLite file/, + ); + }); +}); + +describe("withDbClient", () => { + test("returns the work's value", async () => { + expect(await withDbClient(dbPath, undefined, async () => "done")).toBe("done"); + }); + + test("closes the client even when the work throws", async () => { + // A leaked handle keeps the process alive after the export has written its + // file, which reads as a hang. + await expect( + withDbClient(dbPath, undefined, async () => { + throw new Error("boom"); + }), + ).rejects.toThrow(/boom/); + + // The file is still usable, so nothing is holding it open. + expect(await withDbClient(dbPath, undefined, async () => "reopened")).toBe("reopened"); + }); + + test("attaches a hint to a query failure, not just a connection failure", async () => { + await expect( + withDbClient(dbPath, undefined, (client) => client.query(`SELECT * FROM missing_table`)), + ).rejects.toThrow(/expected table was not found/); + }); + + // A schema the export cannot read is a refusal (exit 2), and the advice is + // about the columns, not the connection. + test("names the columns for a missing one, and exits 2", async () => { + const db = new Database(dbPath); + db.run( + `CREATE TABLE IF NOT EXISTS "User" (id TEXT, name TEXT, email TEXT, email_verified TEXT)`, + ); + db.close(); + + const error = (await withDbClient(dbPath, "authjs", (client) => + client.query(`SELECT no_such_column FROM "User"`), + ).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("reads `id`, `name`, `email` and `emailVerified`"); + // Prisma's Auth.js schema: a quoted "User" table and camelCase "emailVerified". + expect(error.message).toContain( + 'SELECT id, full_name AS name, email, "emailVerified" AS email_verified FROM "User"', + ); + expect(error.message).not.toContain("Check the connection string"); + }); + + // Supabase and Better Auth get the generic advice; only Auth.js has its own. + test("gives the generic column advice outside Auth.js, and exits 2", async () => { + const error = (await withDbClient(dbPath, "supabase", (client) => + client.query(`SELECT no_such_column FROM sqlite_master`), + ).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("missing a column the export reads"); + expect(error.message).not.toContain("reads `id`, `name`, `email`"); + expect(error.message).not.toContain("Check the connection string"); + }); + + test("passes a CliError through unchanged", async () => { + await expect( + withDbClient(dbPath, undefined, async () => { + throw new CliError("already explained"); + }), + ).rejects.toThrow(/already explained/); + }); +}); + +describe("describeDbError", () => { + const withCode = (code: string, message = "") => Object.assign(new Error(message), { code }); + + // Bun reports an unreachable host and a closed port identically, as + // "Connection closed" — precisely where a bare driver error helps least. + test.each([ + ["ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"], + ["ERR_MYSQL_CONNECTION_CLOSED", "Connection closed"], + ])("turns %s into host/port guidance", (code, message) => { + expect(describeDbError(withCode(code, message))).toMatch(/Check the host and port/); + }); + + test("gives Supabase the IPv4 add-on hint, which is the usual cause there", () => { + const hint = describeDbError( + withCode("ERR_POSTGRES_CONNECTION_CLOSED", "Connection closed"), + "supabase", + ); + expect(hint).toMatch(/pooler connection string/); + expect(hint).toMatch(/IPv4/); + }); + + test.each([ + ['password authentication failed for user "postgres"'], + ["Access denied for user 'root'@'localhost' (using password: YES)"], + ])("recognizes the rejected credentials in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/rejected those credentials/); + }); + + test.each([ + ['relation "auth.users" does not exist'], + ["no such table: user"], + ["permission denied for table users"], + ["Table 'app.users' doesn't exist"], + ["Unknown table 'users'"], + ])("recognizes the missing table in %p", (message) => { + expect(describeDbError(new Error(message))).toMatch(/table was not found|cannot read it/); + }); + + test("points Supabase at Auth being enabled and the postgres role", () => { + const hint = describeDbError(new Error('relation "auth.users" does not exist'), "supabase"); + expect(hint).toMatch(/Supabase Auth is enabled/); + expect(hint).toMatch(/postgres` role/); + }); + + test("recognizes an unopenable SQLite file", () => { + expect(describeDbError(new Error("unable to open database file"))).toMatch( + /Could not open the SQLite file/, + ); + }); + + test("still says something useful for an error it does not recognize", () => { + expect(describeDbError(new Error("something odd"))).toMatch(/Check the connection string/); + }); + + test("never echoes the error's own text, which could carry a connection string", () => { + const hint = describeDbError(new Error("failed for postgres://user:secret@host/db")); + expect(hint).not.toContain("secret"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/db.ts b/packages/cli-core/src/commands/migrate/lib/db.ts new file mode 100644 index 000000000..90e8ec76a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/db.ts @@ -0,0 +1,455 @@ +/** + * One query interface over Postgres, MySQL and SQLite. + * + * Rewritten from the standalone migration-tool's `src/lib/db.ts`, which used + * `pg`, `mysql2` and `better-sqlite3`. None of those belong in a statically + * compiled binary — `better-sqlite3` is a native addon outright — so this runs + * on `Bun.sql` (Postgres and MySQL) and `bun:sqlite`, both built into the + * runtime. That swap is the entire reason the `engines.bun` floor exists. + * + * **Placeholders are not unified.** `Bun.sql` passes the query through to each + * server as written, so Postgres wants `$1` and MySQL wants `?` — verified + * against both. Rather than rewrite SQL strings (Postgres uses `?` as a JSONB + * operator, so a naive rewriter would corrupt real queries), callers ask the + * client for the placeholder and the identifier quoting they need. They already + * build per-dialect SQL for table casing, so this adds no new branching. + */ + +import { Database } from "bun:sqlite"; +import { SQL } from "bun"; +import { CliError, ERROR_CODE, EXIT_CODE } from "../../../lib/errors.ts"; + +export type DbType = "postgres" | "mysql" | "sqlite"; + +export interface DbClient { + dbType: DbType; + query>(sql: string, params?: unknown[]): Promise; + /** The bind placeholder for the 1-indexed `position`. */ + placeholder(position: number): string; + /** Quotes an identifier for this dialect. */ + quote(identifier: string): string; + close(): Promise; +} + +/** + * Reads the database type from a connection string. + * + * Anything that is not a recognized URL scheme is treated as a SQLite path, + * matching how the standalone tool behaved and how users actually pass + * `./db.sqlite`. `libsql://` (Turso) is SQLite too — it only differs in how + * the rows are fetched, so callers that branch on the dialect want "sqlite". + */ +export function detectDbType(connectionString: string): DbType { + const lower = connectionString.trim().toLowerCase(); + if (lower.startsWith("postgresql://") || lower.startsWith("postgres://")) return "postgres"; + if (lower.startsWith("mysql://") || lower.startsWith("mysql2://")) return "mysql"; + return "sqlite"; +} + +/** True for a remote libsql/Turso URL, which is read over HTTP rather than opened. */ +export function isLibsqlUrl(connectionString: string): boolean { + return /^libsql:\/\//i.test(connectionString.trim()); +} + +/** + * Replaces any credentials in a connection string with `***`. + * + * Connection strings reach the CLI on the command line and end up in error + * messages and `--verbose` output. Bun's own errors do not echo them, and + * nothing here should either. + */ +export function redactConnectionString(connectionString: string): string { + // Greedy up to the LAST `@`: an unencoded `@` in the password is the most + // common connection-string mistake, and matching the first one would leave + // the rest of the password in the message. Everything before the final `@` + // is userinfo, so redacting all of it is always safe. + // Non-URL forms (SQLite paths) have no `://` and are left alone. + // Turso carries its credential as `?authToken=`, not as userinfo, and a + // `?password=` (which Bun.SQL ignores) still must not be printed. + return connectionString + .replace(/^([a-z][a-z0-9+.-]*:\/\/)(.*)@/i, "$1***@") + .replace(/([?&](?:authToken|password)=)[^&]*/gi, "$1***"); +} + +/** + * A driver's error message with every credential from `connectionString` + * replaced by `***`. + * + * {@link redactConnectionString} covers the string we print; this covers what + * the server or driver says back, which can quote the URL, the password or a + * libsql token. A short password can blank an unrelated word too, and a + * mangled message is the better failure. + */ +export function redactDbMessage( + message: string, + connectionString: string, + env: Record = process.env, +): string { + const trimmed = connectionString.trim(); + let redacted = message.split(trimmed).join(redactConnectionString(trimmed)); + + const secrets = new Set(); + try { + const url = new URL(trimmed); + for (const value of [ + url.password, + url.searchParams.get("authToken"), + url.searchParams.get("password"), + ]) { + if (!value) continue; + secrets.add(value); + try { + secrets.add(decodeURIComponent(value)); + } catch { + // Not valid percent-encoding: the raw form is the only one there is. + } + } + } catch { + // A SQLite path is not a URL, and carries no credentials. + } + if (isLibsqlUrl(trimmed)) { + for (const value of [env.TURSO_AUTH_TOKEN, env.LIBSQL_AUTH_TOKEN]) { + if (value) secrets.add(value); + } + } + + // Longest first, so a secret that contains another is replaced whole. + for (const secret of [...secrets].sort((a, b) => b.length - a.length)) { + redacted = redacted.split(secret).join("***"); + } + return redacted; +} + +/** Strips a `file:` prefix and any URL query, leaving a filesystem path. */ +export function sqlitePath(connectionString: string): string { + const trimmed = connectionString.trim(); + const withoutScheme = trimmed.startsWith("file:") ? trimmed.slice("file:".length) : trimmed; + return withoutScheme.split("?")[0] ?? withoutScheme; +} + +const QUOTING: Record string> = { + // Doubling the delimiter is the escape in every dialect here, so an + // identifier containing one cannot break out of the quotes. + postgres: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + sqlite: (identifier) => `"${identifier.replace(/"/g, '""')}"`, + mysql: (identifier) => `\`${identifier.replace(/`/g, "``")}\``, +}; + +function bunSqlClient(connectionString: string, dbType: "postgres" | "mysql"): DbClient { + const sql = new SQL(connectionString); + + return { + dbType, + async query>(query: string, params: unknown[] = []) { + const rows = await sql.unsafe(query, params); + return (Array.isArray(rows) ? rows : []) as T[]; + }, + placeholder: dbType === "postgres" ? (position) => `$${position}` : () => "?", + quote: QUOTING[dbType], + async close() { + await sql.close(); + }, + }; +} + +/** + * One value in Hrana's wire format, the protocol libsql servers speak. + * + * Integers arrive as strings so 64-bit values survive JSON. + */ +type HranaValue = { type: string; value?: string | number; base64?: string }; + +function decodeHrana(value: HranaValue): unknown { + switch (value.type) { + case "null": + return null; + case "integer": { + const raw = String(value.value ?? "0"); + const asNumber = Number(raw); + // Past 2^53 a number would silently lose digits; ids can get that big. + return Number.isSafeInteger(asNumber) ? asNumber : BigInt(raw); + } + case "float": + return Number(value.value); + case "blob": + // bun:sqlite hands back bytes for a BLOB, so this does too. + return Buffer.from(value.base64 ?? "", "base64"); + default: + return value.value ?? null; + } +} + +function encodeHrana(param: unknown): HranaValue { + if (param === null || param === undefined) return { type: "null" }; + if (typeof param === "bigint") return { type: "integer", value: param.toString() }; + if (typeof param === "boolean") return { type: "integer", value: param ? "1" : "0" }; + if (typeof param === "number") { + return Number.isInteger(param) + ? { type: "integer", value: String(param) } + : { type: "float", value: param }; + } + if (param instanceof Uint8Array) { + return { type: "blob", base64: Buffer.from(param).toString("base64") }; + } + return { type: "text", value: String(param) }; +} + +/** + * Talks to a libsql server (Turso) over its HTTP pipeline endpoint. + * + * `bun:sqlite` opens local files and cannot reach a remote database, and + * `@libsql/client` ships native optional dependencies that do not survive + * `bun build --compile`. The protocol is one POST per statement, so it is + * fewer lines to speak it directly than to carry the dependency. + * + * The token comes from `?authToken=` on the URL — the form the Turso CLI + * prints — or from `TURSO_AUTH_TOKEN`/`LIBSQL_AUTH_TOKEN`. A self-hosted sqld + * with auth disabled needs neither, so a missing token is not an error here. + */ +const LIBSQL_TIMEOUT_MS = 120_000; + +function libsqlClient( + connectionString: string, + env: Record = process.env, +): DbClient { + const url = new URL(connectionString.trim()); + const token = url.searchParams.get("authToken") || env.TURSO_AUTH_TOKEN || env.LIBSQL_AUTH_TOKEN; + const endpoint = `https://${url.host}/v2/pipeline`; + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + const response = await fetch(endpoint, { + method: "POST", + // Generous, since one unpaginated SELECT can be a whole users table. + signal: AbortSignal.timeout(LIBSQL_TIMEOUT_MS), + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + }, + // `close` keeps every request stateless: no baton to carry forward. + body: JSON.stringify({ + requests: [ + { type: "execute", stmt: { sql: query, args: params.map(encodeHrana) } }, + { type: "close" }, + ], + }), + }); + + if (!response.ok) { + throw new Error(`libsql request failed: ${response.status} ${response.statusText}`.trim()); + } + + const body = (await response.json()) as { + results?: { + type: string; + error?: { message?: string }; + response?: { result?: { cols?: { name?: string }[]; rows?: HranaValue[][] } }; + }[]; + }; + + const first = body.results?.[0]; + if (!first || first.type === "error") { + throw new Error(first?.error?.message ?? "libsql returned no result"); + } + + const cols = first.response?.result?.cols ?? []; + const rows = first.response?.result?.rows ?? []; + return rows.map( + (row) => + Object.fromEntries( + row.map((value, index) => [cols[index]?.name ?? String(index), decodeHrana(value)]), + ) as T, + ); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + return Promise.resolve(); + }, + }; +} + +function sqliteClient(connectionString: string): DbClient { + const database = new Database(sqlitePath(connectionString), { readonly: true }); + + return { + dbType: "sqlite", + async query>(query: string, params: unknown[] = []) { + // bun:sqlite is synchronous; the Promise keeps one interface for callers. + return Promise.resolve(database.query(query).all(...(params as never[])) as T[]); + }, + placeholder: () => "?", + quote: QUOTING.sqlite, + async close() { + database.close(); + return Promise.resolve(); + }, + }; +} + +/** + * Connects to the database a connection string names. + * + * @param platform - Tailors the failure hint; the same "Connection closed" + * means something different on Supabase than on a local SQLite file. + */ +export async function createDbClient( + connectionString: string, + platform?: DbPlatform, +): Promise { + const dbType = detectDbType(connectionString); + + try { + if (isLibsqlUrl(connectionString)) { + const client = libsqlClient(connectionString); + await client.query("SELECT 1"); + return client; + } + + if (dbType === "sqlite") { + const client = sqliteClient(connectionString); + // bun:sqlite opens lazily, so a missing file would not surface until the + // first real query — long after the "connecting" spinner has stopped. + await client.query("SELECT 1"); + return client; + } + + const client = bunSqlClient(connectionString, dbType); + await client.query("SELECT 1"); + return client; + } catch (error) { + throw connectionError(error, connectionString, platform); + } +} + +export type DbPlatform = "supabase" | "betterauth" | "authjs"; + +/** + * Turns a driver error into something a user can act on. + * + * Rewritten rather than ported: the standalone tool matched on `pg`'s + * `ENOTFOUND`/`ETIMEDOUT`, which `Bun.sql` never emits. Bun reports both an + * unreachable host and a closed port as `ERR_*_CONNECTION_CLOSED` with the + * message "Connection closed" — exactly the case where a bare driver error + * helps least. + */ +export function describeDbError(error: unknown, platform?: DbPlatform): string { + const message = error instanceof Error ? error.message : String(error); + const code = (error as { code?: string })?.code ?? ""; + + if (code.includes("CONNECTION_CLOSED") || /connection closed|econnrefused/i.test(message)) { + if (platform === "supabase") { + return ( + "Could not reach the database. Check the host and port in the connection string.\n" + + "Supabase direct connections need the IPv4 add-on — use the pooler connection string\n" + + "(Dashboard → Connect → Session pooler), or enable IPv4 under Settings → Add-Ons." + ); + } + return "Could not reach the database. Check the host and port, and that the server accepts connections from here."; + } + + // Turso resolves every `*.turso.io` name, so a typo'd database does not fail + // to connect — it answers 404. "Check the host" would send the reader after + // the half that is right. + if (/\b404\b/.test(message)) { + return ( + "No database at that libsql host. Check the database name in the URL —\n" + + "`turso db show ` prints the URL to use." + ); + } + + if (/\b401\b|unauthorized|not authorized/i.test(message)) { + return ( + "The libsql server rejected that token.\n" + + "Append ?authToken=… to the URL, or set TURSO_AUTH_TOKEN (`turso db tokens create `)." + ); + } + + if (/password authentication failed|access denied/i.test(message)) { + return "The database rejected those credentials. Check the user and password in the connection string."; + } + + // Before the table check: Postgres words a missing column "does not exist" too. + if (/no such column|column .* does not exist|unknown column/i.test(message)) { + const needs: Partial> = { + authjs: + "The Auth.js export reads `id`, `name`, `email` and `emailVerified` (or `email_verified`). " + + 'For a schema that renames another column (Prisma `@map("full_name")` on `name`, for one), ' + + "export the users with your own query that aliases it back, " + + '`SELECT id, full_name AS name, email, "emailVerified" AS email_verified FROM "User"` on Postgres ' + + "(MySQL quotes with backticks), and import that file with the authjs source.", + }; + return ( + needs[platform as DbPlatform] ?? + "The user table is missing a column the export reads. Check the schema matches the platform's default." + ); + } + + // MySQL says "doesn't exist" and "Unknown table" where Postgres says "does not exist". + if ( + /does not exist|doesn't exist|unknown table|unknown database|no such table|permission denied/i.test( + message, + ) + ) { + if (platform === "supabase") { + return ( + "The auth.users table was not readable. It is created automatically when Supabase Auth is enabled.\n" + + "Check Authentication is enabled, and connect as the `postgres` role rather than an application role." + ); + } + return "The expected table was not found, or the user cannot read it. Check the database name and the user's SELECT permission."; + } + + if (/unable to open database|sqlitecantopen|no such file/i.test(message)) { + return "Could not open the SQLite file. Check the path, and that the file exists and is readable."; + } + + return "Check the connection string, that the server is running, and that it is reachable from here."; +} + +function connectionError( + error: unknown, + connectionString: string, + platform?: DbPlatform, +): CliError { + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); + return new CliError( + `Could not connect to ${redactConnectionString(connectionString)}: ${message}\n\n${describeDbError(error, platform)}`, + { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }, + ); +} + +/** + * Runs `work` against a fresh client and always closes it. + * + * A leaked connection keeps the process alive after the export has written its + * file, which looks like a hang. + */ +export async function withDbClient( + connectionString: string, + platform: DbPlatform | undefined, + work: (client: DbClient) => Promise, +): Promise { + const client = await createDbClient(connectionString, platform); + try { + return await work(client); + } catch (error) { + // A query failure carries the same actionable hints as a connection one: + // a missing table is the most common thing that goes wrong here. + if (error instanceof CliError) throw error; + const message = redactDbMessage( + error instanceof Error ? error.message : String(error), + connectionString, + ); + throw new CliError(`${message}\n\n${describeDbError(error, platform)}`, { + code: ERROR_CODE.USAGE_ERROR, + exitCode: EXIT_CODE.USAGE, + }); + } finally { + await client.close().catch(() => {}); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/export-file.ts b/packages/cli-core/src/commands/migrate/lib/export-file.ts new file mode 100644 index 000000000..c9ebb499d --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/export-file.ts @@ -0,0 +1,89 @@ +/** + * The file `clerk migrate export` writes: the users, wrapped in an envelope + * that says where they came from. + * + * The envelope is what lets `clerk migrate import ` run with no + * `--source`: the source, and Firebase's hash parameters, travel with the + * users. A bare array, a CSV, or Firebase's own `{ users: [...] }` still + * import, but need the source named. + */ + +import fs from "node:fs"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +export const ENVELOPE_VERSION = 1; + +export type ExportEnvelope = { + clerkMigrate: typeof ENVELOPE_VERSION; + source: string; + exportedAt: string; + runId: string; + firebase?: FirebaseHashConfig; + users: Record[]; +}; + +export function isEnvelope(value: unknown): value is ExportEnvelope { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const candidate = value as Partial; + return ( + candidate.clerkMigrate === ENVELOPE_VERSION && + typeof candidate.source === "string" && + Array.isArray(candidate.users) + ); +} + +/** + * The envelope in a file, when it has one. + * + * `undefined` for a CSV, a bare array, or anything unreadable: those are the + * shapes that need `--source`, and the load that follows reports what is + * actually wrong with a broken file. + */ +// ponytail: parses the file here and again when the users load; cache by path if exports get large enough to notice. +export function readEnvelope(file: string): ExportEnvelope | undefined { + if (!file.toLowerCase().endsWith(".json")) return undefined; + try { + const parsed = readJsonFile(file); + return isEnvelope(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +/** + * A JSON file's contents, as an import reads it. + * + * A UTF-8 BOM (Excel's "CSV UTF-8", some editors) is stripped. NDJSON, one + * object per line, reads as an array: it is what Auth0's bulk export job + * writes. `.ndjson` and `.jsonl` are always read that way; a `.json` file + * falls back to it only when it doesn't parse whole. + * + * @throws CliError naming the file when it is neither. + */ +export function readJsonFile(file: string): unknown { + const text = fs.readFileSync(file, "utf-8").replace(/^\uFEFF/, ""); + const lines = () => text.split("\n").filter((line) => line.trim()); + const invalid = (error: unknown) => + new CliError(`${file} is not valid JSON: ${(error as Error).message}`, { + code: ERROR_CODE.INVALID_JSON, + }); + + if (/\.(ndjson|jsonl)$/i.test(file)) { + try { + return lines().map((line) => JSON.parse(line) as unknown); + } catch (error) { + throw invalid(error); + } + } + try { + return JSON.parse(text); + } catch (error) { + try { + if (lines().length > 1) return lines().map((line) => JSON.parse(line) as unknown); + } catch { + // Not NDJSON either: report the whole-file error, which names the spot. + } + throw invalid(error); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts new file mode 100644 index 000000000..e842cbc4c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, test } from "bun:test"; +import { firebaseHashConfigProblem, resolveFirebaseHashConfig } from "./firebase-hash.ts"; + +const ALL_FLAGS = { + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, +}; + +describe("gating on the source", () => { + test.each([["clerk"], ["supabase"], ["auth0"], ["authjs"], ["betterauth"]])( + "ignores even explicit flags for the %s source", + (transformer) => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, transformer)).toBeUndefined(); + }, + ); + + test("resolves nothing before the platform is known", () => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, undefined)).toBeUndefined(); + }); +}); + +describe("on a firebase run", () => { + test("builds the config from flags", () => { + expect(resolveFirebaseHashConfig(ALL_FLAGS, "firebase")).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // A digest built from a partial set is well-formed but verifies against + // nothing, so every migrated user would silently fail to sign in. + test.each([ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], + ] as const)("rejects a flag set missing %s, naming it", (omit, flag) => { + const partial = { ...ALL_FLAGS }; + delete (partial as Record)[omit]; + + expect(() => resolveFirebaseHashConfig(partial, "firebase")).toThrow(new RegExp(flag)); + }); + + test("names every missing flag at once", () => { + expect(() => resolveFirebaseHashConfig({ firebaseSignerKey: "SIGNER" }, "firebase")).toThrow( + /--firebase-salt-separator.*--firebase-rounds.*--firebase-mem-cost/, + ); + }); + + test("returns nothing when no flag supplies a config", () => { + expect(resolveFirebaseHashConfig({}, "firebase")).toBeUndefined(); + }); +}); + +// Clerk's bounds: clerk_go pkg/hash/scrypt.go. +describe("firebaseHashConfigProblem", () => { + const good = { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + + test("passes Firebase's usual parameters", () => { + expect(firebaseHashConfigProblem(good)).toBeUndefined(); + }); + + // Clerk pads a short final group itself, and reads the URL-safe alphabet. + test.each(["Bw", "Bw==", "a-b_", "YWJjZA"])("passes %s, which Clerk decodes", (separator) => { + expect( + firebaseHashConfigProblem({ ...good, base64_salt_separator: separator }), + ).toBeUndefined(); + }); + + // Clerk decodes it to no bytes; some projects have no separator. + test("passes an empty salt separator", () => { + expect(firebaseHashConfigProblem({ ...good, base64_salt_separator: "" })).toBeUndefined(); + }); + + test.each([ + // Clerk pads to a multiple of four, then decodes strictly. + ["a one-character key", { base64_signer_key: "A" }, /signer key is not base64/], + ["padding after a full group", { base64_signer_key: "AAAA=" }, /signer key is not base64/], + ["two pads after a full group", { base64_signer_key: "AAAA==" }, /signer key is not base64/], + ["an empty key", { base64_signer_key: "" }, /signer key is not base64/], + [ + "a signer key that is not base64", + { base64_signer_key: "not base64!" }, + /signer key is not base64/, + ], + [ + "a separator with the digest's $", + { base64_salt_separator: "Bw$" }, + /salt separator is not base64/, + ], + ["rounds of 0", { rounds: 0 }, /rounds must be a whole number from 1 to 16, not 0/], + ["rounds of 17", { rounds: 17 }, /rounds must be a whole number from 1 to 16, not 17/], + ["a fractional memory cost", { mem_cost: 14.5 }, /memory cost must be a whole number/], + ])("names %s", (_label, change, expected) => { + expect(firebaseHashConfigProblem({ ...good, ...change })).toMatch(expected); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts new file mode 100644 index 000000000..28e73ca2e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/firebase-hash.ts @@ -0,0 +1,127 @@ +/** + * Firebase's four scrypt parameters, read from the `--firebase-*` flags. + * + * **Only read when the source is `firebase`.** `migrate import` is one + * command serving every platform, and nothing but the Firebase source + * reads the config off {@link TransformContext}. + */ + +import { createHash } from "node:crypto"; +import { throwUsageError } from "../../../lib/errors.ts"; +import type { FirebaseHashConfig } from "../types.ts"; + +/** The `--firebase-*` flags, keyed by their option name. */ +export const FIREBASE_FLAGS = [ + ["firebaseSignerKey", "--firebase-signer-key"], + ["firebaseSaltSeparator", "--firebase-salt-separator"], + ["firebaseRounds", "--firebase-rounds"], + ["firebaseMemCost", "--firebase-mem-cost"], +] as const; + +export type FirebaseHashFlags = { + firebaseSignerKey?: string; + firebaseSaltSeparator?: string; + firebaseRounds?: number; + firebaseMemCost?: number; +}; + +/** + * Resolves the four parameters from the flags. + * + * The four are required as a set: a digest built from a partial set is + * well-formed but verifies against nothing, so every migrated user would fail + * to sign in with no error at import time. + * + * @param source - The platform being migrated. Anything but `firebase` + * returns immediately. + * @returns The config, or `undefined` when none was supplied — which is fine + * for an export that carries no password hashes. + */ +export function resolveFirebaseHashConfig( + flags: FirebaseHashFlags, + source: string | undefined, +): FirebaseHashConfig | undefined { + if (source !== "firebase") return undefined; + + const provided = FIREBASE_FLAGS.filter(([key]) => flags[key] !== undefined); + if (provided.length === 0) return undefined; + + if (provided.length < FIREBASE_FLAGS.length) { + const missing = FIREBASE_FLAGS.filter(([key]) => flags[key] === undefined).map( + ([, flag]) => flag, + ); + throwUsageError( + `The Firebase hash parameters must be supplied together. Missing: ${missing.join(", ")}.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + return { + base64_signer_key: flags.firebaseSignerKey as string, + base64_salt_separator: flags.firebaseSaltSeparator as string, + rounds: flags.firebaseRounds as number, + mem_cost: flags.firebaseMemCost as number, + }; +} + +/** Clerk's bounds on Firebase scrypt costs (clerk_go `pkg/hash/scrypt.go`). */ +const MAX_SCRYPT_COST = 16; + +/** One whole base64 string: full groups of four, padded only at the end. */ +const STRICT_BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/; + +/** + * True when Clerk can decode it: it maps the URL-safe alphabet onto the + * standard one and pads to a multiple of four (`normalizeBase64` in clerk_go + * `pkg/hash/scrypt.go`), then decodes strictly. So `Bw` passes and `A` or + * `AAAA=` do not. + */ +export function decodesLikeClerk(value: string): boolean { + let normalized = value.replace(/-/g, "+").replace(/_/g, "/"); + const rem = normalized.length % 4; + if (rem !== 0) normalized += "=".repeat(4 - rem); + return STRICT_BASE64.test(normalized); +} + +/** + * What is wrong with a set of Firebase hash parameters, or `undefined`. + * + * Each one goes into every `scrypt_firebase` digest, so a bad one costs every + * password in the import: Clerk refuses a cost outside 1..16 and a key that is + * not base64, and a `$` would break the digest's segments. + */ +export function firebaseHashConfigProblem(config: FirebaseHashConfig): string | undefined { + // An empty salt separator is a valid one: Clerk decodes it to no bytes. + const keys = [ + ["signer key", config.base64_signer_key, false], + ["salt separator", config.base64_salt_separator, true], + ] as const; + for (const [label, value, mayBeEmpty] of keys) { + if (typeof value !== "string" || (!value && !mayBeEmpty) || !decodesLikeClerk(value)) { + return `the ${label} is not base64`; + } + } + const costs = [ + ["rounds", config.rounds], + ["memory cost", config.mem_cost], + ] as const; + for (const [label, value] of costs) { + if (!Number.isInteger(value) || value < 1 || value > MAX_SCRYPT_COST) { + return `${label} must be a whole number from 1 to ${MAX_SCRYPT_COST}, not ${String(value)}`; + } + } + return undefined; +} + +/** + * A fingerprint of the hash parameters, for a run record: two runs built + * their digests alike exactly when these match. A hash, so the signer key is + * not written to disk. + */ +export function fingerprintFirebaseHashConfig(config: FirebaseHashConfig): string { + const { base64_signer_key, base64_salt_separator, rounds, mem_cost } = config; + return createHash("sha256") + .update(JSON.stringify([base64_signer_key, base64_salt_separator, rounds, mem_cost])) + .digest("hex"); +} diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts new file mode 100644 index 000000000..9597faf78 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.test.ts @@ -0,0 +1,318 @@ +/** + * `withInputRetry` — the loop that puts a credential prompt back up when the + * far end rejects what it was given. + * + * Its own file because `mock.module` registrations last for the process, and + * `bun test --parallel` puts several files in each worker — a mocked + * `prompts.ts` would leak into any file that later lands in the same worker and + * imports the real one. + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, ERROR_CODE, EXIT_CODE, UserAbortError } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; + +let answers: string[] = []; +let cancelPrompt = false; + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../../lib/prompts.ts", () => ({ + password: async () => { + if (cancelPrompt) throw new UserAbortError(); + return answers.shift() ?? ""; + }, + text: async () => answers.shift() ?? "", + confirm: async () => true, + multiselect: async () => [], + select: async () => "", + editor: async () => "{}", + note: () => {}, +})); + +const { withInputRetry } = await import("./input-retry.ts"); +const { promptDbUrl, resolveDbUrl, withDbConnection } = await import("../export/db-options.ts"); +const { setAssumeYes } = await import("./assume-yes.ts"); + +const captured = useCaptureLog(); + +const CONFIG = { + platform: "authjs", + envVar: "AUTHJS_DB_URL", + prompt: "Auth.js database connection string", +} as const; + +const FIRST = "libsql://typo.turso.io?authToken=t"; +const SECOND = "libsql://right.turso.io?authToken=t"; + +const rejected = () => + new CliError("Could not reach it", { code: ERROR_CODE.USAGE_ERROR, exitCode: EXIT_CODE.USAGE }); + +let originalMode: Mode; + +beforeAll(() => { + originalMode = getMode(); +}); + +afterAll(() => { + setMode(originalMode); +}); + +beforeEach(() => { + setMode("human"); + setAssumeYes(false); + answers = []; + cancelPrompt = false; +}); + +describe("withInputRetry", () => { + test("returns the first result without prompting when the work succeeds", async () => { + const seen: string[] = []; + + const { value, input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(input).toBe(FIRST); + expect(seen).toEqual([FIRST]); + }); + + test("asks again after a failure and runs with the new input", async () => { + answers = [SECOND]; + const seen: string[] = []; + + const { value } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + seen.push(url); + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(value).toBe("rows"); + expect(seen).toEqual([FIRST, SECOND]); + // The operator has to be told what was wrong with a string they cannot see. + expect(captured.err).toContain("Could not reach it"); + }); + + // Later steps run against the credential that worked, not the one first tried + // — a Firebase export reads its project id off the key that Google accepted. + test("reports the input that finally worked", async () => { + answers = [SECOND]; + + const { input } = await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + if (url === FIRST) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(input).toBe(SECOND); + }); + + test("keeps asking until an input works", async () => { + answers = [FIRST, FIRST, SECOND]; + let attempts = 0; + + await withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + (url: string) => { + attempts++; + if (url !== SECOND) throw rejected(); + return Promise.resolve("rows"); + }, + ); + + expect(attempts).toBe(4); + }); + + // Cancelling the prompt is an answer: it ends the command rather than looping + // on a question the operator has already declined. + test("lets a cancelled prompt out of the loop", async () => { + cancelPrompt = true; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + throw rejected(); + }, + ), + ).rejects.toThrow(UserAbortError); + }); + + test("throws without prompting when there is nobody to ask", async () => { + setMode("agent"); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // `--json` is non-interactive by contract, even when a human is at the TTY. + test("throws without prompting under --json", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + { json: true }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // `-y` is a human on a TTY who could be asked and said not to. Agent mode + // cannot reach the prompt at all; this one can and declines to, so it needs + // its own check rather than riding on the mode assertion above. + test("throws without prompting when `-y` said not to ask", async () => { + setAssumeYes(true); + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw rejected(); + }, + ), + ).rejects.toThrow(CliError); + + expect(attempts).toBe(1); + }); + + // A bug inside the work, or an interrupt, is not a wrong answer to a prompt. + // Another credential would not fix an outage, a 429 or a refused connection. + test.each([ + [ + "a refused connection", + new CliError("Could not reach x", { code: ERROR_CODE.NETWORK_UNREACHABLE }), + ], + ["a 503", new CliError("Auth0 returned 503 listing users")], + ])("does not ask again after %s", async (_label, failure) => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw failure; + }, + ), + ).rejects.toBe(failure); + + expect(attempts).toBe(1); + }); + + test("does not retry an error the database layer did not raise", async () => { + let attempts = 0; + + await expect( + withInputRetry( + FIRST, + () => promptDbUrl(CONFIG), + () => { + attempts++; + throw new TypeError("undefined is not a function"); + }, + ), + ).rejects.toThrow(TypeError); + + expect(attempts).toBe(1); + }); +}); + +// Here rather than in `db-exports.test.ts` because the prompt is mocked: a +// missed guard reaches it and returns, where a real prompt would hang the run. +describe("resolveDbUrl", () => { + test("does not prompt under --json, even with a human at the TTY", async () => { + answers = [FIRST]; + + await expect(resolveDbUrl({ json: true }, CONFIG, {})).rejects.toThrow(/cannot prompt here/); + expect(answers).toEqual([FIRST]); + }); + + // `-y` is "do not prompt", at a terminal too. + test("does not prompt under -y, even with a human at the TTY", async () => { + answers = [FIRST]; + setAssumeYes(true); + + await expect(resolveDbUrl({}, CONFIG, {})).rejects.toThrow(/cannot prompt here/); + expect(answers).toEqual([FIRST]); + }); +}); + +// Only the connect proves the connection string. A failure after it is not a +// wrong string, so it is not asked for again, and it is not a usage error. +describe("withDbConnection", () => { + const sqlite = () => { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-dbconn-")), "db.sqlite"); + new Database(file, { create: true }).close(); + return file; + }; + + test("asks again when the connection fails, then runs the work once", async () => { + answers = [sqlite()]; + let runs = 0; + + const value = await withDbConnection("./no-such-dir/missing.sqlite", CONFIG, {}, async () => { + runs++; + return "rows"; + }); + + expect(value).toBe("rows"); + expect(runs).toBe(1); + expect(answers).toEqual([]); + }); + + test("does not ask again when the read fails after connecting, and exits 1", async () => { + answers = [sqlite()]; + const file = sqlite(); + + const error = (await withDbConnection(file, CONFIG, {}, async () => { + throw new Error("canceling statement due to statement timeout"); + }).catch((caught: unknown) => caught)) as CliError; + + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.GENERAL); + expect(error.message).toContain("statement timeout"); + expect(answers).toHaveLength(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/input-retry.ts b/packages/cli-core/src/commands/migrate/lib/input-retry.ts new file mode 100644 index 000000000..c8aed0bb4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/input-retry.ts @@ -0,0 +1,86 @@ +/** + * Retrying the *answer*, not the request. + * + * Distinct from `retry.ts`, which re-sends an identical request after a 429: + * there the request was right and the server was busy. Here the request was + * fine and the input was wrong, so nothing changes until the operator supplies + * something better. + * + * Every credential a migration takes — a connection string, a Firebase service + * account key, an Auth0 client secret — is long, pasted by hand, masked as it + * is typed, and wrong in ways nothing local can check: a typo'd host, an + * expired token, a key that was revoked, the right server but the wrong + * database. Only the remote end can say, and by then the operator has already + * answered every other question the command asked. Ending there charges them a + * full re-run for one line they could not see. + */ + +import { CliError, EXIT_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { log } from "../../../lib/log.ts"; +import { isAgent, isHuman } from "../../../mode.ts"; +import { isAssumeYes } from "./assume-yes.ts"; + +/** + * Runs `work`, and on failure asks for the input again and runs it once more. + * + * Keep `work` to the step that *proves* the input — the connection, the token + * exchange. Everything inside runs again on each attempt, so work that has + * already written a file, or a long fetch the credential has already been + * accepted for, does not belong in here. + * + * `-y`, `--json`, agent mode and a non-TTY get the failure unchanged: there is + * nobody to ask, and a loop that cannot prompt is a loop that cannot end. A cancelled + * prompt throws {@link UserAbortError}, which is not a `CliError` and so leaves + * the loop — declining the question is an answer. + * + * @param input - What to try first: a flag, an environment value, or the + * answer to the prompt the caller has already put up. + * @param reprompt - Asks for a replacement. Called once per failure. + * @param work - The step the input has to survive. + * @param options - The command's options; `--json` never prompts. + * @returns The result, and the input that produced it — which is not `input` + * when it took a retry, and later steps need the one that worked. + */ +export async function withInputRetry( + input: I, + reprompt: () => Promise, + work: (input: I) => Promise, + options: { json?: boolean } = {}, +): Promise<{ value: T; input: I }> { + let candidate = input; + + for (;;) { + try { + return { value: await work(candidate), input: candidate }; + } catch (error) { + // Everything these steps raise for a bad credential is a usage error + // carrying its own explanation. An outage, a 429 or a refused + // connection is not fixed by another credential, and anything else (an + // interrupt, a bug) is not ours to retry. + const badInput = error instanceof CliError && error.exitCode === EXIT_CODE.USAGE; + if (!badInput || options.json || !isHuman() || isAgent() || isAssumeYes()) throw error; + + log.error(error.message); + candidate = await reprompt(); + } + } +} + +/** Statuses that mean the credential is wrong, rather than the service being down. */ +const CREDENTIAL_STATUSES = new Set([400, 401, 403]); + +/** True when an API's status says the credential, not the service, is the problem. */ +export function isCredentialStatus(status: number): boolean { + return CREDENTIAL_STATUSES.has(status); +} + +/** + * Throws an API failure: a usage error (exit 2, and a fresh prompt under + * {@link withInputRetry}) when the status blames the credential; a plain + * error (exit 1) for a 429, a 5xx or anything else another credential would + * not fix. + */ +export function throwApiFailure(status: number, message: string, docsUrl?: string): never { + if (isCredentialStatus(status)) throwUsageError(message, docsUrl); + throw new CliError(message, { docsUrl }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/instance.test.ts b/packages/cli-core/src/commands/migrate/lib/instance.test.ts new file mode 100644 index 000000000..5eac24b94 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { + DEV_USER_LIMIT, + resolveDevUserLimit, + detectInstanceType, + getDefaultConcurrencyLimit, + getDefaultRateLimit, + getRetryDelay, + resolveLimits, +} from "./instance.ts"; + +describe("detectInstanceType", () => { + test.each([ + ["sk_live_abc123", "prod"], + ["sk_test_abc123", "dev"], + ["sk_something_else", "dev"], + ["nonsense", "dev"], + ])("%s -> %s", (key, expected) => { + expect(detectInstanceType(key)).toBe(expected as "dev" | "prod"); + }); +}); + +describe("default limits", () => { + test.each([ + ["prod", 100], + ["dev", 10], + ])("%s instances get %i req/s", (instanceType, expected) => { + expect(getDefaultRateLimit(instanceType as "dev" | "prod")).toBe(expected); + }); + + test.each([ + [100, 9], + [10, 1], + [1, 1], + ])("a %i req/s limit yields %i concurrent calls", (rateLimit, expected) => { + expect(getDefaultConcurrencyLimit(rateLimit)).toBe(expected); + }); + + test("development instances default to 100 users", () => { + expect(DEV_USER_LIMIT).toBe(100); + }); +}); + +describe("resolveDevUserLimit", () => { + test.each([ + [undefined, 100], + ["500", 500], + ["250.7", 250], + ["0", 100], + ["-5", 100], + ["lots", 100], + ])("CLERK_MIGRATE_DEV_USER_LIMIT=%p gives %i", (value, expected) => { + expect(resolveDevUserLimit({ CLERK_MIGRATE_DEV_USER_LIMIT: value })).toBe(expected); + }); +}); + +describe("resolveLimits", () => { + test("derives both limits from the key when nothing is overridden", () => { + expect(resolveLimits("sk_live_x", {})).toEqual({ + instanceType: "prod", + rateLimit: 100, + concurrencyLimit: 9, + }); + }); + + test("honours environment overrides", () => { + expect( + resolveLimits("sk_test_x", { + CLERK_MIGRATE_RATE_LIMIT: "50", + CLERK_MIGRATE_CONCURRENCY_LIMIT: "4", + }), + ).toEqual({ instanceType: "dev", rateLimit: 50, concurrencyLimit: 4 }); + }); + + test("derives concurrency from an overridden rate limit", () => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: "200" }).concurrencyLimit).toBe( + 19, + ); + }); + + test.each([["0"], ["-5"], ["fast"], [""]])( + "ignores the unusable override %p in favour of the default", + (value) => { + expect(resolveLimits("sk_test_x", { CLERK_MIGRATE_RATE_LIMIT: value }).rateLimit).toBe(10); + }, + ); +}); + +describe("getRetryDelay", () => { + test.each([ + [undefined, 10_000, 10_000, 10], + [15, 10_000, 15_000, 15], + [1, 10_000, 1000, 1], + ])("Retry-After %p -> %i ms", (retryAfter, fallback, delayMs, delaySeconds) => { + expect(getRetryDelay(retryAfter, fallback)).toEqual({ delayMs, delaySeconds }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/instance.ts b/packages/cli-core/src/commands/migrate/lib/instance.ts new file mode 100644 index 000000000..14064abcc --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/instance.ts @@ -0,0 +1,112 @@ +/** + * Instance-type detection and the throughput limits that follow from it. + * + * Ported from the standalone migration-tool's `src/envs-constants.ts`, minus + * its dotenv/Zod env bootstrap: the secret key arrives from + * `resolveBapiSecretKey`, and only the two override knobs read the environment. + */ + +/** + * The user limit a development instance is created with. + * + * Only a default: Clerk raises it per instance on request, and the real value + * (`max_allowed_users`) is not served by BAPI, DAPI or FAPI — only by Clerk's + * internal staff API. The checks reject users past it, so an instance Clerk has + * raised overrides it with `CLERK_MIGRATE_DEV_USER_LIMIT` (see + * {@link resolveDevUserLimit}). A production instance's limit comes from its + * plan; the import stops at the first `user_quota_exceeded` either way. + */ +export const DEV_USER_LIMIT = 100; + +/** A positive number from the environment, or undefined. */ +function positive(value: string | undefined): number | undefined { + if (!value) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +/** + * The development-instance user limit for this run: `CLERK_MIGRATE_DEV_USER_LIMIT` + * when it is a positive number, otherwise {@link DEV_USER_LIMIT}. + */ +export function resolveDevUserLimit(env: Record = process.env): number { + const override = positive(env.CLERK_MIGRATE_DEV_USER_LIMIT); + return override ? Math.floor(override) : DEV_USER_LIMIT; +} + +/** How many times a 429 is retried before the user is recorded as failed. */ +export const MAX_RETRIES = 5; + +/** Fallback backoff when a 429 response carries no `Retry-After`. */ +export const RETRY_DELAY_MS = 10_000; + +export type InstanceType = "dev" | "prod"; + +/** + * Derives the instance type from the secret key's prefix. + * + * @example detectInstanceType("sk_live_xxx") // "prod" + * @example detectInstanceType("sk_test_xxx") // "dev" + */ +export function detectInstanceType(secretKey: string): InstanceType { + return secretKey.split("_")[1] === "live" ? "prod" : "dev"; +} + +/** + * Clerk's documented `POST /v1/users` rate limits, as requests per second: + * 1000 per 10s for production, 100 per 10s for development. + */ +export function getDefaultRateLimit(instanceType: InstanceType): number { + return instanceType === "prod" ? 100 : 10; +} + +/** + * Concurrency that saturates ~95% of the rate limit, assuming ~100ms of API + * latency per call: N concurrent requests at 100ms each yield N * 10 req/s. + * + * Override with `CLERK_MIGRATE_CONCURRENCY_LIMIT` when actual latency differs. + */ +export function getDefaultConcurrencyLimit(rateLimit: number): number { + return Math.max(1, Math.floor(rateLimit * 0.095)); +} + +export type ResolvedLimits = { + instanceType: InstanceType; + rateLimit: number; + concurrencyLimit: number; +}; + +/** + * Resolves throughput limits for a run: defaults from the detected instance + * type, each overridable by an environment variable. + * + * Non-numeric or non-positive overrides are ignored in favour of the default + * rather than failing the run — an unusable limit would stall the import. + */ +export function resolveLimits( + secretKey: string, + env: Record = process.env, +): ResolvedLimits { + const instanceType = detectInstanceType(secretKey); + + const rateLimit = positive(env.CLERK_MIGRATE_RATE_LIMIT) ?? getDefaultRateLimit(instanceType); + const concurrencyLimit = + positive(env.CLERK_MIGRATE_CONCURRENCY_LIMIT) ?? getDefaultConcurrencyLimit(rateLimit); + + return { instanceType, rateLimit, concurrencyLimit }; +} + +/** + * Backoff for a 429, preferring the server's `Retry-After` over the default. + * + * @param retryAfterSeconds - `Retry-After` value from the response, if present. + * @param defaultDelayMs - Fallback delay in milliseconds. + */ +export function getRetryDelay( + retryAfterSeconds: number | undefined, + defaultDelayMs: number, +): { delayMs: number; delaySeconds: number } { + const delayMs = retryAfterSeconds ? retryAfterSeconds * 1000 : defaultDelayMs; + const delaySeconds = retryAfterSeconds || defaultDelayMs / 1000; + return { delayMs, delaySeconds }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts new file mode 100644 index 000000000..f48871bf8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport } from "./readiness.ts"; +import { buildChangePayload, buildSettingChanges } from "./modify-settings.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + const identifiers = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }; + return { + identifiers: { + // Hand-built rows have no user overlap, so "any" is the sum. + anyEmail: identifiers.verifiedEmails + identifiers.unverifiedEmails, + anyPhone: identifiers.verifiedPhones + identifiers.unverifiedPhones, + ...identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +/** + * Changes are built from a real report rather than hand-written rows, so a row + * whose `key` stops matching the path table fails here instead of silently + * dropping out of the offer. + */ +function changesFor(input: Parameters[0]) { + return buildSettingChanges(buildReadinessReport(input).blocking); +} + +describe("what gets offered", () => { + test("a required field not every user has is offered as a relaxation", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([ + { + id: "email_address", + label: "Make Email optional at sign-up", + section: "identifiers", + kind: "relax", + writes: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + ]); + }); + + test("a field the file carries but Clerk has switched off is offered as an enable", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { username: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + + expect(changes).toEqual([ + { + id: "username", + label: "Enable Username", + section: "identifiers", + kind: "enable", + writes: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + }, + ]); + }); + + /** + * Clerk refuses a verifiable attribute that is on with no way to verify it — + * `422 phone_number: verifiable attributes need to have at least one + * verification` — and switching the attribute off empties the strategies, so + * every enable that turned one off has to put one back. + */ + test.each([ + ["phone_number", "auth_phone", "phone_code"], + ["email_address", "auth_email", "email_code"], + ])("enabling %s also restores its verification strategy", (attribute, group, strategy) => { + const carriesIt = attribute === "phone_number" ? { verifiedPhones: 2 } : { verifiedEmails: 2 }; + + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { ...carriesIt, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { [attribute]: { enabled: false } } }), + }); + + expect(changes[0]?.writes).toEqual([ + { path: [group, "used_for_sign_up"], value: true }, + { path: [group, "verification_strategies"], value: [strategy] }, + ]); + expect(buildChangePayload(changes)).toEqual({ + [group]: { used_for_sign_up: true, verification_strategies: [strategy] }, + }); + }); + + // Not verifiable, so no strategy to restore — one write is the whole change. + test("enabling username takes a single write", () => { + const changes = changesFor({ + analysis: analysis({ totalUsers: 2, identifiers: { username: 2 } as never }), + settings: settings({ attributes: { username: { enabled: false } } }), + }); + expect(changes[0]?.writes).toHaveLength(1); + }); + + test("a disabled social provider is offered under Clerk's own strategy name", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_x: { enabled: false } }, + }), + // Supabase calls it `twitter`; the config document calls it `oauth_x`. + providerCounts: { twitter: 2 }, + }); + + expect(changes).toEqual([ + { + id: "twitter", + label: "Enable Twitter (X) sign-in", + section: "social", + kind: "enable", + writes: [{ path: ["connection_oauth_x", "enabled"], value: true }], + }, + ]); + }); + + // "Could not read" is not "switched off", so nothing is flagged and nothing + // is offered — the report already degrades to a coverage-only listing. + test("nothing is offered when the instance settings could not be read", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 1, hasAnyIdentifier: 2 } as never, + }), + settings: null, + }); + + expect(changes).toEqual([]); + }); + + test("nothing is offered when every field is already configured", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 2 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + + expect(changes).toEqual([]); + }); +}); + +describe("the payload", () => { + test("collapses changes that share a parent into one branch", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + fieldCounts: { firstName: 2, lastName: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + first_name: { enabled: true, required: true }, + last_name: { enabled: true, required: true }, + }, + }), + }); + + expect(buildChangePayload(changes)).toEqual({ + user_model: { first_name: { required: false }, last_name: { required: false } }, + }); + }); + + test("carries only the changes it is given", () => { + const changes = changesFor({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 2, hasAnyIdentifier: 3 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(changes.map((change) => change.id)).toEqual(["email_address", "password"]); + + expect(buildChangePayload(changes.filter((change) => change.id === "password"))).toEqual({ + auth_password: { required: false }, + }); + }); + + test("is empty when nothing was selected", () => { + expect(buildChangePayload([])).toEqual({}); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/modify-settings.ts b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts new file mode 100644 index 000000000..6ed1cc506 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/modify-settings.ts @@ -0,0 +1,146 @@ +/** + * Turning a flagged readiness row into the instance-config change that would + * stop it being flagged, printed as a `clerk config patch` command. + * + * The checks already know which settings will cost users; without this the + * only way to act on them is to leave the CLI, find the setting in the + * dashboard, and come back. Each change is a single leaf in the Platform API's + * config document. + * + * These are offers, not corrections. A flagged setting is not a wrong setting — + * an instance that genuinely requires an email address is configured exactly as + * its owner intended, and the right answer may well be to fix the export + * instead. Nothing here is preselected and nothing is applied unasked. + * + * Only the two verdicts `buildReadinessReport` produces are mapped: "required + * in Clerk" (relax the requirement) and "not enabled in Clerk" (turn it on). A + * row this file has no path for is simply not offered — the report still names + * it and still points at the dashboard. + */ + +import { clerkOffersProvider, toClerkStrategy } from "./clerk-config.ts"; +import type { ReadinessItem, ReadinessSection } from "./readiness.ts"; + +/** One leaf of the config document, and what to set it to. */ +export type SettingWrite = { path: string[]; value: boolean | string[] }; + +/** One offered change: what it says, and the config leaves it writes. */ +export type SettingChange = { + /** Stable identity for the multiselect, and for tests. */ + id: string; + label: string; + section: ReadinessSection; + /** `enable` turns something on; `relax` drops a requirement. */ + kind: "enable" | "relax"; + writes: SettingWrite[]; +}; + +type ChangeWrites = { enable: SettingWrite[]; relax: SettingWrite[] }; + +/** + * Where each attribute lives in the config document. `used_for_sign_up` is the + * enable field that matters here: `POST /v1/users` validates an import against + * the instance's sign-up requirements, not its sign-in strategies. + * + * Email and phone are **verifiable** attributes, so enabling one takes two + * writes rather than one. Clerk rejects a verifiable attribute that is on with + * no way to verify it — `422 phone_number: verifiable attributes need to have + * at least one verification` — and switching the attribute off empties + * `verification_strategies`, so whatever turns it back on has to put a strategy + * back. Username, password and the name fields are not verifiable and take one + * write each. + */ +const ATTRIBUTE_WRITES: Record = { + email_address: { + enable: [ + { path: ["auth_email", "used_for_sign_up"], value: true }, + { path: ["auth_email", "verification_strategies"], value: ["email_code"] }, + ], + relax: [{ path: ["auth_email", "required_for_sign_up"], value: false }], + }, + phone_number: { + enable: [ + { path: ["auth_phone", "used_for_sign_up"], value: true }, + { path: ["auth_phone", "verification_strategies"], value: ["phone_code"] }, + ], + relax: [{ path: ["auth_phone", "required_for_sign_up"], value: false }], + }, + username: { + enable: [{ path: ["auth_username", "used_for_sign_up"], value: true }], + relax: [{ path: ["auth_username", "required_for_sign_up"], value: false }], + }, + password: { + enable: [{ path: ["auth_password", "enabled"], value: true }], + relax: [{ path: ["auth_password", "required"], value: false }], + }, + first_name: { + enable: [{ path: ["user_model", "first_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "first_name", "required"], value: false }], + }, + last_name: { + enable: [{ path: ["user_model", "last_name", "enabled"], value: true }], + relax: [{ path: ["user_model", "last_name", "required"], value: false }], + }, +}; + +/** `github` → `connection_oauth_github`, via Clerk's own strategy name. */ +function socialPath(provider: string): string[] { + return [`connection_oauth_${toClerkStrategy(provider).replace(/^oauth_/, "")}`, "enabled"]; +} + +function changeFor(item: ReadinessItem): SettingChange | undefined { + // Required-but-not-universal is the only verdict that relaxes rather than + // enables; every other flagged row is something switched off in Clerk. + const relax = item.clerkRequired === true; + + if (item.section === "social") { + // A provider has no "required" in Clerk, so there is nothing to relax, and + // one Clerk doesn't offer has nothing to turn on. + if (relax || !clerkOffersProvider(item.key)) return undefined; + return { + id: item.key, + label: `Enable ${item.label} sign-in`, + section: item.section, + kind: "enable", + writes: [{ path: socialPath(item.key), value: true }], + }; + } + + const writes = ATTRIBUTE_WRITES[item.key]; + if (!writes) return undefined; + + return { + id: item.key, + label: relax ? `Make ${item.label} optional at sign-up` : `Enable ${item.label}`, + section: item.section, + kind: relax ? "relax" : "enable", + writes: relax ? writes.relax : writes.enable, + }; +} + +/** The changes offerable for a report's flagged rows, in report order. */ +export function buildSettingChanges(flagged: ReadinessItem[]): SettingChange[] { + return flagged.map(changeFor).filter((change): change is SettingChange => change !== undefined); +} + +/** + * Collapses the chosen changes into one config payload. + * + * Changes share parents — `first_name` and `last_name` both write `user_model` + * — so leaves are written into a shared tree rather than merged after the fact. + */ +export function buildChangePayload( + changes: Pick[], +): Record { + const payload: Record = {}; + + for (const write of changes.flatMap((change) => change.writes)) { + let node = payload; + for (const key of write.path.slice(0, -1)) { + node = (node[key] ??= {}) as Record; + } + node[write.path[write.path.length - 1] as string] = write.value; + } + + return payload; +} diff --git a/packages/cli-core/src/commands/migrate/lib/progress.test.ts b/packages/cli-core/src/commands/migrate/lib/progress.test.ts new file mode 100644 index 000000000..182fb2899 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/progress.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { formatProgress, formatRemaining, withProgress } from "./progress.ts"; + +const captured = useCaptureLog(); + +describe("formatRemaining", () => { + test.each([ + [9_000, "~9s"], + [90_000, "~1m 30s"], + [900_000, "~15m"], + [500_000, "~8m 20s"], + [7_500_000, "~2h 5m"], + ])("%p ms -> %p", (ms, expected) => { + expect(formatRemaining(ms)).toBe(expected); + }); +}); + +describe("formatProgress", () => { + // 10,000 users at ~100 a second, 1% failing: the agreed mockup. + test("draws an 80-column bar with the report under it", () => { + const [bar, report] = formatProgress({ + total: 10_000, + verb: "created", + counts: { done: 7_500, ok: 7_425, failed: 75 }, + elapsedMs: 75_000, + }); + + expect(bar).toBe(`│ ${"█".repeat(54)}${"░".repeat(18)} 75%`); + expect([...bar]).toHaveLength(80); + expect(report).toBe("│ 7,500/10,000 users · ✓ 7,425 created · ✗ 75 failed · ~25s left"); + }); + + test("shrinks the bar to a narrower terminal", () => { + const [bar] = formatProgress({ + total: 100, + verb: "created", + counts: { done: 50, ok: 50, failed: 0 }, + elapsedMs: 0, + columns: 60, + }); + expect([...bar]).toHaveLength(60); + }); + + test.each([ + ["before there is a rate to go on", 1_000, 500], + ["once every user is done", 10_000, 1_000], + ])("gives no estimate %s", (_label, elapsedMs, done) => { + const [, report] = formatProgress({ + total: 1_000, + verb: "deleted", + counts: { done, ok: done, failed: 0 }, + elapsedMs, + }); + expect(report).not.toContain("left"); + }); + + // Floored, so 999 of 1,000 never reads as finished. + test("reads 100% only when every user is done", () => { + const [bar] = formatProgress({ + total: 1_000, + verb: "created", + counts: { done: 999, ok: 999, failed: 0 }, + elapsedMs: 0, + }); + expect(bar).toEndWith(" 99%"); + expect(bar).toContain("░"); + }); +}); + +describe("withProgress", () => { + let mode: Mode; + let isTTY: boolean | undefined; + + beforeEach(() => { + mode = getMode(); + isTTY = process.stderr.isTTY; + }); + + afterEach(() => { + setMode(mode); + process.stderr.isTTY = isTTY as boolean; + }); + + test("prints nothing for an agent", async () => { + setMode("agent"); + await withProgress({ total: 10, verb: "created" }, async (update) => { + update({ done: 10, ok: 10, failed: 0 }); + }); + expect(captured.err).toBe(""); + }); + + // A log file gets a line per 10%, not a redraw per user. + test("without a terminal, prints the report at each tenth", async () => { + setMode("human"); + process.stderr.isTTY = false; + + await withProgress({ total: 100, verb: "created" }, async (update) => { + for (let done = 1; done <= 100; done++) update({ done, ok: done, failed: 0 }); + }); + + const lines = captured.err.split("\n").filter((line) => line.includes("users")); + // 0% at the start, each tenth from 10% to 100%, and the final state. + expect(lines).toHaveLength(12); + expect(lines.at(-1)).toContain("100/100 users"); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/progress.ts b/packages/cli-core/src/commands/migrate/lib/progress.ts new file mode 100644 index 000000000..19ed123f0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/progress.ts @@ -0,0 +1,131 @@ +/** + * The progress display for the import and undo loops: a bar on one line, the + * counts on the next. + * + * Used instead of a spinner on purpose. A clack spinner puts stdin in raw mode + * and calls `process.exit(0)` on Ctrl-C, so an interrupted import looked like + * a finished one and `clerk migrate import … && rm -rf ` carried on. + * This leaves stdin alone: Ctrl-C reaches the CLI's handler, in-flight + * requests abort, and the process exits 130 (`.claude/rules/interrupts.md`). + */ + +import { dim } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; + +export type ProgressCounts = { done: number; ok: number; failed: number }; + +/** Receives the counts as each user finishes. */ +export type ProgressUpdate = (counts: ProgressCounts) => void; + +type ProgressLine = { + total: number; + /** What happened to the users counted in `ok`: "created", "deleted". */ + verb: string; + counts: ProgressCounts; + elapsedMs: number; + /** Terminal columns; the bar fills the line up to 80. */ + columns?: number; +}; + +const MAX_WIDTH = 80; +/** No estimate until there is this much of a rate to go on. */ +const ESTIMATE_AFTER_MS = 2000; +/** At most one redraw per this many ms: an import can report 100 users a second. */ +const REDRAW_MS = 100; + +const GUTTER = "│ "; +const number = (value: number) => value.toLocaleString("en-US"); + +/** `~9s`, `~1m 30s`, `~15m`, `~2h 5m`. */ +export function formatRemaining(ms: number): string { + const seconds = Math.max(1, Math.round(ms / 1000)); + if (seconds < 60) return `~${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) { + const rest = seconds % 60; + return rest ? `~${minutes}m ${rest}s` : `~${minutes}m`; + } + const hours = Math.floor(minutes / 60); + const rest = minutes % 60; + return rest ? `~${hours}h ${rest}m` : `~${hours}h`; +} + +/** The bar line and the report line, without color. */ +export function formatProgress(line: ProgressLine): [string, string] { + const { total, verb, counts, elapsedMs } = line; + const width = Math.min(MAX_WIDTH, line.columns ?? MAX_WIDTH); + const fraction = total > 0 ? Math.min(1, counts.done / total) : 1; + + // Floored, so the bar reads full only when every user is done. + const percent = `${Math.floor(fraction * 100)}%`.padStart(4); + const cells = Math.max(10, width - GUTTER.length - 1 - percent.length); + const filled = Math.floor(cells * fraction); + const bar = `${GUTTER}${"█".repeat(filled)}${"░".repeat(cells - filled)} ${percent}`; + + const parts = [ + `${number(counts.done)}/${number(total)} users`, + `✓ ${number(counts.ok)} ${verb}`, + `✗ ${number(counts.failed)} failed`, + ]; + if (elapsedMs >= ESTIMATE_AFTER_MS && counts.done > 0 && counts.done < total) { + parts.push(`${formatRemaining(((total - counts.done) * elapsedMs) / counts.done)} left`); + } + return [bar, `${GUTTER}${parts.join(" · ")}`]; +} + +/** + * Runs `fn` with a progress display for `total` users. + * + * At a terminal, the two lines redraw in place. When stderr is not a terminal + * (a log file, CI), the report line is printed at each 10% instead. An agent + * gets nothing, as it got nothing from the spinner. + */ +export async function withProgress( + options: { total: number; verb: string }, + fn: (update: ProgressUpdate) => Promise, +): Promise { + if (!isHuman()) return fn(() => {}); + + const tty = Boolean(process.stderr.isTTY); + const started = Date.now(); + let counts: ProgressCounts = { done: 0, ok: 0, failed: 0 }; + let drawn = false; + let lastDraw = 0; + let lastTenth = -1; + + const draw = (force: boolean) => { + const now = Date.now(); + const [bar, report] = formatProgress({ + ...options, + counts, + elapsedMs: now - started, + ...(process.stderr.columns ? { columns: process.stderr.columns } : {}), + }); + + if (!tty) { + const tenth = Math.floor((counts.done / Math.max(1, options.total)) * 10); + if (!force && tenth === lastTenth) return; + lastTenth = tenth; + log.ui(`${report}\n`); + return; + } + + if (!force && now - lastDraw < REDRAW_MS) return; + lastDraw = now; + // Up over the two lines drawn last time, then clear and rewrite each. + const up = drawn ? "\x1b[2A" : ""; + // The gutter in the gray clack draws it in. + const gutter = (line: string) => `${dim("│")}${line.slice(1)}`; + log.ui(`${up}\r\x1b[2K${gutter(bar)}\n\x1b[2K${gutter(report)}\n`); + drawn = true; + }; + + draw(true); + const result = await fn((next) => { + counts = next; + draw(false); + }); + draw(true); + return result; +} diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.test.ts b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts new file mode 100644 index 000000000..76d9a4eb4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, test } from "bun:test"; +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +import { analyzeFields, type FieldAnalysis } from "./analysis.ts"; +import { buildReadinessReport, type ReadinessItem } from "./readiness.ts"; + +/** Instance settings carrying only the attributes and providers a test names. */ +function settings(config: { + attributes?: Record; + social?: Record; +}): UserSettingsJSON { + return { + attributes: Object.fromEntries( + Object.entries(config.attributes ?? {}).map(([name, value]) => [ + name, + { enabled: value.enabled, required: value.required ?? false }, + ]), + ), + social: config.social ?? {}, + } as unknown as UserSettingsJSON; +} + +/** Field analysis with everything absent unless the test says otherwise. */ +function analysis(overrides: Partial & { totalUsers: number }): FieldAnalysis { + const identifiers = { + verifiedEmails: 0, + unverifiedEmails: 0, + verifiedPhones: 0, + unverifiedPhones: 0, + username: 0, + hasAnyIdentifier: overrides.totalUsers, + ...overrides.identifiers, + }; + return { + identifiers: { + // Hand-built rows have no user overlap, so "any" is the sum. + anyEmail: identifiers.verifiedEmails + identifiers.unverifiedEmails, + anyPhone: identifiers.verifiedPhones + identifiers.unverifiedPhones, + ...identifiers, + }, + fieldCounts: overrides.fieldCounts ?? {}, + totalUsers: overrides.totalUsers, + }; +} + +const item = (report: { items: ReadinessItem[] }, label: string) => + report.items.find((entry) => entry.label === label); + +describe("which rows appear", () => { + test("reports only the fields the file actually carries", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 3, identifiers: { verifiedEmails: 3 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(report.items.map((entry) => entry.label)).toEqual(["Email"]); + }); + + test("counts verified and unverified identifiers together", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 5, + identifiers: { verifiedEmails: 3, unverifiedEmails: 2, hasAnyIdentifier: 5 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + }); + expect(item(report, "Email")?.userCount).toBe(5); + }); + + // A user with both a verified and an unverified phone is one user, not two. + test("counts a user with both kinds of phone once", () => { + const users = [ + { userId: "a", phone: "+15555550100", unverifiedPhoneNumbers: ["+15555550101"] }, + { userId: "b", phone: "+15555550102", unverifiedPhoneNumbers: ["+15555550103"] }, + { userId: "c", email: "c@x.dev" }, + ]; + const report = buildReadinessReport({ + analysis: analyzeFields(users), + settings: settings({ attributes: { phone_number: { enabled: true } } }), + }); + expect(item(report, "Phone")?.userCount).toBe(2); + }); + + test("groups rows into identifiers, auth and user model", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 2, + identifiers: { verifiedEmails: 2, username: 2, hasAnyIdentifier: 2 } as never, + fieldCounts: { password: 2, firstName: 2, lastName: 1 }, + }), + settings: settings({}), + }); + expect(report.items.map((entry) => [entry.label, entry.section])).toEqual([ + ["Email", "identifiers"], + ["Username", "identifiers"], + ["Password", "auth"], + ["First name", "model"], + ["Last name", "model"], + ]); + }); +}); + +describe("required in Clerk but missing from the file", () => { + // The expensive case: those users fail one at a time, mid-import, after + // earlier users have already been created. + test("flags an attribute Clerk requires that not every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 10, username: 10 } as never, + }), + settings: settings({ + attributes: { + email_address: { enabled: true, required: true }, + username: { enabled: true }, + }, + }), + }); + + const email = item(report, "Email"); + expect(email?.blocking).toBe(true); + expect(email?.detail).toContain("required in Clerk"); + // A required identifier is the one verdict Clerk refuses the user over. + expect(email?.consequence).toBe("rejects"); + expect(report.blocking).toHaveLength(1); + }); + + test("does not flag a required attribute every user has", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ attributes: { email_address: { enabled: true, required: true } } }), + }); + expect(report.blocking).toHaveLength(0); + }); + + test("does not flag an enabled-but-optional attribute that some users lack", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 10, hasAnyIdentifier: 10 } as never, + fieldCounts: { firstName: 2 }, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, first_name: { enabled: true } }, + }), + }); + expect(report.blocking).toHaveLength(0); + }); + + // The import sends `skip_password_requirement`, so a required password costs + // the user their password rather than their whole account. + test("a required password drops rather than rejects", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + fieldCounts: { password: 1 }, + }), + settings: settings({ + attributes: { + email_address: { enabled: true }, + password: { enabled: true, required: true }, + }, + }), + }); + expect(item(report, "Password")?.consequence).toBe("drops"); + }); +}); + +describe("present in the file but disabled in Clerk", () => { + test("flags an attribute the instance has switched off", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, username: 3, hasAnyIdentifier: 3 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true }, username: { enabled: false } }, + }), + }); + + const username = item(report, "Username"); + expect(username?.blocking).toBe(true); + expect(username?.detail).toBe("not enabled in Clerk"); + }); + + test("flags a social provider users signed up with that Clerk lacks", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 4, + identifiers: { verifiedEmails: 4, hasAnyIdentifier: 4 } as never, + }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + }), + providerCounts: { google: 3, discord: 1 }, + }); + + expect(item(report, "Google")?.blocking).toBe(false); + expect(item(report, "Discord")?.blocking).toBe(true); + expect(report.blocking.map((entry) => entry.label)).toEqual(["Discord"]); + }); + + test("maps a provider whose Clerk strategy name differs", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ + attributes: { email_address: { enabled: true } }, + social: { oauth_microsoft: { enabled: true } }, + }), + providerCounts: { azure: 1 }, + }); + expect(item(report, "Microsoft (Azure)")?.blocking).toBe(false); + }); + + test("ignores a provider no user actually signed up with", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 1, identifiers: { verifiedEmails: 1 } as never }), + settings: settings({ attributes: { email_address: { enabled: true } } }), + providerCounts: { discord: 0 }, + }); + expect(item(report, "Discord")).toBeUndefined(); + }); +}); + +describe("when the instance settings cannot be read", () => { + const unreadable = () => + buildReadinessReport({ + analysis: analysis({ + totalUsers: 3, + identifiers: { verifiedEmails: 3, hasAnyIdentifier: 3 } as never, + }), + settings: null, + }); + + test("marks the report as degraded rather than failing", () => { + expect(unreadable().settingsUnavailable).toBe(true); + }); + + // `null` means "not read", which must not be confused with `false` + // ("read, and it is off") — the latter blocks, the former cannot. + test("claims nothing about Clerk, so nothing blocks", () => { + const report = unreadable(); + expect(item(report, "Email")?.clerkEnabled).toBeNull(); + expect(report.blocking).toHaveLength(0); + }); + + test("still reports what the file contains", () => { + expect(unreadable().items.map((entry) => entry.label)).toEqual(["Email"]); + }); +}); + +describe("file-level totals", () => { + test("counts users with no identifier at all", () => { + const report = buildReadinessReport({ + analysis: analysis({ + totalUsers: 10, + identifiers: { verifiedEmails: 7, hasAnyIdentifier: 7 } as never, + }), + settings: settings({}), + }); + expect(report.withoutIdentifier).toBe(3); + }); + + test("carries the validation failure count through", () => { + const report = buildReadinessReport({ + analysis: analysis({ totalUsers: 2 }), + settings: settings({}), + validationFailed: 5, + }); + expect(report.validationFailed).toBe(5); + }); +}); + +/** + * The counts an operator actually decides on. Built from the users themselves, + * because per-field coverage cannot answer them: the users missing an email and + * the users missing a password overlap by an amount only a per-user pass knows. + */ diff --git a/packages/cli-core/src/commands/migrate/lib/readiness.ts b/packages/cli-core/src/commands/migrate/lib/readiness.ts new file mode 100644 index 000000000..22b54aff7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/readiness.ts @@ -0,0 +1,238 @@ +/** + * What the import file contains, cross-referenced against what the + * destination instance accepts, one row per field. + * + * Ported from the standalone migration-tool's `displayCrossReference`. The + * import's checks read the flagged rows for their drop warnings and their + * `clerk config patch` fixes; which users are rejected is decided per user in + * `checks.ts`. + */ + +import type { UserSettingsJSON } from "../../../lib/fapi.ts"; +// Pure attribute lookups, shared with the `users` create wizard. +import { isEnabled, isRequired, type AttributeName } from "../../users/interactive/attributes.ts"; +import type { FieldAnalysis } from "./analysis.ts"; +import { clerkOffersProvider, providerLabel, toClerkStrategy } from "./clerk-config.ts"; + +export type ReadinessSection = "identifiers" | "auth" | "social" | "model"; + +/** + * One row of the report. + * + * @property clerkEnabled - `null` when the instance settings could not be read, + * which is different from `false` ("read, and it is off"). + * @property blocking - This row will cost users unless the operator acts. + */ +export type ReadinessItem = { + label: string; + /** + * What the row is about, machine-side: an {@link AttributeName} for every + * section but `social`, and the source platform's provider key for that one. + * `label` is for humans; this is what `modify-settings.ts` looks up. + */ + key: string; + section: ReadinessSection; + /** Users in the file that carry this field or provider. */ + userCount: number; + clerkEnabled: boolean | null; + clerkRequired: boolean | null; + blocking: boolean; + /** + * What this row costs the users it affects. + * + * - `rejects` — Clerk refuses the user outright. Only a required identifier + * does this: `POST /v1/users` enforces the instance's sign-up identifier + * requirements, and a user carrying none of them has nothing to be created + * under. + * - `drops` — the user is created, but this piece of them is not. A required + * password is in this group rather than `rejects` because the import sends + * `skip_password_requirement` (see `import-users.ts`), so the user lands + * without one and signs in another way. With no other way, the checks + * reject the user (`passwordIsOnlySignIn`). + * - `stored` — the user is created with it, but the instance does not use + * it until the setting is turned on. Clerk stores these whatever the + * setting (`create_service.go` validates them but never checks it). + */ + consequence?: "rejects" | "drops" | "stored"; + /** Why it blocks — omitted when it does not. */ + detail?: string; +}; + +export type ReadinessReport = { + totalUsers: number; + /** Users with no identifier at all; they cannot be imported under any settings. */ + withoutIdentifier: number; + validationFailed: number; + items: ReadinessItem[]; + /** Every item flagged `blocking`, in report order. */ + blocking: ReadinessItem[]; + /** True when the instance settings could not be read. */ + settingsUnavailable: boolean; +}; + +type BuildInput = { + analysis: FieldAnalysis; + /** `null` when no publishable key was available, or FAPI could not be read. */ + settings: UserSettingsJSON | null; + validationFailed?: number; + /** Source-platform provider key → user count. Supabase exports only. */ + providerCounts?: Record; +}; + +/** + * Identifiers Clerk creates a user *under*. A required one that a user does not + * carry leaves nothing to create them with, so the API refuses them — which is + * why these are the only attributes whose consequence is `rejects`. + */ +const REJECTING_ATTRIBUTES = new Set([ + "email_address", + "phone_number", + "username", + "first_name", + "last_name", +]); + +/** Fields Clerk stores with their setting off, where it works once turned on. */ +const STORED_WHEN_OFF = new Set(["password", "username", "first_name", "last_name"]); + +/** + * True when Clerk keeps an email or phone on create: its setting is on, or + * the instance signs in or does MFA with it (`IsEnabledOrFactor` in clerk_go). + * Sign-up off alone does not refuse it. + */ +export function acceptsIdentifier( + settings: UserSettingsJSON, + attribute: "email_address" | "phone_number", +): boolean { + const data = settings.attributes?.[attribute]; + return Boolean(data?.enabled || data?.used_for_first_factor || data?.used_for_second_factor); +} + +/** An identifier or user-model row, with its blocking verdict. */ +function buildAttributeItem( + label: string, + section: ReadinessSection, + attribute: AttributeName, + userCount: number, + settings: UserSettingsJSON | null, + totalUsers: number, +): ReadinessItem { + const enabled = settings ? isEnabled(settings, attribute) : null; + const required = settings ? isRequired(settings, attribute) : null; + const accepted = + settings && (attribute === "email_address" || attribute === "phone_number") + ? acceptsIdentifier(settings, attribute) + : enabled; + const missing = totalUsers - userCount; + + // Required but not universal is the expensive case: those users fail one by + // one, mid-import, after earlier users have already been created. + if (required === true && missing > 0) { + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + consequence: REJECTING_ATTRIBUTES.has(attribute) ? "rejects" : "drops", + // How many users this costs is the outcome block's job. Restating it here + // reads as a contradiction, because that block counts each user once and + // this row counts the field — a user missing both an email and a password + // appears in both rows but only in the first outcome. + detail: "required in Clerk, and not every user has one", + }; + } + + // Present in the file but switched off in Clerk: dropped, or stored unused. + if (accepted === false && userCount > 0) { + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: true, + consequence: STORED_WHEN_OFF.has(attribute) ? "stored" : "drops", + detail: "not enabled in Clerk", + }; + } + + return { + label, + key: attribute, + section, + userCount, + clerkEnabled: enabled, + clerkRequired: required, + blocking: false, + }; +} + +/** + * Cross-references the file against the instance. + * + * A field absent from the file contributes no row — the report describes what + * is actually being imported, not every setting Clerk supports. + */ +export function buildReadinessReport(input: BuildInput): ReadinessReport { + const { analysis, settings, validationFailed = 0, providerCounts = {} } = input; + const total = analysis.totalUsers; + const items: ReadinessItem[] = []; + + // Users, not fields: one with both a verified and an unverified phone is one. + const emailCount = analysis.identifiers.anyEmail; + const phoneCount = analysis.identifiers.anyPhone; + + const attributeRows: [string, ReadinessSection, AttributeName, number][] = [ + ["Email", "identifiers", "email_address", emailCount], + ["Phone", "identifiers", "phone_number", phoneCount], + ["Username", "identifiers", "username", analysis.identifiers.username], + ["Password", "auth", "password", analysis.fieldCounts.password ?? 0], + ["First name", "model", "first_name", analysis.fieldCounts.firstName ?? 0], + ["Last name", "model", "last_name", analysis.fieldCounts.lastName ?? 0], + ]; + + for (const [label, section, attribute, count] of attributeRows) { + // A required field nobody has still costs every user, so it gets a row. + if (count > 0 || (settings && isRequired(settings, attribute))) { + items.push(buildAttributeItem(label, section, attribute, count, settings, total)); + } + } + + for (const [provider, count] of Object.entries(providerCounts)) { + if (count === 0) continue; + const enabled = settings + ? (settings.social?.[toClerkStrategy(provider) as keyof typeof settings.social]?.enabled ?? + false) + : null; + items.push({ + label: providerLabel(provider), + key: provider, + section: "social", + userCount: count, + clerkEnabled: enabled, + clerkRequired: null, + blocking: enabled === false, + ...(enabled === false + ? { + consequence: "drops" as const, + detail: clerkOffersProvider(provider) ? "not enabled in Clerk" : "not offered by Clerk", + } + : {}), + }); + } + + const blocking = items.filter((item) => item.blocking); + + return { + totalUsers: total, + withoutIdentifier: total - analysis.identifiers.hasAnyIdentifier, + validationFailed, + items, + blocking, + settingsUnavailable: settings === null, + }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/retry.test.ts b/packages/cli-core/src/commands/migrate/lib/retry.test.ts new file mode 100644 index 000000000..0623c9709 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import { BapiError, CliError, FapiError } from "../../../lib/errors.ts"; +import { RateLimitExceededError, readRetryAfter, retryOn429 } from "./retry.ts"; + +const rateLimited = (headers: Record = {}) => + new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow" }] }), + new Headers(headers), + ); + +const failed = (status: number) => + new BapiError( + status, + JSON.stringify({ errors: [{ code: "e", message: "nope" }] }), + new Headers(), + ); + +describe("readRetryAfter", () => { + test.each([ + ["12", 12], + ["0", undefined], + ["-1", undefined], + ["soon", undefined], + ])("Retry-After: %s -> %p", (header, expected) => { + expect(readRetryAfter(rateLimited({ "retry-after": header }))).toBe( + expected as number | undefined, + ); + }); + + test("falls back to the error body's retryAfter meta", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers(), + ); + expect(readRetryAfter(error)).toBe(7); + }); + + test("prefers the header over the body", () => { + const error = new BapiError( + 429, + JSON.stringify({ errors: [{ code: "e", message: "slow", meta: { retryAfter: 7 } }] }), + new Headers({ "retry-after": "3" }), + ); + expect(readRetryAfter(error)).toBe(3); + }); + + test("returns undefined when neither carries a value", () => { + expect(readRetryAfter(rateLimited())).toBeUndefined(); + }); +}); + +describe("retryOn429", () => { + test("returns the value when the call succeeds first time", async () => { + expect(await retryOn429(async () => "ok")).toBe("ok"); + }); + + test("retries after a 429 and returns the eventual value", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + + // The settings read goes through FAPI, which is rate limited the same way. + test("retries a FAPI 429 as well", async () => { + let attempts = 0; + const result = await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw new FapiError(429, JSON.stringify({ errors: [] })); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + + expect(result).toBe("ok"); + expect(attempts).toBe(2); + }); + + test("waits the interval the server asked for", async () => { + let attempts = 0; + const started = performance.now(); + + await retryOn429(async () => { + attempts++; + if (attempts === 1) throw rateLimited({ "retry-after": "1" }); + return "ok"; + }); + + expect(performance.now() - started).toBeGreaterThanOrEqual(900); + }); + + test("falls back to the default delay when no Retry-After is given", async () => { + let attempts = 0; + await retryOn429( + async () => { + attempts++; + if (attempts === 1) throw rateLimited(); + return "ok"; + }, + { defaultDelayMs: 5 }, + ); + expect(attempts).toBe(2); + }); + + test("reports each backoff to the caller so it can log against its own run", async () => { + const seen: { attempt: number; delaySeconds: number }[] = []; + let attempts = 0; + + await retryOn429( + async () => { + attempts++; + if (attempts <= 2) throw rateLimited(); + return "ok"; + }, + { + defaultDelayMs: 5, + onRetry: ({ attempt, delaySeconds }) => seen.push({ attempt, delaySeconds }), + }, + ); + + expect(seen.map((entry) => entry.attempt)).toEqual([1, 2]); + expect(seen[0]?.delaySeconds).toBe(0.005); + }); + + test("gives up after the ceiling, distinctly from an ordinary failure", async () => { + let attempts = 0; + + await expect( + retryOn429( + async () => { + attempts++; + throw rateLimited(); + }, + { maxRetries: 2, defaultDelayMs: 5 }, + ), + ).rejects.toThrow(RateLimitExceededError); + + // One initial attempt plus maxRetries retries. + expect(attempts).toBe(3); + }); + + // Only rate limiting is transient; retrying a 422 would just repeat it. + test.each([[400], [401], [404], [422], [500]])("lets a %i through untouched", async (status) => { + let attempts = 0; + + await expect( + retryOn429(async () => { + attempts++; + throw failed(status); + }), + ).rejects.toThrow(BapiError); + + expect(attempts).toBe(1); + }); + + test("lets a non-API error through untouched", async () => { + await expect(retryOn429(async () => Promise.reject(new CliError("boom")))).rejects.toThrow( + CliError, + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/retry.ts b/packages/cli-core/src/commands/migrate/lib/retry.ts new file mode 100644 index 000000000..64c4d90e1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/retry.ts @@ -0,0 +1,66 @@ +/** + * Rate-limit backoff, shared by `migrate import` and `migrate undo`. + * + * Both walk the whole user set through BAPI and hit the same limits, so they + * back off identically rather than approximately: extracting this is what + * makes "deletion retries the same as import" true by construction. + */ + +import { ApiError } from "../../../lib/errors.ts"; +import { MAX_RETRIES, RETRY_DELAY_MS, getRetryDelay } from "./instance.ts"; + +/** Seconds to wait per a 429's `Retry-After` header or error meta, if given. */ +export function readRetryAfter(error: ApiError): number | undefined { + const header = error.headers?.get("retry-after"); + if (header) { + const parsed = Number(header); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + const meta = error.meta?.retryAfter; + return typeof meta === "number" && meta > 0 ? meta : undefined; +} + +/** Raised once a 429 has been retried {@link MAX_RETRIES} times. */ +export class RateLimitExceededError extends Error { + constructor(public readonly attempts: number) { + super(`Rate limit exceeded after ${attempts} retries`); + this.name = "RateLimitExceededError"; + } +} + +export type RetryOptions = { + /** Called before each backoff, so the caller can log it against its own run. */ + onRetry?: (info: { attempt: number; delaySeconds: number; message: string }) => void; + maxRetries?: number; + /** Backoff when the response carries no `Retry-After`. */ + defaultDelayMs?: number; +}; + +/** + * Runs `fn`, backing off and retrying whenever BAPI or FAPI answers 429. + * + * Anything other than a 429 propagates untouched — only rate limiting is + * transient. Exhausting the retries raises {@link RateLimitExceededError} so + * the caller can record it distinctly from an ordinary API failure. + */ +export async function retryOn429(fn: () => Promise, options: RetryOptions = {}): Promise { + const maxRetries = options.maxRetries ?? MAX_RETRIES; + const defaultDelayMs = options.defaultDelayMs ?? RETRY_DELAY_MS; + + for (let attempt = 0; ; attempt++) { + try { + return await fn(); + } catch (error) { + if (!(error instanceof ApiError) || error.status !== 429) throw error; + if (attempt >= maxRetries) throw new RateLimitExceededError(maxRetries); + + const { delayMs, delaySeconds } = getRetryDelay(readRetryAfter(error), defaultDelayMs); + options.onRetry?.({ + attempt: attempt + 1, + delaySeconds, + message: `Rate limit hit (429), retrying in ${delaySeconds}s (attempt ${attempt + 1}/${maxRetries})`, + }); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + } + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.test.ts b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts new file mode 100644 index 000000000..e54efef6e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/run-store.test.ts @@ -0,0 +1,340 @@ +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + setSystemTime, + spyOn, + test, +} from "bun:test"; +import * as crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { + continueRun, + latestUserLines, + listRuns, + lockFile, + newRunId, + patchRun, + readRun, + resolveRunsDir, + runState, + RUNS_DIR_ENV, + startRun, +} from "./run-store.ts"; + +let workDir: string; +let configDir: string; +let runsDir: string; +let originalEnv: string | undefined; + +beforeAll(() => { + originalEnv = process.env[RUNS_DIR_ENV]; + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-run-store-config-")); + _setConfigDir(configDir); +}); + +afterAll(() => { + if (originalEnv === undefined) delete process.env[RUNS_DIR_ENV]; + else process.env[RUNS_DIR_ENV] = originalEnv; + _setConfigDir(undefined); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + delete process.env[RUNS_DIR_ENV]; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-run-store-"))); + runsDir = path.join(workDir, "runs"); +}); + +afterEach(() => { + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +const init = { kind: "import" as const, target: { instanceId: "ins_1" }, source: "clerk" }; + +describe("resolveRunsDir", () => { + test("prefers --runs-dir, resolved against the cwd", async () => { + process.env[RUNS_DIR_ENV] = "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/from/env"; + expect(await resolveRunsDir("flag", { cwd: workDir })).toBe(path.join(workDir, "flag")); + }); + + test("falls back to CLERK_MIGRATE_DIR", async () => { + process.env[RUNS_DIR_ENV] = "from-env"; + expect(await resolveRunsDir(undefined, { cwd: workDir })).toBe(path.join(workDir, "from-env")); + }); + + test("defaults to .clerk/migrate in the project, gitignored only when writing", async () => { + expect(await resolveRunsDir(undefined, { cwd: workDir })).toBe( + path.join(workDir, ".clerk", "migrate"), + ); + expect(fs.existsSync(path.join(workDir, ".gitignore"))).toBe(false); + + await resolveRunsDir(undefined, { cwd: workDir, write: true }); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toBe(".clerk/\n"); + }); +}); + +describe("a run's life", () => { + test("IDs sort by start time and read as a date", () => { + expect(newRunId(new Date(2026, 8, 29, 14, 5, 2))).toMatch(/^20260929-140502-[0-9a-f]{4}$/); + }); + + test("starts running, holding a lock", () => { + const run = startRun(runsDir, init); + expect(readRun(runsDir, run.record.id)).toMatchObject({ status: "running", kind: "import" }); + expect(fs.readFileSync(path.join(run.dir, "lock"), "utf-8")).toBe(String(process.pid)); + }); + + test("finishes complete when every user made it", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + + const record = run.finish(); + + expect(record).toMatchObject({ status: "complete", counts: { total: 1, created: 1 } }); + expect(record.finishedAt).toBeDefined(); + expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); + }); + + test.each([["failed"], ["skipped"], ["creating"]] as const)( + "finishes partial when a user was %s", + (status) => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + run.append({ sourceId: "b", status }); + expect(run.finish().status).toBe("partial"); + }, + ); + + test("counts each source ID by its last line", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "failed", error: "boom" }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + + expect(run.finish()).toMatchObject({ status: "complete", counts: { total: 1, created: 1 } }); + expect(latestUserLines(runsDir, run.record.id).get("a")?.clerkId).toBe("user_a"); + }); + + test("reads past a line a crash cut short", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + fs.appendFileSync(path.join(run.dir, "users.ndjson"), '{"sourceId":"b","sta'); + + expect([...latestUserLines(runsDir, run.record.id).keys()]).toEqual(["a"]); + }); + + test("a continued run starts a fresh line after one a crash cut short", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + fs.appendFileSync(path.join(run.dir, "users.ndjson"), '{"sourceId":"b","sta'); + fs.rmSync(path.join(run.dir, "lock")); + + continueRun(runsDir, run.record).append({ sourceId: "c", status: "created" }); + + expect([...latestUserLines(runsDir, run.record.id).keys()]).toEqual(["a", "c"]); + }); + + // A user created with no line is beyond both undo and a re-run, so the + // create that would follow must not go out. + test("append throws when the line cannot be written", () => { + const run = startRun(runsDir, init); + fs.mkdirSync(path.join(run.dir, "users.ndjson")); + + expect(() => run.append({ sourceId: "a", status: "creating" })).toThrow(); + }); +}); + +describe("locks and interruptions", () => { + // A PID no process can have. + const DEAD_PID = "2147483646"; + + test("a run whose process died is interrupted", () => { + const run = startRun(runsDir, init); + fs.writeFileSync(path.join(run.dir, "lock"), DEAD_PID); + expect(runState(runsDir, run.record)).toBe("interrupted"); + }); + + test("a run with no lock and no finish time is interrupted", () => { + const run = startRun(runsDir, init); + fs.rmSync(path.join(run.dir, "lock")); + expect(runState(runsDir, run.record)).toBe("interrupted"); + }); + + // Undo marks an interrupted import undone, but leaves it with no finish time. + test("an interrupted run that was undone reads as undone", () => { + const run = startRun(runsDir, init); + fs.rmSync(path.join(run.dir, "lock")); + expect(runState(runsDir, { ...run.record, status: "undone" })).toBe("undone"); + }); + + test("continuing takes the lock from a dead process and reopens the run", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + run.finish(); + + const again = continueRun(runsDir, readRun(runsDir, run.record.id)!); + expect(again.record).toMatchObject({ id: run.record.id, status: "running" }); + expect(again.record.finishedAt).toBeUndefined(); + again.append({ sourceId: "b", status: "created" }); + expect(again.finish().counts.total).toBe(2); + }); + + // In a container the CLI often gets the same PID every run, so a killed + // run's lock can hold this process's own PID. + test("a lock holding this process's own PID is stale", () => { + const run = startRun(runsDir, init); + expect(runState(runsDir, run.record)).toBe("interrupted"); + expect(() => continueRun(runsDir, run.record)).not.toThrow(); + }); + + test("reads as running while another live process holds the lock", () => { + const run = startRun(runsDir, init); + fs.writeFileSync(path.join(run.dir, "lock"), "1"); + expect(runState(runsDir, run.record)).toBe("running"); + }); + + // Two runs in the same second share an ID one time in 65,536. + test("a run ID that is already taken gets a new one, not a shared folder", () => { + setSystemTime(new Date("2026-10-02T12:00:00")); + const bytes = spyOn(crypto, "randomBytes") + .mockReturnValueOnce(Buffer.from([0xab, 0xcd]) as never) + .mockReturnValueOnce(Buffer.from([0xab, 0xcd]) as never) + .mockReturnValueOnce(Buffer.from([0x12, 0x34]) as never); + try { + const first = startRun(runsDir, init); + const second = startRun(runsDir, init); + expect(first.record.id).toEndWith("-abcd"); + expect(second.record.id).toEndWith("-1234"); + } finally { + bytes.mockRestore(); + setSystemTime(); + } + }); + + // Windows ignores POSIX modes. + test.skipIf(process.platform === "win32")("run folders are owner-only", () => { + const run = startRun(runsDir, init); + expect(fs.statSync(run.dir).mode & 0o777).toBe(0o700); + }); + + // In case a folder's mode is ever looser than the run store sets it. + // Users never sent have no line to count, so the caller says how many. + test("a finish with users never sent is partial", () => { + const sent = startRun(runsDir, init); + sent.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + expect(sent.finish().status).toBe("complete"); + + const stopped = startRun(runsDir, init); + stopped.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + const record = stopped.finish({ notSent: 2 }); + expect(record.status).toBe("partial"); + // Saved, so a later reader can tell how many were never sent. + expect(readRun(runsDir, record.id)?.counts).toEqual({ total: 1, created: 1, notSent: 2 }); + }); + + test("release leaves the run unfinished and unlocked", () => { + const run = startRun(runsDir, init); + run.release(); + expect(readRun(runsDir, run.record.id)?.finishedAt).toBeUndefined(); + expect(fs.existsSync(path.join(run.dir, "lock"))).toBe(false); + }); + + test.skipIf(process.platform === "win32")("run files are owner-only", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "u1", status: "creating" }); + for (const file of ["run.json", "users.ndjson", "lock"]) { + expect(fs.statSync(path.join(run.dir, file)).mode & 0o777).toBe(0o600); + } + }); + + test("refuses a run another live process holds, with exit 2", () => { + const run = startRun(runsDir, init); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(run.dir, "lock"), "1"); + + expect(() => continueRun(runsDir, run.record)).toThrow( + new RegExp( + `in use by another process \\(PID 1\\).*delete ${path.join(run.dir, "lock")}`, + "s", + ), + ); + }); +}); + +// A continue plans from what it read before its consent prompt; an undo or +// another continue can write to the run while that prompt waits. +describe("continueRun against a plan", () => { + const stopped = () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created" }); + return run.finish(); + }; + + test("continues a run nothing touched", () => { + const record = stopped(); + expect(() => continueRun(runsDir, record, 1).finish()).not.toThrow(); + }); + + test("refuses a run an undo marked while it waited, and lets go of the lock", () => { + const record = stopped(); + patchRun(runsDir, record.id, { status: "undone", undoneBy: "20260101-000000-abcd" }); + + expect(() => continueRun(runsDir, record, 1)).toThrow(/changed while this import was waiting/); + expect(fs.existsSync(lockFile(runsDir, record.id))).toBe(false); + expect(readRun(runsDir, record.id)?.status).toBe("undone"); + }); + + test("refuses a run another continue wrote to while it waited", () => { + const record = stopped(); + fs.appendFileSync( + path.join(runsDir, record.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "b", status: "created", clerkId: "user_b" })}\n`, + ); + + expect(() => continueRun(runsDir, record, 1)).toThrow(/changed while this import was waiting/); + }); +}); + +describe("continueRun and an unfinished undo", () => { + // The caller checked before taking the lock; an undo can start, and stop + // part-way, after that. + test("refuses an import with an undo that did not finish, and leaves its record", () => { + const run = startRun(runsDir, init); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + const record = run.finish(); + const undo = startRun(runsDir, { ...init, kind: "undo", undoes: record.id }); + undo.release(); + + expect(() => continueRun(runsDir, record)).toThrow(/has an undo that did not finish/); + expect(readRun(runsDir, record.id)).toMatchObject({ + status: record.status, + finishedAt: record.finishedAt, + }); + expect(fs.existsSync(lockFile(runsDir, record.id))).toBe(false); + }); +}); + +describe("listRuns", () => { + test("lists newest first and ignores folders that are not runs", () => { + const first = startRun(runsDir, init); + first.update({ startedAt: "2026-01-01T00:00:00.000Z" }); + const second = startRun(runsDir, { ...init, kind: "export" }); + second.update({ startedAt: "2026-02-01T00:00:00.000Z" }); + fs.mkdirSync(path.join(runsDir, "not-a-run")); + + expect(listRuns(runsDir).map((record) => record.id)).toEqual([ + second.record.id, + first.record.id, + ]); + }); + + test("is empty when the folder does not exist", () => { + expect(listRuns(path.join(workDir, "nowhere"))).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/run-store.ts b/packages/cli-core/src/commands/migrate/lib/run-store.ts new file mode 100644 index 000000000..ac98a0b38 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/run-store.ts @@ -0,0 +1,572 @@ +/** + * The run store: the one place `clerk migrate` keeps state. + * + * Every import, export and undo is a run. A run is a folder under the runs + * directory holding: + * + * - `run.json` — what ran, against what, from which file, and how it ended. + * - `users.ndjson` — one line per user outcome. The last line for a source ID + * wins, so a continued run appends rather than rewrites. + * - `lock` — the PID of the process working on the run. A live PID refuses a + * second writer; a dead one, or a run with no `finishedAt`, means the run + * was interrupted. + * + * `runs`, `undo`, re-runs and exports all read or write it, so there is no + * second record to drift out of step. + * + * Appends are synchronous so a run interrupted with Ctrl-C still leaves a + * complete record of everything already processed. + */ + +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { resolveProfile } from "../../../lib/config.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { ensureGitignoreEntry, getGitRepoRoot } from "../../../lib/git.ts"; +import { log } from "../../../lib/log.ts"; + +/** Overrides the runs directory, below `--runs-dir`. */ +export const RUNS_DIR_ENV = "CLERK_MIGRATE_DIR"; + +/** The `--runs-dir` option every migrate subcommand takes. */ +export const RUNS_DIR_FLAG = "--runs-dir "; +export const RUNS_DIR_DESCRIPTION = `Where migration runs are kept (default: .clerk/migrate in the project, or ${RUNS_DIR_ENV})`; + +export type RunKind = "import" | "undo" | "export"; +export type RunStatus = "running" | "complete" | "partial" | "undone"; +/** + * `creating` is written as `POST /v1/users` goes out, and stays the latest line + * when no answer says whether the create landed (an abort, a network error, a + * 5xx). The user may then exist in Clerk without its ID on record, so `undo` + * and a continued run look it up by `external_id`. + */ +export type UserStatus = "creating" | "created" | "failed" | "skipped" | "deleted" | "exported"; + +/** + * What a run acted on. For an import or undo, the Clerk instance. For an + * export, the source platform, plus the Clerk instance for `export clerk`. + */ +export type RunTarget = { + env?: string; + appId?: string; + appLabel?: string; + instanceId?: string; + instanceType?: string; + keySource?: string; + /** The platform an export read from. */ + platform?: string; +}; + +export type RunFile = { path: string; sha256: string }; + +/** `notSent`: users a stopped import never sent, so they have no line. */ +export type RunCounts = Partial> & { total: number; notSent?: number }; + +export type RunRecord = { + id: string; + kind: RunKind; + status: RunStatus; + startedAt: string; + finishedAt?: string; + target: RunTarget; + source?: string; + /** Content hash of a custom source, so an edited one is a different source. */ + sourceHash?: string; + file?: RunFile; + /** The export run an import read its file from. */ + fromExport?: string; + /** + * A Firebase import's hash parameters, as SHA-256 of the four (never the + * parameters themselves): every digest it builds carries them, so a continue + * with others is refused. + */ + firebaseHash?: string; + /** The import run an undo reverses. */ + undoes?: string; + /** The undo run that reversed this one. */ + undoneBy?: string; + counts: RunCounts; +}; + +/** An extra email or phone still to attach to a created user. */ +export type PendingIdentifier = { kind: "email" | "phone"; value: string; verified: boolean }; + +export type UserLine = { + sourceId: string; + clerkId?: string; + status: UserStatus; + /** + * On a `created` line: the extra identifiers not yet attached. A continued + * run attaches them; a later `created` line without it means they are done. + */ + pending?: PendingIdentifier[]; + /** Why a user was skipped. */ + reason?: string; + error?: string; + code?: string; + passwordDropped?: boolean; + /** + * On a `creating` line: the create sent this user's unverified identifiers + * reserved. A continued run that adopts the user attaches none of them. + */ + reserved?: boolean; +}; + +/** What `runs` shows for a run: its stored status, or how it stopped. */ +export type RunState = RunStatus | "interrupted"; + +const RUN_FILE = "run.json"; +const USERS_FILE = "users.ndjson"; +const LOCK_FILE = "lock"; + +// --- Location -------------------------------------------------------------- + +/** + * Where the project lives, for the default runs directory. + * + * A profile linked by directory names it outright. One linked by git remote + * or repository names the repository, whose root is the git toplevel. An + * unlinked directory outside git is its own project. + */ +async function projectRoot(cwd: string): Promise { + const profile = await resolveProfile(cwd); + if (profile?.resolvedVia === "directory") return profile.path; + return (await getGitRepoRoot(cwd)) ?? cwd; +} + +/** + * The runs directory: `--runs-dir`, then `CLERK_MIGRATE_DIR`, then + * `/.clerk/migrate`. + * + * @param options.write - The caller is about to write a run. The default + * location is gitignored first, because run files carry user data. + */ +export async function resolveRunsDir( + runsDir: string | undefined, + options: { write?: boolean; cwd?: string } = {}, +): Promise { + const cwd = options.cwd ?? process.cwd(); + if (runsDir) return path.resolve(cwd, runsDir); + + const fromEnv = process.env[RUNS_DIR_ENV]; + if (fromEnv) return path.resolve(cwd, fromEnv); + + const root = await projectRoot(cwd); + if (options.write) await ensureGitignoreEntry(root, ".clerk/"); + return path.join(root, ".clerk", "migrate"); +} + +/** The shape of a run ID, so one can be told apart from a file path. */ +export const RUN_ID_PATTERN = /^\d{8}-\d{6}-[0-9a-f]{4}$/; + +/** The folder one run lives in. */ +export function runDir(runsDir: string, id: string): string { + return path.join(runsDir, id); +} + +// --- IDs and hashes -------------------------------------------------------- + +/** + * `YYYYMMDD-HHmmss-xxxx`, local time. + * + * Sorts by start time, reads as a date, and the random suffix keeps two runs + * started in the same second apart. + */ +export function newRunId(now: Date = new Date()): string { + const pad = (value: number) => String(value).padStart(2, "0"); + const date = `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}`; + const time = `${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`; + return `${date}-${time}-${randomBytes(2).toString("hex")}`; +} + +/** Hex sha256 of a file's bytes. */ +export function sha256File(file: string): string { + return createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +} + +// --- Locking --------------------------------------------------------------- + +function isPidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: the process exists but belongs to someone else. + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +/** + * The PID holding the run's lock, when that process is still alive. + * + * This process's own PID counts as stale: in a container the CLI often gets + * the same PID every run, so a killed run's lock would otherwise read as live. + */ +export function liveLockPid(runsDir: string, id: string): number | undefined { + return livePidIn(lockFile(runsDir, id)); +} + +/** The live PID a lock file holds, by the same rules as {@link liveLockPid}. */ +function livePidIn(file: string): number | undefined { + let raw: string; + try { + raw = fs.readFileSync(file, "utf-8"); + } catch { + return undefined; + } + const pid = Number(raw.trim()); + return Number.isInteger(pid) && pid > 0 && pid !== process.pid && isPidAlive(pid) + ? pid + : undefined; +} + +/** The run's lock file. */ +export function lockFile(runsDir: string, id: string): string { + return path.join(runDir(runsDir, id), LOCK_FILE); +} + +const isExists = (error: unknown) => (error as NodeJS.ErrnoException).code === "EEXIST"; + +/** + * Takes the run's lock. Created exclusively (`wx`), so two processes can't + * both read "free" and both write; a stale lock is removed and taken once. + */ +function acquireLock(runsDir: string, id: string): void { + acquireLockFile(lockFile(runsDir, id), (holder, file) => + throwUsageError( + `Run ${id} is in use by another process${holder ? ` (PID ${holder})` : ""}. Wait for it to finish, then try again. ` + + `If that process is not a migrate run, delete ${file}.`, + ), + ); +} + +/** Takes the lock at `file`, or calls `refuse` with its live holder. */ +function acquireLockFile( + file: string, + refuse: (holder: number | undefined, file: string) => never, +): void { + const take = () => fs.writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 }); + + try { + take(); + return; + } catch (error) { + if (!isExists(error)) throw error; + } + const holder = livePidIn(file); + if (holder !== undefined) refuse(holder, file); + fs.rmSync(file, { force: true }); + try { + take(); + } catch (error) { + // Another process took the stale lock between the remove and the write. + if (isExists(error)) refuse(livePidIn(file), file); + throw error; + } +} + +/** What makes two imports the same job: the file, the source and the instance. */ +export type ImportIdentity = { sha256: string; source: string; instanceId: string }; + +/** + * The lock one import of a file, by a source, into an instance holds. In the + * temp directory, keyed by the runs folder too: it is taken before consent, + * when nothing may be written to the runs folder yet. + */ +export function importLockFile(runsDir: string, identity: ImportIdentity): string { + const key = createHash("sha256") + .update( + JSON.stringify([ + path.resolve(runsDir), + identity.sha256, + identity.source, + identity.instanceId, + ]), + ) + .digest("hex") + .slice(0, 32); + return path.join(os.tmpdir(), `clerk-migrate-import-${key}.lock`); +} + +/** + * Takes the lock for one import of this file, source and instance, before + * its checks, so no second process can pass them and start a run beside it. + * + * @returns Releases the lock. + * @throws UsageError when a live process holds it. + */ +export function lockImport(runsDir: string, identity: ImportIdentity): () => void { + const file = importLockFile(runsDir, identity); + acquireLockFile(file, (holder) => + throwUsageError( + `Another process${holder ? ` (PID ${holder})` : ""} is importing this file into this instance right now. ` + + `Wait for it to finish. If that process is not a migrate run, delete ${file}.`, + ), + ); + return () => fs.rmSync(file, { force: true }); +} + +// --- Reading --------------------------------------------------------------- + +/** One run's record, or `undefined` when the folder holds no readable run. */ +export function readRun(runsDir: string, id: string): RunRecord | undefined { + try { + const parsed = JSON.parse( + fs.readFileSync(path.join(runDir(runsDir, id), RUN_FILE), "utf-8"), + ) as RunRecord; + return parsed && typeof parsed === "object" && parsed.id === id ? parsed : undefined; + } catch { + return undefined; + } +} + +/** Every line of a run's `users.ndjson`, in the order written. */ +export function readUserLines(runsDir: string, id: string): UserLine[] { + let raw: string; + try { + raw = fs.readFileSync(path.join(runDir(runsDir, id), USERS_FILE), "utf-8"); + } catch { + return []; + } + const lines: UserLine[] = []; + for (const line of raw.split("\n")) { + if (!line.trim()) continue; + try { + lines.push(JSON.parse(line) as UserLine); + } catch { + // A line cut short by a crash mid-write. Every earlier line still counts. + } + } + return lines; +} + +/** Each source ID's latest outcome, in first-seen order. */ +export function latestUserLines(runsDir: string, id: string): Map { + const latest = new Map(); + for (const line of readUserLines(runsDir, id)) latest.set(line.sourceId, line); + return latest; +} + +/** + * How a run stands now. A run that never finished and whose process is gone + * was interrupted, whatever its stored status says. + */ +export function runState(runsDir: string, record: RunRecord): RunState { + // An interrupted run that was then undone has no finish time of its own. + if (record.finishedAt || record.status === "undone") return record.status; + return liveLockPid(runsDir, record.id) === undefined ? "interrupted" : "running"; +} + +/** Every readable run, newest first. */ +export function listRuns(runsDir: string): RunRecord[] { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(runsDir, { withFileTypes: true }); + } catch { + return []; + } + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => readRun(runsDir, entry.name)) + .filter((record): record is RunRecord => record !== undefined) + .sort((a, b) => b.startedAt.localeCompare(a.startedAt) || b.id.localeCompare(a.id)); +} + +// --- Writing --------------------------------------------------------------- + +function writeRecord(runsDir: string, record: RunRecord): void { + const file = path.join(runDir(runsDir, record.id), RUN_FILE); + // Written whole and renamed into place, so a reader never sees half a file. + const temp = `${file}.tmp`; + fs.writeFileSync(temp, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 }); + fs.renameSync(temp, file); +} + +/** Totals by each source ID's latest status. */ +export function countLines(latest: Iterable): RunCounts { + const counts: RunCounts = { total: 0 }; + for (const line of latest) { + counts.total++; + counts[line.status] = (counts[line.status] ?? 0) + 1; + } + return counts; +} + +/** A run being written by this process. */ +export type Run = { + readonly runsDir: string; + readonly dir: string; + record: RunRecord; + /** Appends one user outcome. */ + append(line: UserLine): void; + /** Merges fields into `run.json`. */ + update(patch: Partial): void; + /** + * Counts the outcomes, settles the status and releases the lock. + * + * `partial` when any user failed, was skipped, may not have been created or + * was never sent (`notSent`: they have no line to count), `complete` + * otherwise. + */ + finish(options?: { notSent?: number }): RunRecord; + /** + * Releases the lock and leaves the run unfinished: with no `finishedAt` and + * no live holder, it reads as interrupted. + */ + release(): void; +}; + +function openRun(runsDir: string, record: RunRecord): Run { + const dir = runDir(runsDir, record.id); + const usersFile = path.join(dir, USERS_FILE); + + const run: Run = { + runsDir, + dir, + record, + // Throws when the line cannot be written: a user created with no record is + // beyond both `undo` and a re-run, so the create it precedes must not go out. + append(line) { + // Owner-only like the folder: the lines carry emails and phones. + fs.appendFileSync(usersFile, `${JSON.stringify(line)}\n`, { mode: 0o600 }); + }, + update(patch) { + run.record = { ...run.record, ...patch }; + writeRecord(runsDir, run.record); + }, + finish(options) { + const counts = countLines(latestUserLines(runsDir, run.record.id).values()); + if (options?.notSent) counts.notSent = options.notSent; + const unfinished = + (counts.failed ?? 0) + + (counts.skipped ?? 0) + + (counts.creating ?? 0) + + (counts.notSent ?? 0); + run.update({ + counts, + status: unfinished > 0 ? "partial" : "complete", + finishedAt: new Date().toISOString(), + }); + run.release(); + return run.record; + }, + release() { + fs.rmSync(path.join(dir, LOCK_FILE), { force: true }); + }, + }; + return run; +} + +export type StartRunInit = Omit; + +/** Creates a run folder, takes its lock and writes the first `run.json`. */ +export function startRun(runsDir: string, init: StartRunInit): Run { + // Owner-only: a run's files hold user data. + fs.mkdirSync(runsDir, { recursive: true, mode: 0o700 }); + // Created exclusively: two runs started in the same second share an ID one + // time in 65,536, and must not share a folder. + let id = newRunId(); + for (;;) { + try { + fs.mkdirSync(runDir(runsDir, id), { mode: 0o700 }); + break; + } catch (error) { + if (!isExists(error)) throw error; + id = newRunId(); + } + } + acquireLock(runsDir, id); + + const record: RunRecord = { + id, + status: "running", + startedAt: new Date().toISOString(), + counts: { total: 0 }, + ...init, + }; + writeRecord(runsDir, record); + log.debug(`migrate: started ${init.kind} run ${id} in ${runsDir}`); + return openRun(runsDir, record); +} + +/** + * Reopens a finished or interrupted run to keep working on it. + * + * @throws UsageError when another live process holds its lock. + */ +export function continueRun( + runsDir: string, + record: RunRecord, + /** + * How many user lines the caller planned against. The record and the lines + * were read before the lock was taken, while a prompt may have waited: an + * undo, or another continue, can change either in the meantime. + */ + plannedLines?: number, +): Run { + acquireLock(runsDir, record.id); + if (plannedLines !== undefined) { + const fresh = readRun(runsDir, record.id); + const changed = + !fresh || + fresh.status !== record.status || + fresh.undoneBy !== record.undoneBy || + readUserLines(runsDir, record.id).length !== plannedLines; + if (changed) { + fs.rmSync(lockFile(runsDir, record.id), { force: true }); + throwUsageError( + `Run ${record.id} changed while this import was waiting, so its plan is out of date. ` + + "Nothing was imported. Run the command again.", + ); + } + } + // Checked again under the lock, before the record says running: an undo + // can start, and stop part-way, after the caller's own check, leaving the + // import's status and lines as they were. + const undoRun = + record.kind === "import" + ? listRuns(runsDir).find( + (candidate) => + candidate.kind === "undo" && + candidate.undoes === record.id && + runState(runsDir, candidate) !== "complete", + ) + : undefined; + if (undoRun) { + fs.rmSync(lockFile(runsDir, record.id), { force: true }); + throwUsageError( + `Run ${record.id} has an undo that did not finish (run ${undoRun.id}). ` + + `Finish it with \`clerk migrate undo ${record.id}\`, or pass --new-run to import into a new run.`, + ); + } + // A crash mid-write leaves a torn last line; end it so the next append + // starts a line of its own instead of fusing with it. + const usersFile = path.join(runDir(runsDir, record.id), USERS_FILE); + const written = fs.existsSync(usersFile) ? fs.readFileSync(usersFile, "utf-8") : ""; + if (written && !written.endsWith("\n")) fs.appendFileSync(usersFile, "\n"); + const run = openRun(runsDir, record); + run.record = { ...record, status: "running" }; + delete run.record.finishedAt; + writeRecord(runsDir, run.record); + log.debug(`migrate: continuing ${record.kind} run ${record.id} in ${runsDir}`); + return run; +} + +/** + * Takes a run's lock without opening it, so no other process can continue it. + * + * @returns Releases the lock. + * @throws UsageError when another live process holds it. + */ +export function lockRun(runsDir: string, id: string): () => void { + acquireLock(runsDir, id); + return () => fs.rmSync(lockFile(runsDir, id), { force: true }); +} + +/** Merges fields into a run this process is not writing, such as `undoneBy`. */ +export function patchRun(runsDir: string, id: string, patch: Partial): void { + const record = readRun(runsDir, id); + if (record) writeRecord(runsDir, { ...record, ...patch }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts new file mode 100644 index 000000000..15bb6309a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.test.ts @@ -0,0 +1,128 @@ +import { expect, test } from "bun:test"; +import { createApiScheduler } from "./scheduler.ts"; + +/** Resolves after `ms`, so a task can be held open while others queue behind it. */ +const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test("never runs more tasks at once than the concurrency limit", async () => { + const schedule = createApiScheduler(3, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 20 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(5); + active--; + }), + ), + ); + + expect(peak).toBe(3); + expect(active).toBe(0); +}); + +test("frees a slot when a task throws, instead of deadlocking the queue", async () => { + const schedule = createApiScheduler(1, 10_000); + + await expect(schedule(() => Promise.reject(new Error("boom")))).rejects.toThrow("boom"); + + // If release() had been skipped on the failure path, this would hang. + expect(await schedule(async () => "ok")).toBe("ok"); +}); + +test("paces calls to the rate limit", async () => { + // 100 req/s -> 10ms between starts; 5 calls span at least 4 intervals. + const schedule = createApiScheduler(5, 100); + const started = performance.now(); + + await Promise.all(Array.from({ length: 5 }, () => schedule(async () => {}))); + + expect(performance.now() - started).toBeGreaterThanOrEqual(35); +}); + +test("returns each task's own resolved value", async () => { + const schedule = createApiScheduler(2, 10_000); + const results = await Promise.all([1, 2, 3].map((n) => schedule(async () => n * 2))); + expect(results).toEqual([2, 4, 6]); +}); + +test("treats zero or negative limits as one", async () => { + const schedule = createApiScheduler(0, 10_000); + let active = 0; + let peak = 0; + + await Promise.all( + Array.from({ length: 4 }, () => + schedule(async () => { + active++; + peak = Math.max(peak, active); + await wait(2); + active--; + }), + ), + ); + + expect(peak).toBe(1); +}); + +// A 429 means the instance is over its limit for everyone, not just the call +// that hit it. +test("holds every unsent call through a pause", async () => { + const schedule = createApiScheduler(4, 10_000); + const started = performance.now(); + schedule.pause(300); + + const startedAt = await Promise.all( + Array.from({ length: 3 }, async () => schedule(async () => performance.now() - started)), + ); + + for (const at of startedAt) expect(at).toBeGreaterThanOrEqual(290); +}); + +// Released together, the held calls would hit the limit the pause waited out. +test("keeps calls paced when a pause ends", async () => { + const schedule = createApiScheduler(4, 10); + const started = performance.now(); + const first = schedule(async () => schedule.pause(300)); + const held = [1, 2, 3].map(async () => schedule(async () => performance.now() - started)); + + await first; + const startedAt = (await Promise.all(held)).sort((a, b) => a - b); + // 80 of the 100ms: a loaded runner's timers drift (89.97ms seen in CI), + // while a doubled rate would space them 50ms apart and a burst about 0. + for (let i = 1; i < startedAt.length; i++) { + expect(startedAt[i]! - startedAt[i - 1]!).toBeGreaterThanOrEqual(80); + } +}); + +test("holds a call already waiting on the pacing interval", async () => { + // 2 req/s: the second call waits ~500ms. The first, like a 429, pauses the + // run while the second is in that wait. + const schedule = createApiScheduler(2, 2); + const started = performance.now(); + const first = schedule(async () => schedule.pause(800)); + const second = schedule(async () => performance.now() - started); + + await first; + expect(await second).toBeGreaterThanOrEqual(790); +}); + +// A queue that will only find the run stopped should drain, not wait out +// one paced turn per call. +test("a call that skips its wait runs at once and takes no paced turn", async () => { + const schedule = createApiScheduler(1, 2); // 500ms apart + const started = performance.now(); + await schedule(async () => undefined); + const skipped = await Promise.all( + [1, 2, 3].map(async () => + schedule(async () => performance.now() - started, { skipWait: () => true }), + ), + ); + for (const at of skipped) expect(at).toBeLessThan(250); + // Skipped calls took no turn, so the next paced call is the second one. + const next = await schedule(async () => performance.now() - started); + expect(next).toBeLessThan(750); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/scheduler.ts b/packages/cli-core/src/commands/migrate/lib/scheduler.ts new file mode 100644 index 000000000..1d5db6ff4 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/scheduler.ts @@ -0,0 +1,90 @@ +/** + * Concurrency gate plus rate pacing for BAPI calls. + * + * Replaces the standalone migration-tool's `p-limit` dependency: a bounded + * queue is a few lines, and the compiled binary carries one fewer package. + * + * Both limits apply to individual API calls rather than whole users, so a user + * with ten extra email addresses cannot burst past the instance's rate limit. + */ + +/** + * Runs `fn` once a slot is free and the pacing interval has elapsed. + * + * `first` puts `fn` ahead of every queued call without it: a user's extra + * identifiers attach before the next user is created, so a run stopped midway + * leaves few users waiting on attaches. + */ +export type ScheduleOptions = { + first?: boolean; + /** + * Checked once the call has a slot: when it returns true, `fn` runs at once + * and takes no paced turn. For a call that will only find the run stopped, + * so a Ctrl-C or a full instance drains the queue instead of pacing it. + */ + skipWait?: () => boolean; +}; + +export type ApiScheduler = ((fn: () => Promise, options?: ScheduleOptions) => Promise) & { + /** + * Holds every call not yet sent until `ms` from now, so a 429 pauses the + * whole run rather than only the call that hit it. + */ + pause(ms: number): void; +}; + +export function createApiScheduler(concurrencyLimit: number, rateLimit: number): ApiScheduler { + const maxConcurrent = Math.max(1, Math.floor(concurrencyLimit)); + const intervalMs = Math.ceil(1000 / Math.max(1, rateLimit)); + const waiting: (() => void)[] = []; + const waitingFirst: (() => void)[] = []; + let active = 0; + let nextRequestAt = 0; + let pausedUntil = 0; + + async function acquire(first: boolean): Promise { + if (active < maxConcurrent) { + active++; + return Promise.resolve(); + } + return new Promise((resolve) => (first ? waitingFirst : waiting).push(resolve)); + } + + function release(): void { + // One macrotask later, so the finished call's follow-up (a user's attaches) + // is queued before the slot is handed on. + setImmediate(() => { + const next = waitingFirst.shift() ?? waiting.shift(); + // Hand the slot straight to the next waiter; `active` is unchanged + // because the slot never actually frees up. + if (next) next(); + else active--; + }); + } + + const schedule = async (fn: () => Promise, options?: ScheduleOptions) => { + await acquire(options?.first ?? false); + try { + if (options?.skipWait?.()) return await fn(); + // A pause that starts during the wait takes a fresh paced slot after it, + // so calls held through a pause resume an interval apart, not together. + for (;;) { + const pauseSeen = pausedUntil; + const now = Date.now(); + const startAt = Math.max(now, nextRequestAt, pausedUntil); + nextRequestAt = startAt + intervalMs; + if (startAt > now) await new Promise((resolve) => setTimeout(resolve, startAt - now)); + if (pausedUntil === pauseSeen) break; + } + return await fn(); + } finally { + release(); + } + }; + + return Object.assign(schedule, { + pause(ms: number) { + pausedUntil = Math.max(pausedUntil, Date.now() + ms); + }, + }); +} diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts new file mode 100644 index 000000000..00cde92c5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, test } from "bun:test"; +import { toClerkStrategy } from "./clerk-config.ts"; +import { + countProviders, + findDisabledProviders, + findUsersWithOnlyDisabledProviders, + getUserProviders, +} from "./supabase-providers.ts"; + +/** A Supabase row carrying the given providers, in the JSON export's shape. */ +const user = (id: string, providers: string[] | string | undefined) => ({ + id, + raw_app_meta_data: + providers === undefined ? undefined : JSON.stringify({ provider: "email", providers }), +}); + +describe("getUserProviders", () => { + test("reads providers from a JSON-string column, as a CSV export writes it", () => { + expect(getUserProviders(user("u1", ["email", "discord"]))).toEqual(["email", "discord"]); + }); + + test("reads providers from an object column, as a JSON export writes it", () => { + expect(getUserProviders({ id: "u1", raw_app_meta_data: { providers: ["google"] } })).toEqual([ + "google", + ]); + }); + + test("splits a delimited providers string", () => { + expect( + getUserProviders({ id: "u1", raw_app_meta_data: { providers: "email, discord" } }), + ).toEqual(["email", "discord"]); + }); + + test.each([ + ["missing column", { id: "u1" }], + ["unparseable column", { id: "u1", raw_app_meta_data: "{not json" }], + ["array column", { id: "u1", raw_app_meta_data: "[]" }], + ["no providers key", { id: "u1", raw_app_meta_data: '{"provider":"email"}' }], + ])("returns nothing for a %s", (_label, row) => { + expect(getUserProviders(row)).toEqual([]); + }); +}); + +describe("toClerkStrategy", () => { + test.each([ + ["google", "oauth_google"], + ["discord", "oauth_discord"], + ["github", "oauth_github"], + ["azure", "oauth_microsoft"], + ["twitter", "oauth_x"], + ["slack_oidc", "oauth_slack"], + ])("%s -> %s", (provider, strategy) => { + expect(toClerkStrategy(provider)).toBe(strategy); + }); +}); + +describe("countProviders", () => { + test("counts each provider across the export", () => { + expect( + countProviders([ + user("u1", ["email"]), + user("u2", ["email", "discord"]), + user("u3", ["discord"]), + ]), + ).toEqual({ email: 2, discord: 2 }); + }); +}); + +describe("findDisabledProviders", () => { + test("names the social providers Clerk does not have enabled", () => { + const rows = [user("u1", ["email", "google"]), user("u2", ["discord"])]; + expect(findDisabledProviders(rows, ["oauth_google"], toClerkStrategy)).toEqual(["discord"]); + }); + + test("never treats email or phone as disabled", () => { + const rows = [user("u1", ["email"]), user("u2", ["phone"]), user("u3", ["anonymous_users"])]; + expect(findDisabledProviders(rows, [], toClerkStrategy)).toEqual([]); + }); + + test("returns nothing when every provider is enabled", () => { + const rows = [user("u1", ["google"]), user("u2", ["github"])]; + expect(findDisabledProviders(rows, ["oauth_google", "oauth_github"], toClerkStrategy)).toEqual( + [], + ); + }); +}); + +describe("findUsersWithOnlyDisabledProviders", () => { + test("excludes a user whose sole provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], ["discord"]); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1 }); + }); + + test("keeps a user who can still sign in with email", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["email", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("keeps a user who has another enabled social provider", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["google", "discord"])], + ["discord"], + ); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes a user whose every provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders( + [user("u1", ["discord", "twitch"])], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1"]); + expect(result.byProvider).toEqual({ discord: 1, twitch: 1 }); + }); + + test("keeps a user with no provider data at all", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", undefined)], ["discord"]); + expect(result.excludedIds.size).toBe(0); + }); + + test("excludes nobody when no provider is disabled", () => { + const result = findUsersWithOnlyDisabledProviders([user("u1", ["discord"])], []); + expect(result.excludedIds.size).toBe(0); + }); + + test("reports a per-provider breakdown across many users", () => { + const result = findUsersWithOnlyDisabledProviders( + [ + user("u1", ["discord"]), + user("u2", ["discord"]), + user("u3", ["twitch"]), + user("u4", ["email", "discord"]), + ], + ["discord", "twitch"], + ); + expect([...result.excludedIds]).toEqual(["u1", "u2", "u3"]); + expect(result.byProvider).toEqual({ discord: 2, twitch: 1 }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts new file mode 100644 index 000000000..d123dd5e0 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/supabase-providers.ts @@ -0,0 +1,145 @@ +/** + * Cross-references the social providers in a Supabase export against what the + * destination Clerk instance has enabled. + * + * Ported from the standalone migration-tool's `src/lib/supabase.ts`, minus its + * hand-rolled CSV parser — the export is read through the same + * `readRawUsers` path every other Supabase read uses. + */ + +import { readRawUsers } from "./transform.ts"; + +/** + * Supabase lists these alongside social providers in `providers`, but they are + * built into Clerk and can never be "not enabled". + */ +export const NON_SOCIAL_PROVIDERS = new Set(["email", "phone", "anonymous_users"]); + +function parseMaybeJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value); + } catch { + return value; + } +} + +/** + * Reads a user's auth providers from `raw_app_meta_data`. + * + * The column arrives as a JSON string from a CSV export and as an object from + * a JSON one, and its `providers` value is itself sometimes a string. + */ +export function getUserProviders(user: Record): string[] { + const appMeta = parseMaybeJson(user.raw_app_meta_data); + if (!appMeta || typeof appMeta !== "object" || Array.isArray(appMeta)) return []; + + const providers = parseMaybeJson((appMeta as Record).providers); + if (Array.isArray(providers)) { + return providers.map((provider) => String(provider).trim()).filter(Boolean); + } + if (typeof providers === "string") { + return providers + .split(/[,|]/) + .map((provider) => provider.trim()) + .filter(Boolean); + } + return []; +} + +export type ProviderExclusions = { + /** Source IDs of users to skip. */ + excludedIds: Set; + /** How many excluded users each disabled provider accounts for. */ + byProvider: Record; +}; + +/** + * Finds the users whose *only* way in is a provider Clerk does not have + * enabled. + * + * A user keeps their place if any one of their providers still works — + * including email and phone. Excluding on "has at least one disabled provider" + * instead would drop users who could sign in perfectly well another way. + * + * @param disabled - Supabase provider keys not enabled in Clerk. + */ +export function findUsersWithOnlyDisabledProviders( + users: Record[], + disabled: string[], +): ProviderExclusions { + const empty: ProviderExclusions = { excludedIds: new Set(), byProvider: {} }; + if (disabled.length === 0) return empty; + + const disabledSet = new Set(disabled); + const excludedIds = new Set(); + const byProvider: Record = {}; + + for (const user of users) { + const providers = getUserProviders(user); + // No provider data means no basis to exclude — err towards importing. + if (providers.length === 0) continue; + + const hasUsableProvider = providers.some( + (provider) => NON_SOCIAL_PROVIDERS.has(provider) || !disabledSet.has(provider), + ); + if (hasUsableProvider) continue; + + excludedIds.add(String(user.id)); + for (const provider of providers.filter((p) => disabledSet.has(p))) { + byProvider[provider] = (byProvider[provider] ?? 0) + 1; + } + } + + return { excludedIds, byProvider }; +} + +/** Counts users per provider across the export, for reporting. */ +export function countProviders(users: Record[]): Record { + const counts: Record = {}; + for (const user of users) { + for (const provider of getUserProviders(user)) { + counts[provider] = (counts[provider] ?? 0) + 1; + } + } + return counts; +} + +/** + * The same counts, minus Supabase's pseudo-providers. + * + * Supabase lists `email` and `phone` in `providers` next to real connections, + * but Clerk has no `oauth_email` to enable — so anything cross-referencing + * against the instance's social settings must drop them, or every + * password-based user reads as "not enabled in Clerk". + */ +export function countSocialProviders(users: Record[]): Record { + return Object.fromEntries( + Object.entries(countProviders(users)).filter( + ([provider]) => !NON_SOCIAL_PROVIDERS.has(provider), + ), + ); +} + +/** + * Every social provider present in the export that Clerk does not have + * enabled. + * + * @param enabledStrategies - Clerk strategy names (`oauth_google`, …). + * @param toStrategy - Maps a Supabase provider key to its Clerk strategy. + */ +export function findDisabledProviders( + users: Record[], + enabledStrategies: string[], + toStrategy: (provider: string) => string, +): string[] { + const enabled = new Set(enabledStrategies); + return Object.keys(countSocialProviders(users)).filter( + (provider) => !enabled.has(toStrategy(provider)), + ); +} + +/** Reads a Supabase export and returns its raw rows for provider analysis. */ +export async function readSupabaseRows(file: string): Promise[]> { + return readRawUsers(file, "supabase"); +} diff --git a/packages/cli-core/src/commands/migrate/lib/target.test.ts b/packages/cli-core/src/commands/migrate/lib/target.test.ts new file mode 100644 index 000000000..f0f620520 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/target.test.ts @@ -0,0 +1,241 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../../lib/config.ts"; +import { getMode, setMode } from "../../../mode.ts"; +import { EXIT_CODE, type CliError } from "../../../lib/errors.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { + describeTarget, + fetchInstanceIdentity, + printTarget, + resolveClerkTarget, +} from "./target.ts"; + +const captured = useCaptureLog(); + +describe("printTarget", () => { + test.each([ + [ + "a linked app", + { + env: "production", + appId: "app_1", + appLabel: "My App", + instanceId: "ins_1", + keySource: "linked profile", + }, + "Target: My App (app_1), production instance ins_1", + "Key from: linked profile", + ], + [ + "a bare --secret-key", + { env: "development", instanceId: "ins_2", keySource: "--secret-key" }, + "Target: development instance ins_2", + "Key from: --secret-key", + ], + [ + "an exported CLERK_SECRET_KEY", + { env: "development", instanceId: "ins_3", keySource: "CLERK_SECRET_KEY env var" }, + "Target: development instance ins_3", + "Key from: CLERK_SECRET_KEY env var", + ], + ])("names %s", (_label, target, first, second) => { + printTarget(target); + const lines = Bun.stripANSI(captured.err).split("\n"); + expect(lines).toEqual([first, second]); + }); + + test("prints no escape codes for an agent", () => { + const original = getMode(); + setMode("agent"); + try { + printTarget({ env: "development", instanceId: "ins_2", keySource: "--secret-key" }); + } finally { + setMode(original); + } + expect(captured.err).toBe("Target: development instance ins_2\nKey from: --secret-key"); + }); + + test("names an export's source platform", () => { + printTarget({ platform: "auth0" }); + expect(captured.err).toBe("Source: auth0"); + }); + + test("names the Clerk instance a Clerk export reads", () => { + printTarget({ platform: "clerk", env: "production", instanceId: "ins_9" }); + expect(Bun.stripANSI(captured.err)).toBe("Source: Clerk, production instance ins_9"); + }); +}); + +describe("describeTarget", () => { + test("is one line for a list", () => { + expect(describeTarget({ appLabel: "My App", env: "development", instanceId: "ins_1" })).toBe( + "My App (development, ins_1)", + ); + expect(describeTarget({ platform: "auth0" })).toBe("auth0"); + }); +}); + +describe("fetchInstanceIdentity", () => { + let originalFetch: typeof globalThis.fetch; + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + }); + + test("reads the instance behind the key", async () => { + globalThis.fetch = (async () => + Response.json({ + object: "instance", + id: "ins_7", + environment_type: "production", + })) as unknown as typeof fetch; + + expect(await fetchInstanceIdentity("sk_live_x")).toEqual({ + instanceId: "ins_7", + env: "production", + }); + }); + + test("retries a 429 rather than falling back", async () => { + let calls = 0; + globalThis.fetch = (async () => + ++calls === 1 + ? new Response("{}", { status: 429, headers: { "retry-after": "1" } }) + : Response.json({ + id: "ins_7", + environment_type: "development", + })) as unknown as typeof fetch; + + expect((await fetchInstanceIdentity("sk_test_x")).instanceId).toBe("ins_7"); + expect(calls).toBe(2); + }); + + // The same key always addresses the same instance, so a hash still tells two + // instances apart when the API cannot say. + test("falls back to a stable hash of the key when the instance cannot be read", async () => { + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + + const first = await fetchInstanceIdentity("sk_test_a"); + expect(first.instanceId).toMatch(/^key_[0-9a-f]{16}$/); + expect(first.env).toBe("development"); + expect(await fetchInstanceIdentity("sk_test_a")).toEqual(first); + expect((await fetchInstanceIdentity("sk_test_b")).instanceId).not.toBe(first.instanceId); + }); +}); + +describe("resolveClerkTarget --instance", () => { + let originalFetch: typeof globalThis.fetch; + let originalKey: string | undefined; + + beforeAll(() => { + originalFetch = globalThis.fetch; + originalKey = process.env.CLERK_SECRET_KEY; + globalThis.fetch = (async () => + Response.json({ id: "ins_prod", environment_type: "production" })) as unknown as typeof fetch; + }); + + afterAll(() => { + globalThis.fetch = originalFetch; + if (originalKey === undefined) delete process.env.CLERK_SECRET_KEY; + else process.env.CLERK_SECRET_KEY = originalKey; + }); + + // An exported production key would otherwise win over `--instance dev` + // without a word, and the import would write to production. + test.each([ + ["an exported key", { instance: "dev" }, "sk_live_x"], + ["--secret-key", { instance: "dev", secretKey: "sk_live_x" }, undefined], + ["--secret-key, with a literal ID", { instance: "ins_dev", secretKey: "sk_live_x" }, undefined], + ])("refuses %s that addresses another instance", async (_label, options, envKey) => { + if (envKey) process.env.CLERK_SECRET_KEY = envKey; + else delete process.env.CLERK_SECRET_KEY; + + const error = (await resolveClerkTarget(options).catch((e: unknown) => e)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("does not match the key"); + }); + + test("says a literal --instance could not be verified when the lookup fails", async () => { + delete process.env.CLERK_SECRET_KEY; + const working = globalThis.fetch; + globalThis.fetch = (async () => + new Response("nope", { status: 500 })) as unknown as typeof fetch; + try { + const error = (await resolveClerkTarget({ + instance: "ins_prod", + secretKey: "sk_live_x", + }).catch((e: unknown) => e)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Could not verify"); + expect(error.message).not.toContain("does not match"); + } finally { + globalThis.fetch = working; + } + }); + + test.each([["prod"], ["production"], ["ins_prod"]])("accepts --instance %s", async (instance) => { + delete process.env.CLERK_SECRET_KEY; + const { target } = await resolveClerkTarget({ instance, secretKey: "sk_live_x" }); + expect(target.instanceId).toBe("ins_prod"); + }); +}); + +describe("resolveClerkTarget key source", () => { + let originalFetch: typeof globalThis.fetch; + let originalKey: string | undefined; + let originalCwd: string; + let workDir: string; + let configDir: string; + + beforeEach(() => { + originalFetch = globalThis.fetch; + originalKey = process.env.CLERK_SECRET_KEY; + originalCwd = process.cwd(); + delete process.env.CLERK_SECRET_KEY; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-target-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-target-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + globalThis.fetch = (async () => + Response.json({ id: "ins_1", environment_type: "development" })) as unknown as typeof fetch; + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + if (originalKey === undefined) delete process.env.CLERK_SECRET_KEY; + else process.env.CLERK_SECRET_KEY = originalKey; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); + }); + + // A key in `.env` may be any app's, an account's included. + test("names a key from .env by its file, not as an accountless app", async () => { + fs.writeFileSync(path.join(workDir, ".env"), "CLERK_SECRET_KEY=sk_test_x\n"); + const { target } = await resolveClerkTarget({}); + expect(target.keySource).toBe(".env"); + expect(target.appLabel).toBeUndefined(); + }); + + test("names the SDK's own keyless file as an accountless app", async () => { + fs.mkdirSync(path.join(workDir, ".clerk", ".tmp"), { recursive: true }); + fs.writeFileSync( + path.join(workDir, ".clerk", ".tmp", "keyless.json"), + JSON.stringify({ secretKey: "sk_test_x", publishableKey: "pk_test_x" }), + ); + const { target } = await resolveClerkTarget({}); + expect(target).toMatchObject({ + keySource: "accountless app (.clerk/.tmp/keyless.json)", + appLabel: "accountless app", + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/target.ts b/packages/cli-core/src/commands/migrate/lib/target.ts new file mode 100644 index 000000000..f198efc85 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/target.ts @@ -0,0 +1,195 @@ +/** + * The Clerk instance a migrate command acts on, and where its key came from. + * + * `resolveBapiSecretKey` picks the key. This names what that key points at: + * the instance ID and environment come from `GET /v1/instance`, so every key + * source yields the same identity. That identity is what the run store keys + * resumes and undos on, so an app label or an `--instance` alias is never + * enough on its own. + */ + +import { createHash } from "node:crypto"; +import { bapiRequest } from "../../../lib/bapi.ts"; +import { dim } from "../../../lib/color.ts"; +import { resolveBapiSecretKey } from "../../../lib/bapi-command.ts"; +import { INSTANCE_ALIASES, resolveAppContext } from "../../../lib/config.ts"; +import { throwUsageError } from "../../../lib/errors.ts"; +import { resolveKeylessTarget, SDK_KEYLESS_SOURCE } from "../../../lib/keyless-target.ts"; +import { log } from "../../../lib/log.ts"; +import { isHuman } from "../../../mode.ts"; +import { detectInstanceType } from "./instance.ts"; +import { retryOn429 } from "./retry.ts"; +import type { RunTarget } from "./run-store.ts"; + +export type TargetOptions = { + secretKey?: string; + app?: string; + instance?: string; +}; + +/** A resolved Clerk instance. Every field a run needs is present. */ +export type ClerkTarget = RunTarget & { + env: string; + instanceId: string; + instanceType: "dev" | "prod"; + keySource: string; +}; + +/** + * Which rung of `resolveBapiSecretKey`'s chain supplies the key, and the app + * it belongs to when that rung knows one. + * + * Mirrors the chain's order exactly. An exported `CLERK_SECRET_KEY` wins over + * a linked profile, so the profile's app is not named for it: that key may + * belong to any app at all. + */ +async function describeKeySource( + options: TargetOptions, +): Promise<{ keySource: string; appId?: string; appLabel?: string }> { + if (options.secretKey) return { keySource: "--secret-key" }; + + if (options.app) { + const ctx = await resolveAppContext({ app: options.app, instance: options.instance }); + return { keySource: "--app", appId: ctx.appId, appLabel: ctx.appLabel }; + } + + if (process.env.CLERK_SECRET_KEY) return { keySource: "CLERK_SECRET_KEY env var" }; + + // Only the SDK's own file says the app is accountless: a key in `.env` + // may belong to any app, an account's included. + const keyless = await resolveKeylessTarget({ instance: options.instance }); + if (keyless) { + return keyless.source === SDK_KEYLESS_SOURCE + ? { keySource: `accountless app (${keyless.source})`, appLabel: "accountless app" } + : { keySource: keyless.source }; + } + + const ctx = await resolveAppContext({ instance: options.instance }); + return { keySource: "linked profile", appId: ctx.appId, appLabel: ctx.appLabel }; +} + +/** + * The instance behind a key. + * + * Falls back to a hash of the key when `GET /v1/instance` cannot be read: the + * same key always addresses the same instance, so it still tells two + * instances apart, and nothing is sent anywhere. A 429 is retried first: it is + * most likely straight after a large import, which is when `undo` runs. + */ +export async function fetchInstanceIdentity( + secretKey: string, +): Promise<{ instanceId: string; env: string }> { + const fallbackEnv = detectInstanceType(secretKey) === "prod" ? "production" : "development"; + try { + const { body } = await retryOn429(async () => + bapiRequest({ method: "GET", path: "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/v1/instance", secretKey }), + ); + const instance = body as { id?: unknown; environment_type?: unknown }; + if (typeof instance.id === "string" && instance.id.startsWith("ins_")) { + return { + instanceId: instance.id, + env: + typeof instance.environment_type === "string" ? instance.environment_type : fallbackEnv, + }; + } + } catch (error) { + log.debug(`migrate: could not read the instance behind the key: ${String(error)}`); + } + return { instanceId: keyInstanceId(secretKey), env: fallbackEnv }; +} + +/** + * The stand-in ID for a key whose instance Clerk did not name: a hash of the + * key. A run recorded under it is matched by the same key later, once + * `GET /v1/instance` answers again. + */ +export function keyInstanceId(secretKey: string): string { + return `key_${createHash("sha256").update(secretKey).digest("hex").slice(0, 16)}`; +} + +/** + * Refuses an `--instance` the key does not address. + * + * With a key from `--secret-key` or `CLERK_SECRET_KEY`, the key alone picks + * the instance and `--instance` would be ignored without a word. Migrate + * writes and deletes in bulk, so `--instance dev` next to an exported + * `sk_live_` key must not reach production. + */ +export function assertInstanceFlagMatches( + options: TargetOptions, + keySource: string, + identity: { instanceId: string; env: string }, +): void { + const flag = options.instance; + if (!flag || (keySource !== "--secret-key" && !keySource.startsWith("CLERK_SECRET_KEY"))) return; + const wanted = INSTANCE_ALIASES[flag]; + const matches = wanted ? identity.env === wanted : identity.instanceId === flag; + if (matches) return; + // A `key_` ID means Clerk did not name the instance, so a literal ID cannot + // be checked; saying it "does not match" would blame the flag. + if (!wanted && identity.instanceId.startsWith("key_")) { + throwUsageError( + `Could not verify that the key from ${keySource} addresses --instance ${flag}: ` + + "Clerk did not answer GET /v1/instance. Nothing was changed.\n" + + "Try again, or pass --instance dev or --instance prod.", + ); + } + throwUsageError( + `--instance ${flag} does not match the key from ${keySource}, which addresses the ` + + `${identity.env} instance ${identity.instanceId}. Nothing was changed.\n` + + "Pass the key for that instance with --secret-key, or drop --instance.", + ); +} + +/** Resolves the key, then names the instance it addresses. */ +export async function resolveClerkTarget( + options: TargetOptions, +): Promise<{ secretKey: string; target: ClerkTarget }> { + const secretKey = await resolveBapiSecretKey(options); + const source = await describeKeySource(options); + const { instanceId, env } = await fetchInstanceIdentity(secretKey); + assertInstanceFlagMatches(options, source.keySource, { instanceId, env }); + + return { + secretKey, + target: { + env, + instanceId, + instanceType: detectInstanceType(secretKey), + ...source, + }, + }; +} + +/** One line naming a target, for prose: `My App (development, ins_123)`. */ +export function describeTarget(target: RunTarget): string { + if (target.platform && !target.instanceId) return target.platform; + const where = [target.env, target.instanceId].filter(Boolean).join(", "); + const name = target.appLabel ?? (target.platform ? `${target.platform}` : "instance"); + return where ? `${name} (${where})` : name; +} + +/** + * The header every command prints first: which instance it acts on, and where + * the key came from. An export names its source platform instead, plus the + * Clerk instance when that is what it reads. + * + * `--json` carries the same facts as `target`, so this is for humans only. + */ +export function printTarget(target: RunTarget): void { + const heading = target.platform ? "Source" : "Target"; + const instance = target.instanceId + ? `${target.env ?? "unknown"} instance ${target.instanceId}` + : undefined; + const app = target.appLabel + ? `${target.appLabel}${target.appId ? ` (${target.appId})` : ""}` + : undefined; + const platform = target.platform && target.platform !== "clerk" ? target.platform : undefined; + const parts = [platform ?? (target.platform === "clerk" ? "Clerk" : undefined), app, instance]; + log.info(`${heading}: ${parts.filter(Boolean).join(", ")}`); + if (target.keySource) { + // Agents read this without `--json` too; an escape code is noise to them. + const line = `Key from: ${target.keySource}`; + log.info(isHuman() ? dim(line) : line); + } +} diff --git a/packages/cli-core/src/commands/migrate/lib/transform.test.ts b/packages/cli-core/src/commands/migrate/lib/transform.test.ts new file mode 100644 index 000000000..669d03d79 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.test.ts @@ -0,0 +1,349 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import clerkSource from "../sources/clerk.ts"; +import { __resetCustomSourcesForTesting, registerCustomSource } from "../sources/registry.ts"; +import type { SourceEntry } from "../types.ts"; +import { + consolidateClerkIdentifiers, + flattenObjectSelectively, + getFileType, + loadUsersFromFile, + normalizeUserData, + transformKeys, + transformUsers, + validatePreparedUsers, +} from "./transform.ts"; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-transform-")); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("getFileType", () => { + test.each([ + ["users.json", "application/json"], + ["users.CSV", "text/csv"], + ["users.ndjson", "application/json"], + ["users.jsonl", "application/json"], + ["users.txt", undefined], + ["users", undefined], + ])("%s -> %p", (file, expected) => { + expect(getFileType(file)).toBe(expected as never); + }); +}); + +describe("flattenObjectSelectively", () => { + test("flattens only paths the transformer references", () => { + const result = flattenObjectSelectively( + { _id: { $oid: "123" }, meta: { keep: "nested" }, email: "a@example.com" }, + { "_id.$oid": "userId", email: "email" }, + ); + expect(result).toEqual({ + "_id.$oid": "123", + meta: { keep: "nested" }, + email: "a@example.com", + }); + }); + + test("leaves arrays intact", () => { + expect(flattenObjectSelectively({ tags: [{ a: 1 }] }, { "tags.a": "x" })).toEqual({ + tags: [{ a: 1 }], + }); + }); +}); + +describe("transformKeys", () => { + test("renames mapped fields and passes unmapped ones through", () => { + expect( + transformKeys( + { id: "u1", primary_email_address: "a@example.com", extra: "kept" }, + clerkSource, + ), + ).toEqual({ userId: "u1", email: "a@example.com", extra: "kept" }); + }); + + test.each([ + ["empty string", ""], + ["stringified empty object", '"{}"'], + ["null", null], + ])("drops fields whose value is %s", (_label, value) => { + expect(transformKeys({ id: "u1", first_name: value }, clerkSource)).toEqual({ + userId: "u1", + }); + }); +}); + +describe("normalizeUserData", () => { + test.each([ + ["comma-delimited emails", { email: "a@x.dev,b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["pipe-delimited emails", { email: "a@x.dev|b@x.dev" }, { email: ["a@x.dev", "b@x.dev"] }], + ["JSON array string", { email: '["a@x.dev"]' }, { email: ["a@x.dev"] }], + ["string boolean", { banned: "true" }, { banned: true }], + ["numeric boolean", { banned: 1 }, { banned: true }], + ["numeric string limit", { createOrganizationsLimit: "5" }, { createOrganizationsLimit: 5 }], + ["JSON metadata", { publicMetadata: '{"plan":"pro"}' }, { publicMetadata: { plan: "pro" } }], + ["date string", { createdAt: "2024-01-01" }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ["numeric user ID", { userId: 42 }, { userId: "42" }], + ["epoch milliseconds", { createdAt: 1704067200000 }, { createdAt: "2024-01-01T00:00:00.000Z" }], + // As milliseconds this would be January 1970. + ["epoch seconds", { createdAt: 1704067200 }, { createdAt: "2024-01-01T00:00:00.000Z" }], + ])("normalizes %s", (_label, input, expected) => { + expect(normalizeUserData(input)).toMatchObject(expected); + }); + + test("deletes fields that normalize to nothing", () => { + const result = normalizeUserData({ email: " ", publicMetadata: "", createdAt: "" }); + expect("email" in result).toBe(false); + expect("publicMetadata" in result).toBe(false); + expect("createdAt" in result).toBe(false); + }); + + test("leaves an unparseable date as-is for the schema to reject", () => { + expect(normalizeUserData({ createdAt: "yesterday" }).createdAt).toBe("yesterday"); + }); +}); + +describe("consolidateClerkIdentifiers", () => { + test("merges primary and verified emails, deduping", () => { + const user: Record = { + email: "a@x.dev", + emailAddresses: ["a@x.dev", "b@x.dev"], + unverifiedEmailAddresses: ["b@x.dev", "c@x.dev"], + }; + consolidateClerkIdentifiers(user); + expect(user.email).toEqual(["a@x.dev", "b@x.dev"]); + expect(user.emailAddresses).toBeUndefined(); + // b@x.dev is already verified, so it must not reappear as unverified. + expect(user.unverifiedEmailAddresses).toEqual(["c@x.dev"]); + }); + + test("drops the unverified list when every entry is already verified", () => { + const user: Record = { + phone: "+15555550100", + phoneNumbers: ["+15555550101"], + unverifiedPhoneNumbers: ["+15555550101"], + }; + consolidateClerkIdentifiers(user); + expect(user.phone).toEqual(["+15555550100", "+15555550101"]); + expect("unverifiedPhoneNumbers" in user).toBe(false); + }); + + // The Dashboard lists an unverified primary under unverified_email_addresses. + test.each([ + ["email", "unverifiedEmailAddresses", "a@x.dev"], + ["phone", "unverifiedPhoneNumbers", "+15555550100"], + ])("keeps an unverified primary %s unverified", (primaryKey, unverifiedKey, value) => { + const user: Record = { [primaryKey]: value, [unverifiedKey]: [value] }; + consolidateClerkIdentifiers(user); + expect(primaryKey in user).toBe(false); + expect(user[unverifiedKey]).toEqual([value]); + }); +}); + +describe("validatePreparedUsers", () => { + test("keeps valid users and counts the rest", () => { + const result = validatePreparedUsers([ + { userId: "u1", email: "a@x.dev" }, + { userId: "u2" }, + { userId: "u3", username: "carol" }, + ]); + expect(result.users.map((user) => user.userId)).toEqual(["u1", "u3"]); + expect(result.validationFailed).toBe(1); + expect(result.failures).toMatchObject([{ userId: "u2", row: 2 }]); + }); + + // A failed row is never imported, so a bad hasher is an ordinary reject that + // `--allow-partial` can skip, whatever else is wrong with the row. + test("fails only the rows with an unknown password hasher, naming it", () => { + const result = validatePreparedUsers([ + { userId: "u1", email: "a@x.dev", password: "d", passwordHasher: "rot13" }, + { email: "b@x.dev", password: "d", passwordHasher: "rot13" }, + { userId: "u3", email: "c@x.dev" }, + ]); + expect(result.users.map((user) => user.userId)).toEqual(["u3"]); + expect(result.failures).toMatchObject([ + { userId: "u1", path: ["passwordHasher"] }, + { userId: "row-2", row: 2 }, + ]); + expect(result.failures[0]?.error).toStartWith( + 'Unknown password hasher "rot13". Expected one of:', + ); + }); +}); + +describe("transformUsers", () => { + test("maps, consolidates and validates a Clerk export", () => { + const { transformedData, validationFailed } = transformUsers( + [ + { + id: "u1", + primary_email_address: "a@x.dev", + verified_email_addresses: ["a@x.dev", "b@x.dev"], + first_name: "Alice", + }, + ], + "clerk", + ); + expect(validationFailed).toBe(0); + expect(transformedData[0]).toMatchObject({ + userId: "u1", + email: ["a@x.dev", "b@x.dev"], + firstName: "Alice", + }); + }); + + test("skips validation when asked, so analysis passes see every row", () => { + const { transformedData, validationFailed } = transformUsers([{ id: "u1" }], "clerk", { + validate: false, + }); + expect(transformedData).toHaveLength(1); + expect(validationFailed).toBe(0); + }); +}); + +describe("loadUsersFromFile", () => { + test("reads a JSON export", async () => { + fs.writeFileSync( + path.join(workDir, "users.json"), + JSON.stringify([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("users.json", "clerk"); + expect(users).toHaveLength(1); + expect(users[0]?.userId).toBe("u1"); + }); + + test("reads a CSV export, including quoted commas", async () => { + fs.writeFileSync( + path.join(workDir, "users.csv"), + 'id,primary_email_address,verified_email_addresses\nu2,a@x.dev,"a@x.dev,b@x.dev"\n', + ); + const { users } = await loadUsersFromFile("users.csv", "clerk"); + expect(users[0]?.userId).toBe("u2"); + expect(users[0]?.email).toEqual(["a@x.dev", "b@x.dev"]); + }); + + // Excel's "CSV UTF-8" starts with a BOM, which hid the first column. + test("reads a CSV that starts with a BOM", async () => { + fs.writeFileSync(path.join(workDir, "bom.csv"), "\uFEFFid,primary_email_address\nu3,a@x.dev\n"); + const { users } = await loadUsersFromFile("bom.csv", "clerk"); + expect(users[0]?.userId).toBe("u3"); + }); + + test("reads a JSON file that starts with a BOM", async () => { + fs.writeFileSync( + path.join(workDir, "bom.json"), + `\uFEFF${JSON.stringify([{ id: "u4", primary_email_address: "a@x.dev" }])}`, + ); + const { users } = await loadUsersFromFile("bom.json", "clerk"); + expect(users[0]?.userId).toBe("u4"); + }); + + // What Auth0's bulk export job writes, for tenants over 1,000 users. + test.each([["users-bulk.json"], ["users-bulk.ndjson"]])("reads NDJSON from %s", async (file) => { + fs.writeFileSync( + path.join(workDir, file), + '{"id":"u5","primary_email_address":"a@x.dev"}\n{"id":"u6","primary_email_address":"b@x.dev"}\n', + ); + const { users } = await loadUsersFromFile(file, "clerk"); + expect(users.map((user) => user.userId)).toEqual(["u5", "u6"]); + }); + + test("names the file when it is not valid JSON", async () => { + fs.writeFileSync(path.join(workDir, "broken.json"), "[{"); + const error = (await loadUsersFromFile("broken.json", "clerk").catch( + (e: unknown) => e, + )) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.message).toContain("broken.json is not valid JSON"); + }); + + // `#` starts a value, not a comment: dropping the row would lose the user + // and record nothing. + test("keeps a CSV row whose first cell starts with #", async () => { + fs.writeFileSync(path.join(workDir, "hash.csv"), "id,primary_email_address\n#7,a@x.dev\n"); + const { users } = await loadUsersFromFile("hash.csv", "clerk"); + expect(users.map((user) => user.userId)).toEqual(["#7"]); + }); + + test("a custom preTransform's rows win over a CSV file", async () => { + registerCustomSource({ + ...clerkSource, + key: "rows-from-pretransform", + preTransform: (filePath) => ({ + filePath, + data: [{ id: "from-pretransform", primary_email_address: "p@x.dev" }], + }), + }); + try { + fs.writeFileSync( + path.join(workDir, "ignored.csv"), + "id,primary_email_address\nfile,f@x.dev\n", + ); + const { users } = await loadUsersFromFile("ignored.csv", "rows-from-pretransform"); + expect(users.map((user) => user.userId)).toEqual(["from-pretransform"]); + } finally { + __resetCustomSourcesForTesting(); + } + }); + + test("keeps a row's own value over the source's default", async () => { + const source: SourceEntry = clerkSource; + source.defaults = { passwordHasher: "bcrypt" }; + try { + fs.writeFileSync( + path.join(workDir, "hasher.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "argon2id", + }, + ]), + ); + const { users } = await loadUsersFromFile("hasher.json", "clerk"); + expect(users[0]?.passwordHasher).toBe("argon2id"); + } finally { + delete source.defaults; + } + }); + + test("rejects a JSON file that is not an array of users", async () => { + fs.writeFileSync(path.join(workDir, "wrapped.json"), JSON.stringify({ users: [] })); + await expect(loadUsersFromFile("wrapped.json", "clerk")).rejects.toThrow(CliError); + }); + + test("rejects a JSON row that is not a user object, naming the row", async () => { + fs.writeFileSync(path.join(workDir, "null-row.json"), JSON.stringify([{ id: "u1" }, null])); + await expect(loadUsersFromFile("null-row.json", "clerk")).rejects.toThrow( + new CliError("null-row.json: row 2 is not a user object."), + ); + }); + test("reads the users out of an export's envelope", async () => { + const envelope = (users: unknown[]) => + JSON.stringify({ clerkMigrate: 1, source: "clerk", exportedAt: "", runId: "r", users }); + fs.writeFileSync( + path.join(workDir, "envelope.json"), + envelope([{ id: "u1", primary_email_address: "a@x.dev" }]), + ); + const { users } = await loadUsersFromFile("envelope.json", "clerk"); + expect(users.map((user) => user.userId)).toEqual(["u1"]); + + fs.writeFileSync(path.join(workDir, "envelope-null.json"), envelope([null])); + await expect(loadUsersFromFile("envelope-null.json", "clerk")).rejects.toThrow( + new CliError("envelope-null.json: row 1 is not a user object."), + ); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/transform.ts b/packages/cli-core/src/commands/migrate/lib/transform.ts new file mode 100644 index 000000000..c638899a1 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/transform.ts @@ -0,0 +1,518 @@ +/** + * The load → transform → validate pipeline. + * + * Ported from the standalone migration-tool's `src/migrate/functions.ts` and + * the transform helpers in its `src/lib/index.ts`. Two dependencies were + * dropped along the way: `mime-types` (an extension check covers the two + * formats we accept) and the repo-specific `/samples/` path special-case. + */ + +import fs from "node:fs"; +import path from "node:path"; +import csvParser from "csv-parser"; +import { CliError, ERROR_CODE } from "../../../lib/errors.ts"; +import { getSource } from "../sources/registry.ts"; +import { normalizeBooleanField } from "../sources/shared.ts"; +import { type TransformContext, type SourceEntry, type User } from "../types.ts"; +import { userSchema } from "../validator.ts"; +import { isEnvelope, readJsonFile } from "./export-file.ts"; + +export type FileType = "application/json" | "text/csv"; + +/** A user that failed schema validation before any API call was made. */ +export type ValidationFailure = { + /** The source ID, or `row-` when the row has none. */ + userId: string; + row: number; + error: string; + path: (string | number)[]; +}; + +export type TransformOptions = { + /** Set `false` to keep invalid rows, for analysis passes that count fields. */ + validate?: boolean; + /** Per-run values `postTransform` may need. */ + context?: TransformContext; +}; + +/** Resolves an import path against the current working directory. */ +export function resolveImportFilePath(file: string): string { + return path.resolve(process.cwd(), file.trim()); +} + +export function fileExists(file: string): boolean { + return fs.existsSync(resolveImportFilePath(file)); +} + +/** + * Classifies an import file by extension. + * + * @returns The MIME type, or `undefined` for anything that is not JSON or CSV. + */ +export function getFileType(file: string): FileType | undefined { + const ext = path.extname(resolveImportFilePath(file)).toLowerCase(); + if (ext === ".json" || ext === ".ndjson" || ext === ".jsonl") return "application/json"; + if (ext === ".csv") return "text/csv"; + return undefined; +} + +// --- Field mapping --------------------------------------------------------- + +/** + * Flattens only the nested paths a transformer actually references. + * + * Lets a transformer map `"_id.$oid"` onto `userId` without flattening + * (and thereby mangling) metadata objects it does not mention. + */ +export function flattenObjectSelectively( + obj: Record, + transformer: Record, + prefix = "", +): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + const currentPath = prefix ? `${prefix}.${key}` : key; + const hasNestedMapping = Object.keys(transformer).some((mapped) => + mapped.startsWith(`${currentPath}.`), + ); + + if (hasNestedMapping && value && typeof value === "object" && !Array.isArray(value)) { + Object.assign( + result, + flattenObjectSelectively(value as Record, transformer, currentPath), + ); + } else { + result[currentPath] = value; + } + } + + return result; +} + +/** Renames source fields onto Clerk's import schema, dropping empty values. */ +export function transformKeys( + data: Record, + transformerConfig: { transformer: Record }, +): Record { + const transformed: Record = {}; + const { transformer } = transformerConfig; + const flat = flattenObjectSelectively(data, transformer); + + for (const [key, value] of Object.entries(flat)) { + if (value !== "" && value !== '"{}"' && value !== null) { + transformed[transformer[key] ?? key] = value; + } + } + + return transformed; +} + +// --- Value normalization --------------------------------------------------- + +function parseJsonValue(value: string): unknown { + const trimmed = value.trim(); + if (!trimmed) return value; + if (!["[", "{", '"'].includes(trimmed[0] ?? "")) return value; + try { + return JSON.parse(trimmed); + } catch { + return value; + } +} + +function parseDelimitedStrings(field: unknown): string[] { + // Trimmed like a split string, so a JSON array compares on the same terms. + if (Array.isArray(field)) return field.map((value) => String(value).trim()).filter(Boolean); + if (typeof field === "string" && field) { + const parsed = parseJsonValue(field); + if (Array.isArray(parsed)) { + return parsed.map((value) => String(value).trim()).filter(Boolean); + } + return field + .split(/[,|]/) + .map((value) => value.trim()) + .filter(Boolean); + } + return []; +} + +function normalizeStringArrayField(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = parseJsonValue(trimmed); + if (Array.isArray(parsed)) { + return parsed.map((item) => String(item).trim()).filter(Boolean); + } + if (typeof parsed === "string") { + const parsedString = parsed.trim(); + if (parsedString.includes(",") || parsedString.includes("|")) { + return parsedString + .split(/[,|]/) + .map((item) => item.trim()) + .filter(Boolean); + } + return parsedString; + } + return parsed; +} + +function normalizeNumberField(value: unknown): unknown { + if (typeof value === "number") return value; + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const parsed = Number(trimmed); + return Number.isFinite(parsed) ? parsed : value; +} + +function normalizeMetadataField(value: unknown): unknown { + if (value === undefined || value === null || value === "") return undefined; + if (typeof value !== "string") return value; + + const parsed = parseJsonValue(value); + return typeof parsed === "string" ? value : parsed; +} + +function normalizeDateField(value: unknown): unknown { + if (value instanceof Date) return value.toISOString(); + if (typeof value === "number") { + // Epoch seconds, not milliseconds, below 1e11: as milliseconds that is + // before March 1973, which no signup date is. + const date = new Date(value < 1e11 ? value * 1000 : value); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); + } + if (typeof value !== "string") return value; + + const trimmed = value.trim(); + if (!trimmed) return undefined; + + const date = new Date(trimmed); + return Number.isNaN(date.getTime()) ? value : date.toISOString(); +} + +const ARRAY_FIELDS = [ + "email", + "emailAddresses", + "unverifiedEmailAddresses", + "phone", + "phoneNumbers", + "unverifiedPhoneNumbers", + "backupCodes", +] as const; + +const BOOLEAN_FIELDS = [ + "backupCodesEnabled", + "banned", + "bypassClientTrust", + "createOrganizationEnabled", + "deleteSelfEnabled", + "skipLegalChecks", + "skipPasswordChecks", +] as const; + +const METADATA_FIELDS = ["unsafeMetadata", "publicMetadata", "privateMetadata"] as const; + +const DATE_FIELDS = ["createdAt", "legalAcceptedAt"] as const; + +/** + * Coerces CSV's all-strings-everything into the shapes the schema expects. + * + * A field that normalizes to `undefined` is deleted rather than set, so an + * empty CSV column does not look like an explicitly-null value to Clerk. + */ +export function normalizeUserData(user: Record): Record { + const normalized = { ...user }; + // Integer keys (Better Auth, a custom JSON file) are still IDs. + if (typeof normalized.userId === "number") normalized.userId = String(normalized.userId); + + const setOrDelete = (field: string, value: unknown) => { + if (value === undefined) delete normalized[field]; + else normalized[field] = value; + }; + + for (const field of ARRAY_FIELDS) { + setOrDelete(field, normalizeStringArrayField(normalized[field])); + } + for (const field of BOOLEAN_FIELDS) { + normalized[field] = normalizeBooleanField(normalized[field]); + } + for (const field of METADATA_FIELDS) { + setOrDelete(field, normalizeMetadataField(normalized[field])); + } + for (const field of DATE_FIELDS) { + setOrDelete(field, normalizeDateField(normalized[field])); + } + setOrDelete( + "createOrganizationsLimit", + normalizeNumberField(normalized.createOrganizationsLimit), + ); + + return normalized; +} + +/** + * Merges a Clerk export's three email fields (and three phone fields) into the + * verified/unverified pair the schema models, deduping across all of them. + */ +export function consolidateClerkIdentifiers(user: Record): void { + const merge = (primaryKey: string, verifiedKey: string, unverifiedKey: string) => { + // Read like the lists, so the two compare on the same terms. The schema + // takes an array here too: its first entry is the primary, the rest verified. + const [primary, ...morePrimary] = parseDelimitedStrings(user[primaryKey]); + const verified = [...morePrimary, ...parseDelimitedStrings(user[verifiedKey])]; + const unverified = parseDelimitedStrings(user[unverifiedKey]); + + // The Dashboard lists an unverified primary under the unverified field. + // Leading the verified list would put it on POST /v1/users, verified. + const all: string[] = []; + if (primary && !unverified.includes(primary)) all.push(primary); + for (const value of verified) { + if (!all.includes(value)) all.push(value); + } + if (all.length > 0) user[primaryKey] = all; + else delete user[primaryKey]; + delete user[verifiedKey]; + + const extraUnverified = unverified.filter((value) => !all.includes(value)); + if (extraUnverified.length > 0) user[unverifiedKey] = extraUnverified; + else delete user[unverifiedKey]; + }; + + merge("email", "emailAddresses", "unverifiedEmailAddresses"); + merge("phone", "phoneNumbers", "unverifiedPhoneNumbers"); +} + +// --- Validation ------------------------------------------------------------ + +/** Every field the import schema declares; anything else is stripped. */ +const SCHEMA_FIELDS: ReadonlySet = new Set(Object.keys(userSchema.shape)); + +/** + * Validates prepared users, dropping each failure from the run and returning + * it for the caller to record. + */ +export function validatePreparedUsers(users: Record[]): { + users: User[]; + validationFailed: number; + failures: ValidationFailure[]; + /** Fields a source produced that Clerk has no place for → how many users carry each. */ + unknownFields: Record; +} { + const validated: User[] = []; + const failures: ValidationFailure[] = []; + const unknownFields: Record = {}; + let validationFailed = 0; + + for (let i = 0; i < users.length; i++) { + const user = users[i] as Record; + for (const [field, value] of Object.entries(user)) { + if (!SCHEMA_FIELDS.has(field) && value !== undefined && value !== null && value !== "") { + unknownFields[field] = (unknownFields[field] ?? 0) + 1; + } + } + const result = userSchema.safeParse(user); + + if (result.success) { + validated.push(result.data); + continue; + } + + validationFailed++; + const firstIssue = result.error.issues[0]; + if (!firstIssue) continue; + + failures.push({ + error: firstIssue.message, + path: firstIssue.path as (string | number)[], + // 1-based, like `assertUserRows`: the second user is row 2. + userId: (user.userId as string) || `row-${i + 1}`, + row: i + 1, + }); + } + + return { users: validated, validationFailed, failures, unknownFields }; +} + +function addDefaultFields( + users: Record[], + transformer: SourceEntry, +): Record[] { + if (!transformer.defaults) return users; + // Defaults go first: `transformKeys` maps keys in order, so a later default + // `passwordHasher` would overwrite the row's own `password_hasher`. + return users.map((user) => ({ ...transformer.defaults, ...user })); +} + +/** + * Maps, normalizes and (unless disabled) validates a batch of raw users. + * + * @param options.validate - Set `false` to get the mapped shape without + * dropping invalid rows, for analysis passes that count fields. + * @param options.context - Per-run values `postTransform` may need, e.g. + * Firebase's hash parameters. + */ +export function transformUsers( + users: Record[], + key: string, + options: TransformOptions = {}, +): { + transformedData: User[]; + validationFailed: number; + failures: ValidationFailure[]; + unknownFields: Record; +} { + const transformer = getSource(key); + const context = options.context ?? {}; + const transformed: Record[] = []; + + for (const user of users) { + const mapped = transformKeys(user, transformer); + + // The source's own cleanup first: the Clerk Dashboard's formula-safety TAB + // has to come off before identifiers are compared, or "\t+1555…" and + // "+1555…" read as two phones. + transformer.postTransform?.(mapped, context); + if (key === "clerk") { + consolidateClerkIdentifiers(mapped); + } + + transformed.push(normalizeUserData(mapped)); + } + + if (options.validate === false) { + return { + transformedData: transformed as User[], + validationFailed: 0, + failures: [], + unknownFields: {}, + }; + } + + const result = validatePreparedUsers(transformed); + return { + transformedData: result.users, + validationFailed: result.validationFailed, + failures: result.failures, + unknownFields: result.unknownFields, + }; +} + +// --- File loading ---------------------------------------------------------- + +/** + * Every row of a CSV. `#` is not a comment: a row whose first cell starts + * with one is a row. + * + * @param headers - Column names, for a CSV with no header row. + */ +async function readCsv(filePath: string, headers?: string[]): Promise[]> { + return new Promise((resolve, reject) => { + const users: Record[] = []; + fs.createReadStream(filePath) + // Excel's "CSV UTF-8" starts with a BOM, which would otherwise become + // part of the first header and hide that column from every row. + .pipe( + csvParser({ + ...(headers ? { headers } : {}), + mapHeaders: ({ header }) => header.replace(/^\uFEFF/, ""), + }), + ) + .on("data", (row: Record) => users.push(row)) + .on("error", reject) + .on("end", () => resolve(users)); + }); +} + +async function readUsersFromFile( + file: string, + transformer: SourceEntry, +): Promise[]> { + let filePath = resolveImportFilePath(file); + const type = getFileType(file); + let preExtracted: Record[] | undefined; + let csvHeaders: string[] | undefined; + + // An export's envelope already holds the users in the source's own shape, + // so there is nothing left for a pre-transform to unwrap. + if (type === "application/json") { + const parsed = readJsonFile(filePath); + if (isEnvelope(parsed)) return assertUserRows(parsed.users, file); + if (!transformer.preTransform) return assertUserRows(parsed, file); + } + + if (transformer.preTransform) { + const result = await transformer.preTransform(filePath, type ?? ""); + filePath = result.filePath; + preExtracted = result.data; + csvHeaders = result.csvHeaders; + } + + // A pre-transform's rows win over the file, CSV or not. + if (preExtracted) return assertUserRows(preExtracted, file); + if (type === "text/csv") return readCsv(filePath, csvHeaders); + + return assertUserRows(readJsonFile(filePath), file); +} + +/** A JSON export's rows, refusing anything but an array of objects. */ +function assertUserRows(parsed: unknown, file: string): Record[] { + if (!Array.isArray(parsed)) { + throw new CliError(`Expected ${file} to contain a JSON array of users, got ${typeof parsed}.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } + const bad = parsed.findIndex((row) => !row || typeof row !== "object" || Array.isArray(row)); + if (bad !== -1) { + throw new CliError(`${file}: row ${bad + 1} is not a user object.`, { + code: ERROR_CODE.INVALID_JSON, + }); + } + return parsed as Record[]; +} + +/** + * Reads the export exactly as the transformer sees it, before any field + * mapping. + * + * Used by the Supabase provider cross-reference, which reads + * `raw_app_meta_data` — a column no transformer maps, so it is gone by the time + * users are transformed. + */ +export async function readRawUsers(file: string, key: string): Promise[]> { + return readUsersFromFile(file, getSource(key)); +} + +/** + * Reads a JSON or CSV export and returns the users ready to import. + * + * @param options - Passed through to {@link transformUsers}. + */ +export async function loadUsersFromFile( + file: string, + key: string, + options: TransformOptions = {}, +): Promise<{ + users: User[]; + validationFailed: number; + failures: ValidationFailure[]; + unknownFields: Record; +}> { + const transformer = getSource(key); + const raw = await readUsersFromFile(file, transformer); + const withDefaults = addDefaultFields(raw, transformer); + const { transformedData, validationFailed, failures, unknownFields } = transformUsers( + withDefaults, + key, + options, + ); + return { users: transformedData, validationFailed, failures, unknownFields }; +} diff --git a/packages/cli-core/src/commands/migrate/lib/user-lookup.test.ts b/packages/cli-core/src/commands/migrate/lib/user-lookup.test.ts new file mode 100644 index 000000000..d7c4aed0a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/user-lookup.test.ts @@ -0,0 +1,39 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createApiScheduler } from "./scheduler.ts"; +import { LOOKUP_BATCH, lookupUsers } from "./user-lookup.ts"; + +let originalFetch: typeof globalThis.fetch; + +beforeEach(() => { + originalFetch = globalThis.fetch; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; +}); + +// The instance is over its limit for every lookup, not just the one that hit it. +test("holds every other lookup through a 429's wait", async () => { + const started = performance.now(); + const sentAt: number[] = []; + globalThis.fetch = (async () => { + sentAt.push(performance.now() - started); + return sentAt.length === 1 + ? new Response(JSON.stringify({ errors: [{ code: "x", message: "slow down" }] }), { + status: 429, + headers: { "retry-after": "1" }, + }) + : Response.json([]); + }) as unknown as typeof fetch; + + await lookupUsers({ + filter: "external_id", + // Two batches, so a second lookup is waiting when the first hits the limit. + values: Array.from({ length: LOOKUP_BATCH + 1 }, (_, index) => `u${index}`), + secretKey: "sk_test_x", + schedule: createApiScheduler(1, 10_000), + }); + + expect(sentAt).toHaveLength(3); + for (const at of sentAt.slice(1)) expect(at).toBeGreaterThanOrEqual(900); +}); diff --git a/packages/cli-core/src/commands/migrate/lib/user-lookup.ts b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts new file mode 100644 index 000000000..0ac9980a6 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/lib/user-lookup.ts @@ -0,0 +1,124 @@ +/** + * Batched `GET /v1/users` lookups by a list of values. + * + * BAPI filters on up to 100 values per call and leaves out any it does not + * find, so the work is proportional to the list rather than to the instance. + * Every call goes through the run's scheduler and backs off on a 429, exactly + * as the writes do. + */ + +import { bapiRequest } from "../../../lib/bapi.ts"; +import type { SpinnerControls } from "../../../lib/spinner.ts"; +import { retryOn429 } from "./retry.ts"; +import type { ApiScheduler } from "./scheduler.ts"; + +/** BAPI accepts at most 100 values per filter on `GET /v1/users`. */ +export const LOOKUP_BATCH = 100; + +/** The fields of a BAPI user these lookups read. */ +export type LookedUpUser = { + id: string; + external_id?: string | null; + username?: string | null; + private_metadata?: Record | null; + last_sign_in_at?: number | null; + email_addresses?: { email_address?: string }[]; + phone_numbers?: { phone_number?: string }[]; +}; + +export type LookupFilter = + | "user_id" + | "external_id" + | "email_address" + | "phone_number" + | "username"; + +/** Splits `items` into chunks of at most `size`. */ +export function batch(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) batches.push(items.slice(i, i + size)); + return batches; +} + +/** Every user matching any of `values` on `filter`. */ +export async function lookupUsers(options: { + filter: LookupFilter; + values: string[]; + secretKey: string; + schedule: ApiScheduler; + spinner?: SpinnerControls; + label?: string; +}): Promise { + const batches = batch([...new Set(options.values)], LOOKUP_BATCH); + let done = 0; + + const pages = await Promise.all( + batches.map(async (values) => { + const params = new URLSearchParams(); + params.set("limit", String(LOOKUP_BATCH)); + // BAPI reads a leading `-` on an `external_id` as "exclude" and strips a + // leading `+`, so an explicit `+` keeps a source ID like `-abc` literal. + for (const value of values) { + params.append(options.filter, options.filter === "external_id" ? `+${value}` : value); + } + + // A 429 pauses every lookup still queued, as on import. + const response = await retryOn429( + async () => + options.schedule(async () => + bapiRequest({ + method: "GET", + path: `/v1/users?${params.toString()}`, + secretKey: options.secretKey, + }), + ), + { onRetry: ({ delaySeconds }) => options.schedule.pause(delaySeconds * 1000) }, + ); + done++; + options.spinner?.update(`${options.label ?? "Looking up users"}: ${done}/${batches.length}`); + const users = response.body; + return Array.isArray(users) ? (users as LookedUpUser[]) : []; + }), + ); + + return pages.flat().filter((user) => typeof user.id === "string"); +} + +/** + * The `private_metadata` key every import create carries: the ID of the run + * that sent it. It is how a continue or an undo knows a user found by + * `external_id` is that run's own create, and not one an app or another tool + * made with the same source ID. + */ +export const RUN_MARKER_KEY = "clerkMigrateRun"; + +/** + * The users behind creates that were in flight when run `runId` stopped: + * found by `external_id`, and counted only when they carry the run's marker. + * One without it stays unresolved: a continue's checks find it in the + * instance, and `undo` leaves it alone. That includes a create from a run + * recorded before the marker existed. + */ +export async function findInFlight(options: { + runId: string; + sourceIds: string[]; + secretKey: string; + schedule: ApiScheduler; +}): Promise<{ sourceId: string; clerkId: string }[]> { + if (options.sourceIds.length === 0) return []; + const found = await lookupUsers({ + filter: "external_id", + values: options.sourceIds, + secretKey: options.secretKey, + schedule: options.schedule, + }); + const wanted = new Set(options.sourceIds); + return found + .filter( + (user) => + user.external_id && + wanted.has(user.external_id) && + user.private_metadata?.[RUN_MARKER_KEY] === options.runId, + ) + .map((user) => ({ sourceId: user.external_id as string, clerkId: user.id })); +} diff --git a/packages/cli-core/src/commands/migrate/readme.test.ts b/packages/cli-core/src/commands/migrate/readme.test.ts new file mode 100644 index 000000000..b64b5a1fb --- /dev/null +++ b/packages/cli-core/src/commands/migrate/readme.test.ts @@ -0,0 +1,178 @@ +/** + * Keeps README.md and the command tree honest about each other. + * + * This README documents six commands, seven export platforms, seven sources + * and the run store across ~1000 lines. Checking it by eye at review time does not + * scale, and a doc that names a flag the binary rejects is worse than no doc: + * the reader trusts it and gets a usage error. + * + * Both directions are checked — every example must resolve, and every flag must + * be written down — so neither renaming a flag nor adding one passes silently. + */ + +import { describe, expect, test } from "bun:test"; +import type { Command } from "commander"; +import { createProgram } from "../../cli-program.ts"; + +const README = await Bun.file(new URL("./README.md", import.meta.url)).text(); + +/** Fenced blocks only, so prose that merely mentions a flag is not parsed. */ +function fencedBlocks(markdown: string): string[] { + return [...markdown.matchAll(/^```[a-z]*\n([\s\S]*?)^```/gm)].map((match) => match[1] ?? ""); +} + +/** + * Every `clerk migrate …` invocation the README puts in front of a reader, from + * fenced blocks and inline backticks alike — both get copied. + */ +function documentedCommands(markdown: string): string[] { + const found = new Set(); + + for (const block of fencedBlocks(markdown)) { + // Line continuations first: the Firebase example spans three lines. + for (const line of block.replace(/\\\n\s*/g, " ").split("\n")) { + const start = line.indexOf("clerk migrate"); + // A command never contains a backtick, a `#`, or a run of two spaces; + // the sample error output that quotes `clerk migrate` mid-sentence does, + // and so does a pasted listing whose descriptions sit in a padded column. + if (start !== -1) + found.add( + line + .slice(start) + .split(/[`#]|\s{2,}/)[0]! + .trim(), + ); + } + } + + for (const match of markdown.matchAll(/`(clerk migrate[^`]*)`/g)) { + found.add(match[1]!.trim()); + } + + return [...found]; +} + +/** Walks as deep as the tree allows; the first flag or positional stops it. */ +function resolve(tokens: string[]): { command: Command; rest: string[] } { + let command = createProgram() as Command; + let index = 0; + for (; index < tokens.length; index++) { + const child = command.commands.find( + (candidate) => + candidate.name() === tokens[index] || candidate.aliases().includes(tokens[index]!), + ); + if (!child) break; + command = child; + } + return { command, rest: tokens.slice(index) }; +} + +/** + * The flags a command accepts, including those of a default subcommand. + * + * A group that registers a subcommand `isDefault` hands it everything after + * the group name, so this has to see the same flags Commander does or every + * such example reads as unsupported. + */ +function flagsOf(command: Command): string[] { + const own = command.options.flatMap( + (option) => [option.short, option.long].filter(Boolean) as string[], + ); + + const defaultChild = command.commands.find( + // Commander records the default subcommand on the parent, not the child. + (child) => child.name() === (command as any)._defaultCommandName, + ); + + return defaultChild ? [...own, ...flagsOf(defaultChild)] : own; +} + +/** Every command under `migrate`, so no subcommand escapes the flag sweep. */ +function migrateTree(): { path: string; command: Command }[] { + const collected: { path: string; command: Command }[] = []; + const visit = (command: Command, path: string) => { + collected.push({ path, command }); + for (const child of command.commands) { + if (child.name() !== "help") visit(child, `${path} ${child.name()}`); + } + }; + visit(resolve(["migrate"]).command, "migrate"); + return collected; +} + +const EXAMPLES = documentedCommands(README); + +/** + * The words of an example, minus grammar: a synopsis writes `` and + * `[--json]` to say what goes there, and those are not values to resolve. + */ +function words(example: string): string[] { + return example + .split(/\s+/) + .slice(1) + .filter((token) => !/^[<[]/.test(token) && !/[>\]]$/.test(token)); +} + +/** One case per (example, flag) pair, so a failure names the exact flag. */ +const FLAG_USES: [string, string][] = EXAMPLES.flatMap((example) => + words(example) + // Every command accepts `--help`; Commander does not list it as an option. + .filter((token) => token.startsWith("-") && token !== "--help") + .map((token) => [example, token.split("=")[0]!] as [string, string]), +); + +describe("migrate README", () => { + // Guards the extractor: a regex that silently matched nothing would make + // every check below pass vacuously. + test("finds the documented examples", () => { + expect(EXAMPLES.length).toBeGreaterThan(10); + expect(FLAG_USES.length).toBeGreaterThan(10); + }); + + test.each(EXAMPLES)("`%s` resolves to a real command", (example) => { + // `help` is Commander's own subcommand on every group, not a registered one. + const tokens = words(example).filter((token) => token !== "help"); + const { command, rest } = resolve(tokens); + const firstFlag = rest.findIndex((token) => token.startsWith("-")); + const positionals = firstFlag === -1 ? rest : rest.slice(0, firstFlag); + // Leftover words before any flag are positionals — only some commands take + // them, and a subcommand that does not exist lands here too. + if (positionals.length > 0) expect(command.registeredArguments.length).toBeGreaterThan(0); + // A parent means at least `migrate` resolved. Checking the name instead + // would be wrong: `migrate export clerk` is itself named `clerk`. + expect(command.parent).not.toBeNull(); + }); + + test.each(FLAG_USES)("`%s` uses %s, which the command accepts", (example, flag) => { + const { command } = resolve(words(example)); + expect(flagsOf(command)).toContain(flag); + }); + + // The README is organised around the commands, so each one gets a heading. + test.each(resolve(["migrate"]).command.commands.map((child) => child.name()))( + "has a section for `clerk migrate %s`", + (name) => { + expect(README).toContain(`### \`clerk migrate ${name}\``); + }, + ); + + test("states the rules every command follows", () => { + expect(README).toContain("## The rules"); + for (const rule of [ + "Nothing writes without consent", + "`--dry-run` checks against the real instance", + "State lives in one place", + "Every command prints its target first", + "Every subcommand takes `--json`", + ]) { + expect(README).toContain(rule); + } + }); + + test.each(migrateTree())("$path documents every flag it accepts", ({ command }) => { + const undocumented = flagsOf(command).filter( + (flag) => flag.startsWith("--") && flag !== "--help" && !README.includes(flag), + ); + expect(undocumented).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts new file mode 100644 index 000000000..672b03e75 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-export-recheck.test.ts @@ -0,0 +1,108 @@ +/** + * An export run's file checked again after the import's last read of it. + * + * Its own file because `mock.module` lasts for the file: this one swaps in a + * provider read that overwrites the export, as a concurrent export to a shared + * `--output` path would between the user load and the provider read. + */ + +import { afterAll, beforeAll, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; +import * as providers from "./lib/supabase-providers.ts"; + +let exportFile = ""; + +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); +mock.module("./lib/supabase-providers.ts", () => ({ + ...providers, + readSupabaseRows: async () => { + const envelope = JSON.parse(fs.readFileSync(exportFile, "utf-8")); + fs.writeFileSync(exportFile, JSON.stringify({ ...envelope, users: [] })); + return []; + }, +})); + +const { _setConfigDir } = await import("../../lib/config.ts"); +const { sha256File, startRun } = await import("./lib/run-store.ts"); +const { run } = await import("./run.ts"); + +useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +const originalFetch = globalThis.fetch; +const creates: string[] = []; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-recheck-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-recheck-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + if (url.pathname === "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/v1/users" && init?.method === "POST") { + creates.push(url.pathname); + return Response.json({ id: "user_created" }); + } + if (url.pathname === "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/v1/users") return Response.json([]); + return new Response("unavailable", { status: 503 }); + }) as typeof fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +test("refuses an export overwritten between the user load and the provider read", async () => { + const exportRun = startRun(path.join(workDir, ".clerk", "migrate"), { + kind: "export", + target: { platform: "supabase" }, + source: "supabase", + }); + exportFile = path.join(exportRun.dir, "export.json"); + fs.writeFileSync( + exportFile, + JSON.stringify({ + clerkMigrate: 1, + source: "supabase", + exportedAt: "2026-09-01T00:00:00.000Z", + runId: exportRun.record.id, + users: [{ id: "s1", email: "a@x.dev", email_confirmed_at: "2024-01-01" }], + }), + ); + exportRun.update({ file: { path: exportFile, sha256: sha256File(exportFile) } }); + exportRun.finish(); + + await expect( + run({ input: exportRun.record.id, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/has changed since it was exported/); + expect(creates).toEqual([]); +}); + +// A path import reads the envelope and the users separately, so a file +// rewritten between them would mix two revisions. +test("refuses a path import whose file is rewritten mid-import", async () => { + exportFile = path.join(workDir, "supabase-users.json"); + fs.writeFileSync( + exportFile, + JSON.stringify([{ id: "s2", email: "b@x.dev", email_confirmed_at: "2024-01-01" }]), + ); + + await expect( + run({ input: exportFile, source: "supabase", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/changed while it was being imported/); + expect(creates).toEqual([]); +}); diff --git a/packages/cli-core/src/commands/migrate/run-interactive.test.ts b/packages/cli-core/src/commands/migrate/run-interactive.test.ts new file mode 100644 index 000000000..c2bf8d1d5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run-interactive.test.ts @@ -0,0 +1,338 @@ +/** + * The human-mode half of `migrate import`: the prompts fill in what was not + * passed, and nothing is written until the operator says yes. + * + * Kept in its own file because `mock.module` replaces `prompts.ts` for the whole + * file; `bun test --parallel` isolates each file, so the mock ends with it. + * Human mode is set through `CLERK_MODE` rather than `setMode`, because + * `mode.ts` has no way to clear a forced mode once the file is done. + */ + +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +const mockSelect = mock(async () => "clerk" as unknown); +const mockText = mock(async () => "export.json" as unknown); +let confirmAnswer = true; +/** The answer to the reserved-identifiers question, apart from consent. */ +let reserveAnswer = true; +/** User settings the instance serves, or none (the settings read fails). */ +let instanceSettings: Record | undefined; +/** Every confirmation the run put up, in order — the wording is the assertion. */ +let confirmMessages: string[] = []; +let originalMode: string | undefined; + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (...args: unknown[]) => mockSelect(...(args as [])), +})); + +// Every export of the real module must appear here — a missing one is a link +// error at import time, which takes down the whole file rather than one prompt. +mock.module("../../lib/prompts.ts", () => ({ + confirm: async ({ message }: { message: string }) => { + confirmMessages.push(message); + return message.includes("never verified") ? reserveAnswer : confirmAnswer; + }, + multiselect: async () => [], + text: (...args: unknown[]) => mockText(...(args as [])), + password: async () => "", + editor: async () => "{}", +})); + +const { run } = await import("./run.ts"); +const { UserAbortError } = await import("../../lib/errors.ts"); +const { _setConfigDir } = await import("../../lib/config.ts"); +const { listRuns, sha256File, startRun } = await import("./lib/run-store.ts"); + +const captured = useCaptureLog(); + +let workDir: string; +let configDir: string; +let originalCwd: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string; body: unknown }[]; + +const EXPORT = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev" }, +]; + +const runsDir = () => path.join(workDir, ".clerk", "migrate"); + +beforeAll(() => { + originalMode = process.env.CLERK_MODE; + process.env.CLERK_MODE = "human"; + originalCwd = process.cwd(); + originalFetch = globalThis.fetch; + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-interactive-config-")); + _setConfigDir(configDir); + process.chdir(workDir); +}); + +afterAll(() => { + if (originalMode === undefined) delete process.env.CLERK_MODE; + else process.env.CLERK_MODE = originalMode; + globalThis.fetch = originalFetch; + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + requests = []; + confirmAnswer = true; + reserveAnswer = true; + instanceSettings = undefined; + confirmMessages = []; + mockSelect.mockReset(); + mockText.mockReset(); + mockSelect.mockResolvedValue("clerk"); + mockText.mockResolvedValue("export.json"); + fs.rmSync(path.join(workDir, ".clerk"), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(EXPORT)); + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + requests.push({ + method, + url: url.toString(), + body: init?.body ? JSON.parse(init.body as string) : null, + }); + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "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/v1/users" && method === "GET") return Response.json([]); + if (url.pathname === "/v1/users/count") return Response.json({ total_count: 0 }); + if (url.pathname === "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/v1/domains") { + if (!instanceSettings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.pathname.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.pathname.includes("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/v1/environment")) { + return Response.json({ user_settings: instanceSettings }); + } + return Response.json({ id: "user_created" }); + }) as unknown as typeof fetch; +}); + +afterEach(() => { + process.exitCode = 0; +}); + +const created = () => + requests.filter((r) => r.method === "POST" && new URL(r.url).pathname === "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/v1/users"); + +const importOptions = { input: "export.json", source: "clerk", secretKey: "sk_test_x" }; + +describe("prompts for what was not passed", () => { + test("bare `clerk migrate import` asks for the file, then the source, then imports", async () => { + await run({ secretKey: "sk_test_x" }); + + expect(mockText).toHaveBeenCalledTimes(1); + expect(mockSelect).toHaveBeenCalledTimes(1); + expect(created()).toHaveLength(2); + }); + + test("asks only for what was not passed", async () => { + await run(importOptions); + + expect(mockText).not.toHaveBeenCalled(); + expect(mockSelect).not.toHaveBeenCalled(); + }); + + // The envelope names the source, so asking would only invite a wrong answer. + test("does not ask for a source when the file names its own", async () => { + const exportRun = startRun(runsDir(), { + kind: "export", + target: { platform: "clerk" }, + source: "clerk", + }); + const file = path.join(exportRun.dir, "export.json"); + fs.writeFileSync( + file, + JSON.stringify({ + clerkMigrate: 1, + source: "clerk", + exportedAt: "2026-09-01T00:00:00.000Z", + runId: exportRun.record.id, + users: EXPORT, + }), + ); + exportRun.update({ file: { path: file, sha256: sha256File(file) } }); + exportRun.finish(); + + await run({ input: exportRun.record.id, secretKey: "sk_test_x" }); + + expect(mockSelect).not.toHaveBeenCalled(); + expect(created()).toHaveLength(2); + }); +}); + +describe("consent", () => { + test("asks before writing, naming how many users", async () => { + await run(importOptions); + + expect(confirmMessages).toEqual(["Import 2 users?"]); + }); + + test("names the users it would skip", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([...EXPORT, { id: "u3" }])); + await run({ ...importOptions, allowPartial: true }); + + expect(confirmMessages).toEqual(["Create 2 users and skip 1 user?"]); + }); + + test("prints the checks before the question", async () => { + await run(importOptions); + + expect(captured.err).toContain("Checks"); + }); + + test("declining writes nothing to Clerk, and records no run", async () => { + confirmAnswer = false; + + await expect(run(importOptions)).rejects.toThrow(UserAbortError); + + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("--yes does not ask", async () => { + await run({ ...importOptions, yes: true }); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(2); + }); + + // `--json` means nobody reads a prompt, even at a terminal. + test("--json never asks, and without --yes refuses", async () => { + await expect(run({ ...importOptions, json: true })).rejects.toThrow(/Pass --yes to confirm/); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(0); + }); + + test("an unrecognized password hasher is rejected before any request", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + await expect(run(importOptions)).rejects.toThrow(/1 user would be rejected/); + expect(created()).toHaveLength(0); + }); +}); + +describe("unverified identifiers", () => { + const withUnverified = [ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev", unverified_email_addresses: "c@x.dev" }, + ]; + const statuses = () => + created().map((r) => (r.body as Record)?.email_address_identification_status); + + beforeEach(() => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(withUnverified)); + }); + + test("asks whether to reserve them, before consent, and a yes creates them reserved", async () => { + await run(importOptions); + + expect(confirmMessages).toEqual([ + "1 user has an email or phone the source never verified. Create them reserved (usable for sign-in, locked to the user) instead of unverified?", + "Import 2 users?", + ]); + expect(statuses()).toEqual([undefined, ["verified", "reserved"]]); + }); + + test("a no keeps them unverified, attached after the user exists", async () => { + reserveAnswer = false; + + await run(importOptions); + + expect(statuses()).toEqual([undefined, undefined]); + expect(requests.some((r) => new URL(r.url).pathname === "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/v1/email_addresses")).toBe(true); + }); + + // `-y` is consent to write, not a yes to making unconfirmed addresses usable. + test("--yes does not ask, and keeps them unverified", async () => { + await run({ ...importOptions, yes: true }); + + expect(confirmMessages).toEqual([]); + expect(statuses()).toEqual([undefined, undefined]); + }); + + // A yes at the prompt is a choice; the re-run the refusal suggests must + // keep it, or it would reject the users it let through. + test("a yes carries into the command a refusal suggests", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + ...withUnverified, + { + id: "u3", + primary_email_address: "d@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run(importOptions).catch((caught: unknown) => caught)) as { + examples?: { command: string }[]; + }; + + expect(error.examples?.map((example) => example.command)).toEqual([ + "clerk migrate import export.json --source clerk --reserve-unverified --secret-key --allow-partial --yes", + ]); + }); + + test("--reserve-unverified does not ask, and creates them reserved", async () => { + await run({ ...importOptions, reserveUnverified: true }); + + expect(confirmMessages).toEqual(["Import 2 users?"]); + expect(statuses()).toEqual([undefined, ["verified", "reserved"]]); + }); +}); + +describe("legal consent", () => { + beforeEach(() => { + instanceSettings = { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }; + }); + + test("asks, and a yes imports the users without it", async () => { + await run(importOptions); + + expect(confirmMessages[0]).toContain("no legal acceptance on record"); + expect(created()).toHaveLength(2); + }); + + // `-y` is "import without prompting": the checks reject these users instead. + test("--yes does not ask, and the checks reject them", async () => { + await expect(run({ ...importOptions, yes: true })).rejects.toThrow(/2 users would be rejected/); + + expect(confirmMessages).toEqual([]); + expect(created()).toHaveLength(0); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.test.ts b/packages/cli-core/src/commands/migrate/run.test.ts new file mode 100644 index 000000000..f0f0b2d5f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.test.ts @@ -0,0 +1,1797 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { _setConfigDir } from "../../lib/config.ts"; +import { type CliError, EXIT_CODE } from "../../lib/errors.ts"; +import { credentialStoreStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +// Every test below names its own `--secret-key`, which short-circuits the +// signed-in check — except the one that asserts what happens without it. +mock.module("../../lib/credential-store.ts", () => credentialStoreStubs); +import { + latestUserLines, + listRuns, + patchRun, + readRun, + sha256File, + startRun, + importLockFile, +} from "./lib/run-store.ts"; +import { __resetCustomSourcesForTesting } from "./sources/registry.ts"; +import { explainErrors, run, validateRunOptions } from "./run.ts"; +// After run.ts: imported first, signals.ts loads version.ts ahead of its Bun +// macro here, and the file fails to load. +import { _resetInterruptState, abortInFlight, beginInterrupt } from "../../lib/signals.ts"; + +/** A real-shaped bcrypt digest: the checks reject anything that is not. */ +const BCRYPT = "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"; + +let workDir: string; +let configDir: string; +let originalCwd: string; + +/** Where runs land for a project rooted at `workDir`. */ +const runsDir = () => path.join(workDir, ".clerk", "migrate"); + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-"))); + configDir = fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-run-config-")); + _setConfigDir(configDir); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "users.txt"), ""); +}); + +afterAll(() => { + _setConfigDir(undefined); + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); +}); + +describe("validateRunOptions", () => { + test("accepts a source and an existing JSON file", () => { + expect(validateRunOptions({ source: "clerk", file: "users.json" })).toEqual({ + source: "clerk", + file: "users.json", + }); + }); + + test.each([ + ["no source", { file: "users.json" }, /Missing --source/], + ["no file", { source: "clerk" }, /Missing the file to import/], + ["a missing file", { source: "clerk", file: "nope.json" }, /File not found/], + ["an unsupported extension", { source: "clerk", file: "users.txt" }, /Unsupported file type/], + ])("rejects %s", (_label, options, message) => { + expect(() => validateRunOptions(options)).toThrow(message); + }); + + test("names the valid sources when one is missing", () => { + expect(() => validateRunOptions({ file: "users.json" })).toThrow(/clerk/); + }); +}); + +type Stub = { + /** What `/v1/environment` reports; `null` makes the settings unreadable. */ + settings?: { + attributes?: object; + social?: object; + sign_up?: object; + enterprise_sso?: object; + } | null; + /** Users already in the instance, as `GET /v1/users` returns them. */ + existing?: { + id: string; + external_id?: string; + username?: string; + private_metadata?: Record; + email_addresses?: { email_address: string }[]; + }[]; + /** `GET /v1/users/count`. */ + count?: number; + /** Source IDs whose `POST /v1/users` fails with a 422. */ + failing?: Set; + /** `GET /v1/instance` fails, so the target falls back to the key's stand-in ID. */ + instanceDown?: boolean; +}; + +describe("run", () => { + const captured = useCaptureLog(); + let originalFetch: typeof globalThis.fetch; + let requests: { method: string; url: string; body: unknown }[]; + + const export2 = [ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: BCRYPT, + password_hasher: "bcrypt", + }, + { id: "u2", primary_email_address: "b@x.dev" }, + ]; + + /** A fake Clerk: the instance, its settings, its users, and the writes. */ + function stubClerk(stub: Stub = {}): void { + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + const body = init?.body ? (JSON.parse(init.body as string) as Record) : null; + requests.push({ method, url: url.toString(), body }); + + if (url.pathname === "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/v1/instance" && stub.instanceDown) { + return new Response("unavailable", { status: 503 }); + } + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (url.pathname === "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/v1/domains") { + if (!stub.settings) return new Response("nope", { status: 500 }); + return Response.json({ + data: [{ is_satellite: false, frontend_api_url: "https://fapi.example.com" }], + }); + } + if (url.pathname.includes("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/v1/dev_browser")) return Response.json({ token: "jwt" }); + if (url.pathname.includes("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/v1/environment")) { + return Response.json({ user_settings: stub.settings }); + } + if (url.pathname === "/v1/users/count") { + return Response.json({ object: "total_count", total_count: stub.count ?? 0 }); + } + if (method === "GET" && url.pathname === "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/v1/users") { + // BAPI strips the `+` the lookup puts on each external_id. + const wanted = new Set( + [...url.searchParams.values()].map((value) => value.replace(/^\+(?!\d)/, "")), + ); + return Response.json( + (stub.existing ?? []).filter( + (user) => + wanted.has(user.external_id ?? "") || + wanted.has(user.username ?? "") || + (user.email_addresses ?? []).some((email) => wanted.has(email.email_address)), + ), + ); + } + if (method === "POST" && url.pathname === "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/v1/users") { + const externalId = body?.external_id as string; + if (stub.failing?.has(externalId)) { + return Response.json( + { + errors: [{ code: "form_identifier_exists", message: "That email address is taken." }], + }, + { status: 422 }, + ); + } + return Response.json({ id: `user_${externalId}` }); + } + return Response.json({ id: "ok" }); + }) as typeof fetch; + } + + beforeAll(() => { + originalFetch = globalThis.fetch; + }); + + beforeEach(() => { + requests = []; + delete process.env.CLERK_MIGRATE_RATE_LIMIT; + fs.rmSync(runsDir(), { recursive: true, force: true }); + fs.rmSync(path.join(configDir, "config.json"), { force: true }); + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(export2)); + stubClerk(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + }); + + const baseOptions = { + source: "clerk", + input: "export.json", + yes: true, + secretKey: "sk_test_x", + }; + + const created = () => + requests + .filter((r) => r.method === "POST" && r.url.endsWith("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/v1/users")) + .map((r) => (r.body as { external_id: string }).external_id); + + const exitCodeOf = async (promise: Promise) => + ((await promise.catch((caught: unknown) => caught)) as CliError | undefined)?.exitCode; + + test("refuses before anything else when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect(run({ source: "clerk", input: "export.json", yes: true })).rejects.toThrow( + /Not logged in/, + ); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + + // --app needs an account to resolve its key, unlike --secret-key. + test("refuses an --app import when nobody is signed in", async () => { + const previous = process.env.CLERK_SECRET_KEY; + delete process.env.CLERK_SECRET_KEY; + try { + await expect( + run({ source: "clerk", input: "export.json", yes: true, app: "app_123" }), + ).rejects.toThrow(/Not logged in/); + expect(requests).toHaveLength(0); + } finally { + if (previous !== undefined) process.env.CLERK_SECRET_KEY = previous; + } + }); + + // resolveBapiSecretKey takes --app over the exported key, so the sign-in is + // still needed: without it the Platform API call fails instead. + test("refuses an --app import when nobody is signed in, even with CLERK_SECRET_KEY set", async () => { + const previous = process.env.CLERK_SECRET_KEY; + process.env.CLERK_SECRET_KEY = "sk_test_x"; + try { + await expect( + run({ source: "clerk", input: "export.json", yes: true, app: "app_123" }), + ).rejects.toThrow(/Not logged in/); + expect(requests).toHaveLength(0); + } finally { + if (previous === undefined) delete process.env.CLERK_SECRET_KEY; + else process.env.CLERK_SECRET_KEY = previous; + } + }); + + test("imports every user in the file end to end", async () => { + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(captured.err).toContain("Imported:"); + }); + + test("prints the target first", async () => { + await run(baseOptions); + expect(captured.err).toContain("Target: development instance ins_1"); + expect(captured.err.indexOf("Target:")).toBeLessThan(captured.err.indexOf("Checks")); + }); + + test("records the run in the project's run store", async () => { + await run(baseOptions); + + const [record, ...rest] = listRuns(runsDir()); + expect(rest).toHaveLength(0); + expect(record).toMatchObject({ + kind: "import", + status: "complete", + source: "clerk", + counts: { total: 2, created: 2 }, + target: { keySource: "--secret-key", instanceType: "dev", instanceId: "ins_1" }, + }); + expect(record?.file?.path).toBe(path.join(workDir, "export.json")); + expect(record?.file?.sha256).toMatch(/^[0-9a-f]{64}$/); + + const lines = [...latestUserLines(runsDir(), record!.id).values()]; + expect(lines.map((line) => [line.sourceId, line.status, line.clerkId])).toEqual([ + ["u1", "created", "user_u1"], + ["u2", "created", "user_u2"], + ]); + }); + + // A complete import leaves the export's user data behind; say how to remove it. + test("names the folders a complete import no longer needs", async () => { + await run(baseOptions); + const [record] = listRuns(runsDir()); + expect(captured.err).toContain(`rm -rf ${path.join(runsDir(), record!.id)}`); + }); + + // Pasted unquoted, `rm -rf …/app copy/…` deletes `…/app`. + test("quotes the cleanup paths", async () => { + const spaced = path.join(workDir, "app copy"); + await run({ ...baseOptions, runsDir: spaced }); + + const [record] = listRuns(spaced); + expect(captured.err).toContain(`rm -rf '${path.join(spaced, record!.id)}'`); + }); + + describe("export envelopes", () => { + /** An export run whose envelope holds `users`, as `clerk migrate export` writes it. */ + function exportRun(source: string, rows: unknown[], extra: Record = {}) { + const run = startRun(runsDir(), { kind: "export", target: { platform: source }, source }); + const file = path.join(run.dir, "export.json"); + fs.writeFileSync( + file, + JSON.stringify({ + clerkMigrate: 1, + source, + exportedAt: "2026-09-01T00:00:00.000Z", + runId: run.record.id, + users: rows, + ...extra, + }), + ); + run.update({ file: { path: file, sha256: sha256File(file) } }); + return { record: run.finish(), file }; + } + + const { source: _source, input: _input, ...noSource } = baseOptions; + + // The run ID stands for the file that run wrote: a later export or an edit + // at the same path must not import under the old run's name. + test("refuses an export file that changed since its run wrote it", async () => { + const { record, file } = exportRun("clerk", export2); + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, users: [envelope.users[0]] })); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + // A shared --output path can be overwritten after the run-ID check, while + // the import is still naming its target. + test("refuses an export file overwritten after the run ID was checked", async () => { + const { record, file } = exportRun("clerk", export2); + const served = globalThis.fetch; + let overwritten = false; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (!overwritten && new URL(input.toString()).pathname === "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/v1/instance") { + overwritten = true; + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, users: [envelope.users[0]] })); + } + return served(input, init); + }) as typeof fetch; + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + expect(overwritten).toBe(true); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("refuses an export file another run's export overwrote", async () => { + const { record, file } = exportRun("clerk", export2); + const envelope = JSON.parse(fs.readFileSync(file, "utf-8")); + fs.writeFileSync(file, JSON.stringify({ ...envelope, runId: "20260101-000000-abcd" })); + patchRun(runsDir(), record.id, { file: { path: file, sha256: sha256File(file) } }); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /has changed since it was exported/, + ); + }); + + test("refuses an export run whose file is gone", async () => { + const { record, file } = exportRun("clerk", export2); + fs.rmSync(file); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow(/is gone/); + }); + + test("names an --output export file outside the run folder for deletion too", async () => { + const { record, file } = exportRun("clerk", export2); + const outside = path.join(workDir, `users-${record.id}.json`); + fs.renameSync(file, outside); + patchRun(runsDir(), record.id, { file: { path: outside, sha256: sha256File(outside) } }); + + await run({ ...noSource, input: record.id }); + + expect(captured.err).toContain(`rm ${outside}`); + }); + + test("imports by export run ID, with the source the envelope names", async () => { + const { record } = exportRun("clerk", export2); + + await run({ ...noSource, input: record.id }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + const imported = listRuns(runsDir()).find((candidate) => candidate.kind === "import"); + expect(imported).toMatchObject({ source: "clerk", fromExport: record.id }); + }); + + // Started by path, continued by run ID: the run still names its export. + test("a continue by export run ID links the run to the export", async () => { + const { record, file } = exportRun("clerk", export2); + stubClerk({ failing: new Set(["u2"]) }); + await run({ ...noSource, input: file }); + expect(listRuns(runsDir()).find((c) => c.kind === "import")?.fromExport).toBeUndefined(); + + stubClerk(); + process.exitCode = 0; + await run({ ...noSource, input: record.id }); + + const imported = listRuns(runsDir()).filter((c) => c.kind === "import"); + expect(imported).toHaveLength(1); + expect(imported[0]).toMatchObject({ fromExport: record.id }); + expect(captured.err).toContain(`The export in run ${record.id} holds your users' data`); + }); + + test("imports an envelope file with no source named", async () => { + const { file } = exportRun("clerk", export2); + + await run({ ...noSource, input: file }); + + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(2); + }); + + test("refuses a source that contradicts the envelope", async () => { + const { record } = exportRun("clerk", export2); + + await expect(run({ ...noSource, source: "auth0", input: record.id })).rejects.toThrow( + /exported from clerk, but --source names auth0/, + ); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("refuses a run ID that is not an export", async () => { + await run(baseOptions); + const [imported] = listRuns(runsDir()); + + await expect(run({ ...noSource, input: imported!.id })).rejects.toThrow( + /is an import run, which has no file to import/, + ); + }); + + test("refuses a run ID with no run behind it", async () => { + await expect(run({ ...noSource, input: "20260101-000000-abcd" })).rejects.toThrow( + /No run `20260101-000000-abcd`/, + ); + expect(requests).toHaveLength(0); + }); + + test("reads Firebase's hash parameters from the envelope", async () => { + const firebase = { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }; + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", salt: "SALT" }], + { firebase }, + ); + + await run({ ...noSource, input: record.id }); + + const created = requests.find((r) => r.url.endsWith("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/v1/users")); + expect(created?.body).toMatchObject({ + password_hasher: "scrypt_firebase", + password_digest: "HASH$SALT$SIGNER$Bw==$8$14", + }); + }); + + // Each goes into every digest: a bad one would cost every password. + test("refuses Firebase hash parameters Clerk would reject, from the envelope or the flags", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", salt: "SALT" }], + { + firebase: { + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 0, + mem_cost: 14, + }, + }, + ); + + await expect(run({ ...noSource, input: record.id })).rejects.toThrow( + /from the export file won't work: rounds must be a whole number from 1 to 16/, + ); + await expect( + run({ + ...noSource, + input: record.id, + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 17, + }), + ).rejects.toThrow(/from the --firebase-\* flags won't work: memory cost/); + expect(requests.filter((r) => r.url.endsWith("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/v1/users"))).toHaveLength(0); + }); + + test("lets the --firebase-* flags override the envelope", async () => { + const { record } = exportRun( + "firebase", + [{ localId: "f1", email: "f@x.dev", passwordHash: "HASH", salt: "SALT" }], + { + firebase: { + base64_signer_key: "OLD", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }, + }, + ); + + await run({ + ...noSource, + input: record.id, + firebaseSignerKey: "NEW", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const created = requests.find((r) => r.url.endsWith("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/v1/users")); + expect((created!.body as { password_digest: string }).password_digest).toContain("$NEW$"); + }); + }); + + test("gitignores the project's .clerk folder before writing a run", async () => { + await run(baseOptions); + expect(fs.readFileSync(path.join(workDir, ".gitignore"), "utf-8")).toContain(".clerk/"); + }); + + test("leaves .gitignore alone when the import is refused for lack of consent", async () => { + fs.rmSync(path.join(workDir, ".gitignore"), { force: true }); + + await run({ ...baseOptions, yes: false }).catch(() => undefined); + + expect(fs.existsSync(path.join(workDir, ".gitignore"))).toBe(false); + }); + + test("--runs-dir puts the run somewhere else", async () => { + await run({ ...baseOptions, runsDir: "elsewhere" }); + expect(listRuns(path.join(workDir, "elsewhere"))).toHaveLength(1); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("--require-password leaves out the users without one", async () => { + await run({ ...baseOptions, requirePassword: true }); + + expect(created()).toEqual(["u1"]); + expect(captured.err).toContain("leaving out 1 user without a password"); + // On record, so the run is partial rather than "Already imported". + const [record] = listRuns(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")).toMatchObject({ + status: "skipped", + reason: "no password (--require-password)", + }); + expect(record?.status).toBe("partial"); + }); + + // They never reach the checks, so the preview has to count them itself. + test("--require-password counts the users it left out in the --json preview", async () => { + await run({ ...baseOptions, requirePassword: true, dryRun: true, json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + dryRun: true, + withoutPassword: 1, + checks: { total: 1 }, + }); + }); + + test("rejects a user with an unrecognized hasher, naming it", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "u1", + primary_email_address: "a@x.dev", + password_digest: "d", + password_hasher: "rot13", + }, + ]), + ); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + expect(captured.err).toContain('Unknown password hasher "rot13"'); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(created()).toHaveLength(0); + }); + + test("exits 1 when some users failed", async () => { + stubClerk({ failing: new Set(["u2"]) }); + + await run(baseOptions); + + expect(process.exitCode).toBe(1); + expect(listRuns(runsDir())[0]).toMatchObject({ status: "partial", counts: { failed: 1 } }); + }); + + // Tests run non-TTY, the same signal an agent gives. + describe("without a file or a source", () => { + test("names what to pass rather than prompting for the file", async () => { + await expect(run({ source: "clerk", yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import, or the export run that wrote it, and cannot prompt here/, + ); + expect(requests).toHaveLength(0); + }); + + test("asks for --source when the file does not name its own", async () => { + await expect( + run({ input: "export.json", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Missing --source/); + }); + }); + + describe("continuing an earlier run", () => { + test("a complete run is not imported again", async () => { + await run(baseOptions); + requests = []; + + await run(baseOptions); + + expect(created()).toHaveLength(0); + expect(captured.err).toContain("Already imported in run"); + expect(listRuns(runsDir())).toHaveLength(1); + }); + + test("--new-run imports it again as a new run", async () => { + await run(baseOptions); + requests = []; + + await run({ ...baseOptions, newRun: true }); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir())).toHaveLength(2); + }); + + test("a partial run retries only the users that did not make it, in the same run", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + expect(captured.err).toContain(`Continuing run ${first!.id}, which finished partial`); + const runs = listRuns(runsDir()); + expect(runs).toHaveLength(1); + expect(runs[0]).toMatchObject({ id: first!.id, status: "complete", counts: { created: 2 } }); + }); + + test("an interrupted run skips the users it already created", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + // A crash never writes a finish time. + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + expect(captured.err).toContain("which was interrupted"); + }); + + /** Rewrites a finished run as one a crash stopped: no finish time. */ + const interrupt = (id: string, patch: Record = {}) => { + const record = readRun(runsDir(), id)!; + delete record.finishedAt; + fs.writeFileSync( + path.join(runsDir(), id, "run.json"), + JSON.stringify({ ...record, ...patch }), + ); + }; + + // The create went out and the run stopped before the answer came back. + test("an interrupted run adopts a user Clerk created with no ID on record", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ + existing: [ + { id: "user_found", external_id: "u2", private_metadata: { clerkMigrateRun: first!.id } }, + ], + }); + await run(baseOptions); + + expect(created()).toEqual([]); + expect(captured.err).toContain("1 user whose create was cut off is already in the instance"); + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + expect(readRun(runsDir(), first!.id)?.status).toBe("complete"); + }); + + // The `creating` line records what the create did; a continued run reads it + // back rather than guessing from this run's flags. + test("an adopted user whose create reserved its unverified email gets no attach for it", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", primary_email_address: "b@x.dev", unverified_email_addresses: "c@x.dev" }, + ]), + ); + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating", reserved: true })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ + existing: [ + { id: "user_found", external_id: "u2", private_metadata: { clerkMigrateRun: first!.id } }, + ], + }); + await run(baseOptions); + + expect(created()).toEqual([]); + expect(requests.filter((r) => r.url.endsWith("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/v1/email_addresses"))).toEqual([]); + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + }); + + // Created reserved by the run that stopped: its unverified email exists + // reserved, and meets the requirement, though this run has no flag. + test("an adopted user created reserved meets a required email without the flag", async () => { + const settings = { + attributes: { email_address: { enabled: true, required: true } }, + enterprise_sso: { enabled: true }, + }; + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", unverified_email_addresses: "c@x.dev" }, + ]), + ); + stubClerk({ settings, failing: new Set(["u2"]) }); + await run({ ...baseOptions, reserveUnverified: true }); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating", reserved: true })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ + settings, + existing: [ + { id: "user_found", external_id: "u2", private_metadata: { clerkMigrateRun: first!.id } }, + ], + }); + await run(baseOptions); + + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + }); + + // A reject would stop a create; an adopted user needs none. Rejected, it + // stayed `creating` and the run could never finish. + test("an adopted user a check would reject still finishes the run", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ + // Legal consent turned on since: u2 has no acceptance on record. + settings: { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + enterprise_sso: { enabled: true }, + }, + existing: [ + { id: "user_found", external_id: "u2", private_metadata: { clerkMigrateRun: first!.id } }, + ], + }); + await run(baseOptions); + + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "created", + clerkId: "user_found", + }); + expect(readRun(runsDir(), first!.id)?.status).toBe("complete"); + }); + + // Same source ID, but no marker from this run: an app or another tool's + // user, so it is never adopted. + test("an interrupted run does not adopt a match without its marker", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk({ + existing: [{ id: "user_theirs", external_id: "u2", private_metadata: {} }], + }); + await run({ ...baseOptions, allowPartial: true }); + + expect(created()).toEqual([]); + expect(captured.err).not.toContain("whose create was cut off"); + expect(latestUserLines(runsDir(), first!.id).get("u2")).toMatchObject({ + status: "skipped", + reason: "already in the instance, with this source ID", + }); + }); + + test("an interrupted run creates a user whose in-flight create never landed", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + fs.appendFileSync( + path.join(runsDir(), first!.id, "users.ndjson"), + `${JSON.stringify({ sourceId: "u2", status: "creating" })}\n`, + ); + interrupt(first!.id); + + requests = []; + process.exitCode = 0; + stubClerk(); + await run(baseOptions); + + expect(created()).toEqual(["u2"]); + }); + + test("a continued run with nothing left to do is finished", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + interrupt(first!.id); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual([]); + expect(captured.err).toContain("No users left to import"); + expect(readRun(runsDir(), first!.id)).toMatchObject({ status: "complete" }); + expect(readRun(runsDir(), first!.id)?.finishedAt).toBeDefined(); + }); + + // "Interrupted, so undo it and start over": the undo marks the run undone + // but leaves it with no finish time. + test("an interrupted run that was then undone is imported again as a new run", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + interrupt(first!.id, { status: "undone" }); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir()).filter((record) => record.kind === "import")).toHaveLength(2); + }); + + test("a run with an undo that did not finish refuses with exit 2", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + const undoRun = startRun(runsDir(), { + kind: "undo", + target: { instanceId: "ins_1" }, + undoes: first!.id, + }); + undoRun.append({ sourceId: "u1", status: "deleted", clerkId: "user_u1" }); + undoRun.append({ sourceId: "u2", status: "failed", clerkId: "user_u2" }); + undoRun.finish(); + requests = []; + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(`clerk migrate undo ${first!.id}`); + expect(created()).toEqual([]); + }); + + test("an undone run is imported again as a new run", async () => { + await run(baseOptions); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + fs.writeFileSync( + path.join(runsDir(), first!.id, "run.json"), + JSON.stringify({ ...record, status: "undone" }), + ); + requests = []; + + await run(baseOptions); + + expect(created()).toEqual(["u1", "u2"]); + expect(listRuns(runsDir())).toHaveLength(2); + }); + + test("a run another live process holds refuses with exit 2", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + // A new run would send the same external IDs alongside it. + expect(await exitCodeOf(run({ ...baseOptions, newRun: true }))).toBe(EXIT_CODE.USAGE); + }); + + // The scan of runs alone has a gap: a second process can pass it before + // either has written a run. The import's own lock closes it. + test("refuses while another process holds this import's lock, with no run yet", async () => { + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + // PID 1 is always alive, and never this test. + fs.writeFileSync(lock, "1"); + try { + await expect(run(baseOptions)).rejects.toThrow( + /importing this file into this instance right now/, + ); + } finally { + fs.rmSync(lock, { force: true }); + } + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + test("lets go of the import's lock when the run ends", async () => { + await run(baseOptions); + const lock = importLockFile(runsDir(), { + sha256: sha256File(path.join(workDir, "export.json")), + source: "clerk", + instanceId: "ins_1", + }); + expect(fs.existsSync(lock)).toBe(false); + }); + + // An edited file is a different job. + // Rate limiting right after a large import is when this lookup fails, and + // that is when someone re-runs after an interruption. + test("refuses to start over when Clerk can't name the instance a run of this file was in", async () => { + stubClerk({ failing: new Set(["u2"]) }); + await run(baseOptions); + + requests = []; + process.exitCode = 0; + stubClerk({ instanceDown: true }); + await expect(run(baseOptions)).rejects.toThrow(/did not confirm which instance/); + expect(created()).toEqual([]); + expect(listRuns(runsDir())).toHaveLength(1); + + await run({ ...baseOptions, newRun: true, allowPartial: true }); + expect(listRuns(runsDir())).toHaveLength(2); + }); + + test("a changed file is a new run", async () => { + await run(baseOptions); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3", primary_email_address: "c@x.dev" }]), + ); + requests = []; + + await run(baseOptions); + + expect(listRuns(runsDir())).toHaveLength(2); + }); + }); + + describe("checks", () => { + test("a reject stops the import, with the command that imports the rest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("1 user would be rejected, so nothing was imported"); + expect(error.examples?.[0]?.command).toContain("--allow-partial --yes"); + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + }); + + // "will create 0 users" hides that the run records the rest as skipped. + test("--allow-partial without --yes names the users it would skip", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify([{ id: "u3" }])); + + await expect(run({ ...baseOptions, allowPartial: true, yes: false })).rejects.toThrow( + "will create 0 users and skip 1 user. Pass --yes to confirm.", + ); + }); + + test("--allow-partial imports the rest and records each reject as skipped", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, allowPartial: true }); + + expect(created()).toEqual(["u1", "u2"]); + const [record] = listRuns(runsDir()); + expect(record).toMatchObject({ status: "partial", counts: { created: 2, skipped: 1 } }); + expect(latestUserLines(runsDir(), record!.id).get("u3")).toMatchObject({ + status: "skipped", + reason: expect.stringContaining("invalid:"), + }); + // The operator accepted the skips; only a failed user exits 1. + expect(process.exitCode).toBe(0); + }); + + test("--dry-run writes nothing, and exits 2 when the import would be refused", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(listRuns(runsDir())).toHaveLength(0); + expect(captured.err).toContain("Dry run: nothing was written."); + expect(process.exitCode).toBe(2); + }); + + test("--dry-run exits 0 when nothing would be rejected", async () => { + await run({ ...baseOptions, dryRun: true }); + + expect(created()).toHaveLength(0); + expect(process.exitCode).toBe(0); + }); + + test("a user whose only email is unverified is rejected where email is required, with a fix", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true, required: true } }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", unverified_email_addresses: "b@x.dev" }, + ]), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain( + "only has an unverified email, and this instance requires an email", + ); + expect(captured.err).toContain( + `clerk config patch --app APP_ID --instance ins_1 --json '{"auth_email":{"required_for_sign_up":false}}'`, + ); + }); + + // Reserved meets the requirement, so neither the reject nor the fix applies. + test("with --reserve-unverified, an unverified-only user is neither rejected nor a fix", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true, required: true } }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev" }, + { id: "u2", unverified_email_addresses: "b@x.dev" }, + ]), + ); + + await run({ ...baseOptions, dryRun: true, reserveUnverified: true }); + + expect(captured.err).not.toContain("only has an unverified email"); + expect(captured.err).not.toContain("required_for_sign_up"); + expect(process.exitCode).toBe(0); + }); + + test("legal consent: refused without --skip-legal-checks, sent with skip_legal_checks with it", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + sign_up: { legal_consent_enabled: true }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + + const error = (await run(baseOptions).catch((caught: unknown) => caught)) as CliError; + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(captured.err).toContain("no legal acceptance on record"); + expect(created()).toEqual([]); + + await run({ ...baseOptions, skipLegalChecks: true }); + const bodies = requests + .filter((r) => r.method === "POST" && r.url.endsWith("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/v1/users")) + .map((r) => r.body); + expect(bodies).toEqual([ + expect.objectContaining({ external_id: "u1", skip_legal_checks: true }), + expect.objectContaining({ external_id: "u2", skip_legal_checks: true }), + ]); + }); + + test("a user already in the instance is rejected", async () => { + stubClerk({ + existing: [{ id: "user_old", email_addresses: [{ email_address: "b@x.dev" }] }], + }); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain("email is already used by a user in the instance"); + expect(Bun.stripANSI(captured.err)).toContain("u2"); + }); + + test("the dev quota rejects users past the headroom; --allow-partial imports up to it", async () => { + stubClerk({ count: 99 }); + + expect(await exitCodeOf(run(baseOptions))).toBe(EXIT_CODE.USAGE); + expect(created()).toHaveLength(0); + + await run({ ...baseOptions, allowPartial: true }); + expect(created()).toEqual(["u1"]); + const [record] = listRuns(runsDir()); + expect(latestUserLines(runsDir(), record!.id).get("u2")?.reason).toContain("100-user limit"); + }); + + test("supabase users whose only provider is disabled are rejected", async () => { + stubClerk({ + settings: { + attributes: { email_address: { enabled: true } }, + social: { oauth_google: { enabled: true } }, + // A way in besides a password, so users without one import. + enterprise_sso: { enabled: true }, + }, + }); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["discord"]}', + }, + { + id: "sb2", + email: "b@x.dev", + email_confirmed_at: "2024-01-01 00:00:00+00", + raw_app_meta_data: '{"providers":["email","discord"]}', + }, + ]), + ); + + await run({ ...baseOptions, source: "supabase", dryRun: true }); + + expect(captured.err).toContain("only signs in with Discord, which is not enabled in Clerk"); + expect(captured.err).toContain( + "1 user signed in with Discord, which is not enabled in Clerk", + ); + }); + + test("names the fields Clerk won't store", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify(export2.map((user) => ({ ...user, department: "Sales" }))), + ); + + await run({ ...baseOptions, dryRun: true }); + + expect(captured.err).toContain("Clerk won't store: department (2 users)"); + }); + }); + + describe("consent", () => { + test("without --yes where nobody can be asked: the preview, then exit 2 with the command", async () => { + const error = (await run({ ...baseOptions, yes: false }).catch( + (caught: unknown) => caught, + )) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain(". Pass --yes to confirm"); + expect(error.examples?.[0]?.command).toBe( + "clerk migrate import export.json --source clerk --secret-key --yes", + ); + expect(captured.err).toContain("Checks"); + expect(created()).toHaveLength(0); + }); + + test("--json --yes returns { target, run, checks, result }", async () => { + await run({ ...baseOptions, json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + target: { instanceId: "ins_1" }, + run: { kind: "import", status: "complete" }, + checks: { total: 2, importable: 2, rejects: [] }, + result: { created: 2, failed: 0, skipped: 0 }, + }); + }); + + test("--json leaves colour codes out of stderr", async () => { + await run({ ...baseOptions, json: true }); + expect(captured.err).not.toContain("\x1b["); + }); + + test("--json without --yes returns the preview with consent required, and exits 2", async () => { + expect(await exitCodeOf(run({ ...baseOptions, yes: false, json: true }))).toBe( + EXIT_CODE.USAGE, + ); + expect(JSON.parse(captured.out)).toMatchObject({ + consent: "required", + checks: { importable: 2 }, + }); + expect(created()).toHaveLength(0); + }); + }); + + describe("--source ", () => { + const CUSTOM = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email", given: "firstName", pw: "password" }, + carries: { + passwords: { level: "yes", note: "bcrypt." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { if (!user.firstName) delete user.firstName; }, + };`; + + let customFile: string; + let customCounter = 0; + + beforeEach(() => { + // A fresh filename each time: dynamic import() caches by URL, so reusing + // one would silently return a previous test's module. + customFile = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, customFile), CUSTOM); + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { account_ref: "mp_1", contact_email: "a@x.dev", given: "Ada", pw: BCRYPT }, + { account_ref: "mp_2", contact_email: "b@x.dev", given: "", pw: BCRYPT }, + ]), + ); + }); + + afterEach(() => { + __resetCustomSourcesForTesting(); + }); + + const created = () => requests.filter((r) => r.url.endsWith("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/v1/users")); + + // The registered key isn't something --source accepts; the path is. + test("the printed command names the source's path, not its key", async () => { + const error = (await run({ + input: "export.json", + source: customFile, + secretKey: "sk_test_x", + json: true, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toBe( + `clerk migrate import export.json --source ${customFile} --secret-key --json --yes`, + ); + }); + + test("imports through a user-authored source", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + expect(created().map((r) => (r.body as { external_id: string }).external_id)).toEqual([ + "mp_1", + "mp_2", + ]); + expect(captured.err).toContain("myplatform"); + expect(captured.err).toContain("source from"); + }); + + test("applies the custom source's defaults and postTransform", async () => { + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ first_name: "Ada", password_hasher: "bcrypt" }); + // postTransform dropped the empty given name rather than sending "". + expect("first_name" in (bodies[1] ?? {})).toBe(false); + }); + + // An edited source misses the live run's `sourceHash`, but both would send + // the same external IDs. + test("refuses while a run of this file with an earlier version of the source is live", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + const [first] = listRuns(runsDir()); + const record = readRun(runsDir(), first!.id)!; + delete record.finishedAt; + fs.writeFileSync(path.join(runsDir(), first!.id, "run.json"), JSON.stringify(record)); + // PID 1 is always alive, and never this test. + fs.writeFileSync(path.join(runsDir(), first!.id, "lock"), "1"); + + const edited = `./custom-run-${customCounter++}.ts`; + fs.writeFileSync(path.join(workDir, edited), `${CUSTOM}\n// edited`); + requests = []; + await expect( + run({ input: "export.json", source: edited, yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/importing this file right now in another process/); + expect(created()).toHaveLength(0); + }); + + // The loader caches by URL: an edit at the same path must still load the + // edited mappings, the ones the run's hash names. + test("an edited source at the same path maps with its edited code", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + fs.writeFileSync( + path.join(workDir, customFile), + CUSTOM.replace('given: "firstName"', 'given: "lastName"'), + ); + requests = []; + await run({ + input: "export.json", + source: customFile, + yes: true, + secretKey: "sk_test_x", + newRun: true, + }); + + const bodies = created().map((r) => r.body as Record); + expect(bodies[0]).toMatchObject({ last_name: "Ada" }); + expect("first_name" in (bodies[0] ?? {})).toBe(false); + }); + + // An edited source is a different source, so the run records which one. + test("records the custom source's content hash on the run", async () => { + await run({ input: "export.json", source: customFile, yes: true, secretKey: "sk_test_x" }); + + const [record] = listRuns(runsDir()); + expect(record?.source).toBe("myplatform"); + expect(record?.sourceHash).toMatch(/^[0-9a-f]{64}$/); + }); + + test("an unknown built-in key is a usage error listing the valid ones", async () => { + await expect( + run({ input: "export.json", source: "okta", yes: true, secretKey: "sk_test_x" }), + ).rejects.toThrow(/Unknown source "okta". Valid sources: clerk, auth0/); + expect(created()).toHaveLength(0); + }); + + test("fails before any request when the file is not there", async () => { + await expect( + run({ + input: "export.json", + source: "./nope.ts", + yes: true, + secretKey: "sk_test_x", + }), + ).rejects.toThrow(/No source file at/); + expect(requests).toHaveLength(0); + }); + + test("fails before any request when the file is malformed", async () => { + const bad = `./bad-${customCounter++}.ts`; + fs.writeFileSync( + path.join(workDir, bad), + `export default { key: "x", label: "X", transformer: {} };`, + ); + + const error = (await run({ + input: "export.json", + source: bad, + yes: true, + secretKey: "sk_test_x", + }).catch((caught: unknown) => caught)) as CliError; + expect(error.message).toMatch(/no source field maps to `userId`/); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(requests).toHaveLength(0); + }); + + test("still requires a file", async () => { + await expect(run({ source: customFile, yes: true, secretKey: "sk_test_x" })).rejects.toThrow( + /needs the file to import/, + ); + }); + }); + + // The two `--json` outcomes that stop before a run exists: each says why, + // with `run: null`. + describe("--json, stopped before any run", () => { + test("a refused import reports refused, and exits 2", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([...export2, { id: "u3" }]), + ); + expect(await exitCodeOf(run({ ...baseOptions, json: true }))).toBe(EXIT_CODE.USAGE); + expect(JSON.parse(captured.out)).toMatchObject({ run: null, refused: true }); + expect(created()).toHaveLength(0); + }); + + test("a file with nobody to import reports nothingToImport", async () => { + fs.writeFileSync(path.join(workDir, "export.json"), "[]"); + await run({ ...baseOptions, json: true }); + expect(JSON.parse(captured.out)).toMatchObject({ run: null, nothingToImport: true }); + expect(created()).toHaveLength(0); + }); + }); + + describe("stopped part-way", () => { + // `importUsers` returns normally on a Ctrl-C; the run must not read as done. + test("a Ctrl-C after the first create leaves the run unfinished, as interrupted", async () => { + process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT = "1"; + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const response = await clerk(input, init); + if (init?.method === "POST" && new URL(input.toString()).pathname === "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/v1/users") { + beginInterrupt(); + abortInFlight(); + } + return response; + }) as typeof fetch; + try { + // Thrown, so the gutter closes as paused rather than done. + await expect(run(baseOptions)).rejects.toThrow(); + } finally { + _resetInterruptState(); + delete process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT; + } + + const [record] = listRuns(runsDir()); + // The run folder is the only record of who was created. + expect(captured.err).toContain(`Run ${record?.id} records who was created:`); + expect(record?.finishedAt).toBeUndefined(); + expect(fs.existsSync(path.join(runsDir(), record?.id ?? "", "lock"))).toBe(false); + expect(created()).toHaveLength(1); + }); + + test("the user quota leaves the rest not sent, and the run partial", async () => { + stubClerk(); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => + init?.method === "POST" && new URL(input.toString()).pathname === "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/v1/users" + ? Response.json( + { + errors: [ + { + code: "user_quota_exceeded", + message: "user quota exceeded", + long_message: "You have reached your limit of 100 users.", + }, + ], + }, + { status: 403 }, + ) + : clerk(input, init)) as typeof fetch; + process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT = "1"; + try { + await run({ ...baseOptions, json: true }); + } finally { + delete process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT; + } + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { status: "partial" }, + result: { + created: 0, + failed: 1, + notSent: 1, + stopReason: "You have reached your limit of 100 users.", + }, + }); + }); + + // In the summary, after the progress bar: printed mid-run, the bar redrew + // over it. + test("the summary says why the import stopped", async () => { + stubClerk(); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => + init?.method === "POST" && new URL(input.toString()).pathname === "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/v1/users" + ? Response.json( + { + errors: [ + { + code: "user_quota_exceeded", + message: "user quota exceeded", + long_message: "You have reached your limit of 100 users.", + }, + ], + }, + { status: 403 }, + ) + : clerk(input, init)) as typeof fetch; + await run(baseOptions); + + const err = Bun.stripANSI(captured.err); + expect(err).toContain("Not sent: 1"); + expect(err).toContain( + "You have reached your limit of 100 users. No more users were sent: running the import again picks up the rest once the limit is raised.", + ); + }); + }); + + describe("a phone Clerk refuses", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + + beforeEach(() => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify([ + { id: "u1", primary_email_address: "a@x.dev", primary_phone_number: "+33612345678" }, + ]), + ); + const clerk = globalThis.fetch; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const body = init?.body ? (JSON.parse(init.body as string) as Record) : {}; + const isCreate = + init?.method === "POST" && new URL(input.toString()).pathname === "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/v1/users"; + if (isCreate && body.phone_number) { + requests.push({ method: "POST", url: input.toString(), body }); + return Response.json( + { + errors: [ + { code: "unsupported_country_code", message: "unsupported", long_message: COUNTRY }, + ], + }, + { status: 403 }, + ); + } + return clerk(input, init); + }) as typeof fetch; + }); + + // The user imports, so it is no failure, but the phone is gone. + test("is counted in the summary, with the SMS note", async () => { + await run(baseOptions); + const err = Bun.stripANSI(captured.err); + expect(err).toContain("Failed: 0"); + expect(err).toContain(`Imported without their phone:\n 1 user: ${COUNTRY}`); + expect(err).toContain("Development instances block SMS"); + }); + + test("is a warning in --json", async () => { + await run({ ...baseOptions, json: true }); + expect(JSON.parse(captured.out)).toMatchObject({ + result: { + created: 1, + failed: 0, + errors: [], + warnings: [{ warning: `imported without their phone: ${COUNTRY}`, count: 1 }], + }, + }); + }); + }); + + describe("per-platform imports", () => { + /** One realistic record per platform, in that platform's export shape. */ + const PLATFORMS: [string, unknown, string][] = [ + [ + "auth0", + [ + { + user_id: "auth0|1", + email: "a@x.dev", + email_verified: true, + given_name: "Ada", + family_name: "L", + }, + ], + "auth0|1", + ], + ["authjs", [{ id: "aj1", email: "a@x.dev", email_verified: "2024-01-01T00:00:00Z" }], "aj1"], + [ + "betterauth", + [{ user_id: "ba1", email: "a@x.dev", email_verified: true, password_hash: BCRYPT }], + "ba1", + ], + [ + "supabase", + [ + { + id: "sb1", + email: "a@x.dev", + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: BCRYPT, + }, + ], + "sb1", + ], + ]; + + test.each(PLATFORMS)( + "%s transforms, validates and imports its export", + async (key, records, externalId) => { + fs.writeFileSync(path.join(workDir, "export.json"), JSON.stringify(records)); + + await run({ ...baseOptions, source: key }); + + const created = requests.filter((r) => r.url.endsWith("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/v1/users")); + expect(created).toHaveLength(1); + expect((created[0]?.body as { external_id: string } | undefined)?.external_id).toBe( + externalId, + ); + }, + ); + + test("firebase imports its wrapped export and builds the scrypt digest", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ + users: [ + { + localId: "fb1", + email: "a@x.dev", + emailVerified: true, + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + }, + ], + }), + ); + + await run({ + ...baseOptions, + source: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }); + + const body = requests.find((r) => r.url.endsWith("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/v1/users"))?.body as Record< + string, + unknown + >; + expect(body).toMatchObject({ + external_id: "fb1", + password_digest: "SGFzaA==$U2FsdA==$SIGNER$Bw==$8$14", + password_hasher: "scrypt_firebase", + }); + }); + + // The users a run created carry its parameters; the rest must match them. + test("a firebase run continues only with the hash parameters it started with", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ + users: ["fb1", "fb2"].map((localId) => ({ + localId, + email: `${localId}@x.dev`, + emailVerified: true, + passwordHash: "SGFzaA==", + salt: "U2FsdA==", + })), + }), + ); + const flags = { + ...baseOptions, + source: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }; + stubClerk({ failing: new Set(["fb2"]) }); + await run(flags); + expect(readRun(runsDir(), listRuns(runsDir())[0]!.id)?.firebaseHash).toMatch( + /^[0-9a-f]{64}$/, + ); + + requests = []; + process.exitCode = 0; + stubClerk(); + await expect(run({ ...flags, firebaseRounds: 9 })).rejects.toThrow( + /different Firebase hash parameters/, + ); + expect(created()).toEqual([]); + + await run(flags); + expect(created()).toEqual(["fb2"]); + }); + + test("the printed command keeps the --firebase-* flags, as placeholders", async () => { + fs.writeFileSync( + path.join(workDir, "export.json"), + JSON.stringify({ users: [{ localId: "fb1", email: "a@x.dev", emailVerified: true }] }), + ); + + const error = (await run({ + ...baseOptions, + yes: false, + source: "firebase", + firebaseSignerKey: "SIGNER", + firebaseSaltSeparator: "Bw==", + firebaseRounds: 8, + firebaseMemCost: 14, + }).catch((caught: unknown) => caught)) as CliError; + + expect(error.examples?.[0]?.command).toContain( + "--firebase-signer-key SIGNER_KEY --firebase-salt-separator SALT_SEPARATOR --firebase-rounds ROUNDS --firebase-mem-cost MEM_COST", + ); + }); + + test("a partial firebase flag set fails before anything is read", async () => { + await expect( + run({ ...baseOptions, source: "firebase", firebaseSignerKey: "SIGNER" }), + ).rejects.toThrow(/--firebase-salt-separator/); + expect(requests).toHaveLength(0); + }); + + test("an unknown source fails listing the valid keys", async () => { + await expect(run({ ...baseOptions, source: "okta" })).rejects.toThrow( + /Unknown source "okta".*clerk.*supabase/s, + ); + }); + }); +}); + +describe("explainErrors", () => { + const COUNTRY = + "Phone numbers from this country (France) are currently not supported. For more information, please contact support."; + const QUOTA = + "You have reached your limit of 100 users. If you need more users, please use a Production instance."; + + test("names the development instance as the reason countries are blocked", () => { + const [note] = explainErrors([COUNTRY], "dev"); + expect(note).toContain("Development instances block SMS to most countries"); + expect(note).toContain("test-emails-and-phones"); + }); + + test("sends a production operator to the Dashboard instead of support", () => { + const [note] = explainErrors([COUNTRY], "prod"); + expect(note).toContain("customization/sms/settings"); + expect(note).not.toContain("Development instances"); + }); + + test("explains the user quota only where one applies", () => { + expect(explainErrors([QUOTA], "dev").join(" ")).toContain("development-instance quota"); + // Production has no such quota by default, and the API's message already + // names the plan upgrade when a plan sets one. + expect(explainErrors([QUOTA], "prod")).toEqual([]); + }); + + test("says nothing about errors it does not recognize", () => { + expect(explainErrors(["Something else went wrong."], "dev")).toEqual([]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/run.ts b/packages/cli-core/src/commands/migrate/run.ts new file mode 100644 index 000000000..e4bbd4e96 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/run.ts @@ -0,0 +1,1189 @@ +/** + * `clerk migrate import ` — the user import itself. + * + * Registered as the `import` subcommand. The exported handler keeps the name + * `run` because `import` is a reserved word. + * + * An import goes through the same steps every time: + * + * 1. Settle the file (an export run ID stands for the file that run wrote), + * the source (a file from `clerk migrate export` names its own) and the + * target instance, and print the target. + * 2. Decide whether this continues an earlier run of the same file, source and + * instance, from the run store. + * 3. Run `checkImport()` against the real instance. `--dry-run` stops here. + * 4. Refuse on any reject unless `--allow-partial`, then ask for consent. + * Nothing is written without `--yes` or a yes at the prompt. + * 5. Import, one run line per user. + */ + +import path from "node:path"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { resolveProfile } from "../../lib/config.ts"; +import { hasAccountCredentials } from "../../lib/credential-store.ts"; +import { + AUTH_ERROR_REASON, + AuthError, + CliError, + ERROR_CODE, + throwUsageError, + throwUserAbort, +} from "../../lib/errors.ts"; +import { resolveKeylessTarget } from "../../lib/keyless-target.ts"; +import { quoteArg } from "../../lib/json-body.ts"; +import { log } from "../../lib/log.ts"; +import { NEXT_STEPS, printAgentNextSteps } from "../../lib/next-steps.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { interruptedExitCode } from "../../lib/signals.ts"; +import { withGutter, withSpinner } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { importUsers, splitIdentifiers } from "./import-users.ts"; +import { checkImport, type ImportChecks } from "./lib/checks.ts"; +import { fetchInstanceSettings, fetchUserCount } from "./lib/clerk-config.ts"; +import { readEnvelope, type ExportEnvelope } from "./lib/export-file.ts"; +import { + firebaseHashConfigProblem, + fingerprintFirebaseHashConfig, + resolveFirebaseHashConfig, + type FirebaseHashFlags, +} from "./lib/firebase-hash.ts"; +import { DEV_USER_LIMIT, resolveLimits, type InstanceType } from "./lib/instance.ts"; +import { + continueRun, + latestUserLines, + listRuns, + liveLockPid, + lockFile, + lockImport, + readRun, + readUserLines, + resolveRunsDir, + RUN_ID_PATTERN, + runDir, + runState, + sha256File, + startRun, + type Run, + type RunRecord, + type UserLine, +} from "./lib/run-store.ts"; +import { withProgress } from "./lib/progress.ts"; +import { createApiScheduler } from "./lib/scheduler.ts"; +import { readSupabaseRows } from "./lib/supabase-providers.ts"; +import { keyInstanceId, printTarget, resolveClerkTarget } from "./lib/target.ts"; +import { findInFlight } from "./lib/user-lookup.ts"; +import { + fileExists, + getFileType, + loadUsersFromFile, + resolveImportFilePath, +} from "./lib/transform.ts"; +import { resolveSource, sourceKeys } from "./sources/registry.ts"; +import type { ImportSummary, User } from "./types.ts"; +import { promptForFile, promptForFirebaseHashConfig, promptForSource } from "./wizard.ts"; +import { login } from "../auth/login.ts"; +import { link } from "../link/index.ts"; + +export type MigrateRunOptions = { + /** An export file, or the ID of the export run that wrote one. */ + input?: string; + /** A built-in source key, or the path to a source you wrote. */ + source?: string; + /** Content hash of a custom `--source`, set once it is loaded. */ + sourceHash?: string; + /** `--source` as typed, for printed commands: a custom source's path. Not a flag. */ + sourceArg?: string; + /** The file to read, once `input` is resolved. Not a flag. */ + file?: string; + requirePassword?: boolean; + /** + * Import users with no legal acceptance into an instance that requires it. + * Without it, a prompt asks; where nobody can be asked, they are rejected. + */ + skipLegalChecks?: boolean; + /** + * Create the emails and phones the source never verified as reserved, not + * unverified. Without it, a prompt asks; where nobody can be asked, they + * stay unverified. + */ + reserveUnverified?: boolean; + /** Check against the instance, report, and write nothing. */ + dryRun?: boolean; + /** Import the users that pass, and record the rest as skipped. */ + allowPartial?: boolean; + /** Start a fresh run even when an earlier one matches. */ + newRun?: boolean; + yes?: boolean; + json?: boolean; + secretKey?: string; + app?: string; + instance?: string; + /** Where runs are kept; overrides `CLERK_MIGRATE_DIR`. */ + runsDir?: string; +} & FirebaseHashFlags; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +/** Nobody can answer a prompt: an agent, a non-TTY run, or `--json`. */ +function canPrompt(options: MigrateRunOptions): boolean { + return !options.json && isHuman() && !isAgent(); +} + +/** + * Validates what a run needs before anything is read or sent. + * + * `--source` has already been resolved to a registered key by the time this + * runs, custom sources included. + * + * @returns The source key and file path, both guaranteed present. + */ +export function validateRunOptions(options: MigrateRunOptions): { + source: string; + file: string; +} { + if (!options.source) { + throwUsageError( + `Missing --source. Valid values: ${sourceKeys().join(", ")}, or the path to a source you wrote.`, + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a Clerk export", + }, + ], + ); + } + if (!options.file) { + throwUsageError( + "Missing the file to import (a JSON or CSV export, or an export run ID).", + undefined, + ERROR_CODE.USAGE_ERROR, + [ + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a Clerk export", + }, + ], + ); + } + if (!fileExists(options.file)) { + throw new CliError(`File not found: ${options.file}`, { code: ERROR_CODE.FILE_NOT_FOUND }); + } + if (!getFileType(options.file)) { + throwUsageError(`Unsupported file type for ${options.file}. Provide a .json or .csv file.`); + } + + return { source: options.source, file: options.file }; +} + +/** Where a production instance's operator changes the SMS country blocklist. */ +const SMS_SETTINGS_URL = "https://dashboard.clerk.com/~/customization/sms/settings"; + +/** Clerk's fictional email addresses and phone numbers, for development. */ +const TEST_NUMBERS_URL = "https://clerk.com/docs/guides/development/testing/test-emails-and-phones"; + +/** + * What the API's error messages leave out: whether the operator can do + * something about them, and where. + * + * Both of these read as account-level restrictions and are not. Blocked + * countries are a per-instance SMS blocklist that development instances are + * created with far more of, and the user limit is a development-instance quota + * that production does not have by default — so "contact support", which both + * messages point at, is the wrong first move for most readers. + * + * @returns One note per recognized error family, empty when none apply. + */ +export function explainErrors(errors: Iterable, instanceType: InstanceType): string[] { + const all = [...errors]; + const notes: string[] = []; + + if (all.some((error) => error.includes("Phone numbers from this country"))) { + notes.push( + instanceType === "dev" + ? `Development instances block SMS to most countries by default — this is not a limit on your account. ` + + `Use Clerk's test phone numbers while developing (${TEST_NUMBERS_URL}), and contact support only if ` + + `you need real numbers in a specific country before going to production.` + : `Unblock the countries you need under SMS settings in the Dashboard (${SMS_SETTINGS_URL}). ` + + `Plans without SMS support cannot remove them; contact support if the setting is refused.`, + ); + } + + // Production has no user limit unless a plan imposes one, and the API's own + // message already names the fix ("upgrade to a paid plan") in that case. + if ( + instanceType === "dev" && + all.some((error) => /You have reached your limit of \d+ users/.test(error)) + ) { + notes.push( + `The user limit is a development-instance quota (${DEV_USER_LIMIT} by default). Import into a production ` + + `instance to bring everyone across, or contact support to raise this instance's limit.`, + ); + } + + return notes; +} + +// --- Settling the file, source and target ---------------------------------- + +/** + * Makes sure there is somewhere to import *into* before anything else happens. + * + * Without this the first complaint comes from deep inside the secret-key chain, + * which resolves the linked profile before it ever asks for a token — so a + * signed-out operator in an unlinked directory is told to `clerk link`, a + * command that will only turn around and ask them to sign in. + * + * A human gets the same sign-in-then-link flow `clerk link` already runs. An + * agent cannot answer a browser login or an application picker, so it gets the + * error naming whichever half is missing. + */ +async function ensureImportTarget(options: MigrateRunOptions): Promise { + // A secret key names the destination instance on its own, with no account + // and no linked directory involved — mirroring resolveBapiSecretKey, which + // takes `--app` over an exported CLERK_SECRET_KEY. + if (options.secretKey || (!options.app && process.env.CLERK_SECRET_KEY)) return; + // `--app` names it too, but resolves its key through the Platform API, which + // needs an account: it goes through the sign-in below, though not the link. + // An unclaimed accountless application keeps its only secret key on disk. + if (!options.app && (await resolveKeylessTarget({ instance: options.instance }))) return; + + const interactive = canPrompt(options); + + if (!(await hasAccountCredentials())) { + if (!interactive) { + throw new AuthError({ + reason: AUTH_ERROR_REASON.NOT_LOGGED_IN, + message: + "Not logged in, so there is no Clerk instance to import into. Run `clerk auth login`, then `clerk link`.", + examples: [ + { command: "clerk auth login", description: "Sign in, then re-run the import" }, + { + command: + "clerk migrate import users.json --source clerk --yes --secret-key sk_test_...", + description: "Import without signing in", + }, + ], + }); + } + log.info("Not logged in. Signing in first..."); + await login({ showNextSteps: false }); + } + + // Left to the secret-key chain when non-interactive: its `not_linked` error + // is the one every other command raises, and there is nothing to add to it. + if (interactive && !options.app && !(await resolveProfile(process.cwd()))) { + log.info("This directory isn't linked to a Clerk application. Linking one first..."); + await link({ skipIfLinked: true }); + } +} + +/** + * Settles which file to read. The argument may name the export run that wrote + * the file; a human who gave none is asked for a path. + */ +async function resolveInput( + options: MigrateRunOptions, +): Promise<{ file: string; fromExport?: string; exportSha256?: string }> { + const value = options.input; + if (!value) { + if (!canPrompt(options)) { + throwUsageError( + "`clerk migrate import` needs the file to import, or the export run that wrote it, and cannot prompt here.", + undefined, + undefined, + [ + { + command: "clerk migrate import 20260929-141502-a1b2 --yes", + description: "Import what an export run wrote", + }, + { + command: "clerk migrate import users.json --source clerk --yes", + description: "Import a file", + }, + ], + ); + } + return { file: await promptForFile() }; + } + if (!RUN_ID_PATTERN.test(value) || fileExists(value)) return { file: value }; + + const runsDir = await resolveRunsDir(options.runsDir); + const record = readRun(runsDir, value); + if (!record) { + throwUsageError(`No run \`${value}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + if (record.kind !== "export" || !record.file) { + throwUsageError( + `Run ${value} is an ${record.kind} run, which has no file to import. Name an export run, or a file.`, + ); + } + // `--output` can point a later export, or an edit, at the same path. The run + // ID has to still mean the file that run wrote, not whatever is there now. + const { path: file, sha256 } = record.file; + if (!fileExists(file)) { + throwUsageError( + `The file run ${value} wrote, ${quoteArg(file)}, is gone. Export again, or import a file by its path.`, + ); + } + if (sha256File(file) !== sha256 || readEnvelope(file)?.runId !== record.id) { + throwChangedExport(value, file); + } + // Carried on, so the file is checked again where the users are read: another + // export can still overwrite the path after this. + return { file, fromExport: record.id, exportSha256: sha256 }; +} + +/** A file the import read changed before it finished reading it. */ +function throwChangedFile(file: string): never { + throwUsageError( + `${quoteArg(file)} changed while it was being imported. Nothing was imported. ` + + "Run the import again once nothing else is writing to it.", + ); +} + +function throwChangedExport(runId: string, file: string): never { + throwUsageError( + `The file run ${runId} wrote, ${quoteArg(file)}, has changed since it was exported. Nothing was imported. ` + + "Import it by its path if you mean its current contents, or export again.", + ); +} + +/** + * Resolves `--source` to a registered key, loading a custom source from its + * path so the rest of the run treats it exactly like a built-in. + * + * @throws UsageError for an unknown key. + */ +async function applySource(options: MigrateRunOptions): Promise { + if (!options.source) return options; + const resolved = await resolveSource(options.source); + if (resolved.path && !options.json) { + log.info(`Loaded the \`${resolved.key}\` source from ${options.source}.`); + } + return { + ...options, + source: resolved.key, + sourceArg: options.source, + ...(resolved.hash ? { sourceHash: resolved.hash } : {}), + }; +} + +/** + * The source an export file names for itself, checked against the one the + * flags name. + * + * @throws UsageError when the two disagree: importing an Auth0 export through + * the Supabase mapping would create users with the wrong fields. + */ +function applyEnvelope( + options: MigrateRunOptions, + envelope: ExportEnvelope | undefined, +): MigrateRunOptions { + if (!envelope) return options; + if (options.source && options.source !== envelope.source) { + throwUsageError( + `The file was exported from ${envelope.source}, but --source names ${options.source}. ` + + "Drop --source: the file already says where it came from.", + ); + } + return { ...options, source: envelope.source }; +} + +// --- Continuing an earlier run --------------------------------------------- + +/** + * Refuses while another process is importing this file, with this source key, + * into this instance. Checked before any resume matching, and under + * --new-run too: an edited custom source (another `sourceHash`) or a new run + * would otherwise send the same external IDs alongside it. + */ +function assertNoActiveImport( + runsDir: string, + match: { sha256: string; source: string; instanceId: string; keyInstanceId?: string }, +): void { + const active = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId) && + runState(runsDir, record) === "running", + ); + if (active) { + throwUsageError( + `Run ${active.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, active.id)}). ` + + `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, active.id)}.`, + ); + } +} + +/** How this import relates to earlier runs of the same file, source and instance. */ +export type ResumeCase = + | { kind: "new" } + | { kind: "continue"; record: RunRecord; because: "interrupted" | "partial" } + | { kind: "complete"; record: RunRecord }; + +/** + * Finds the latest import run of this file (by sha256), this source (and a + * custom source's hash) and this instance, and decides what a re-run does: + * + * - none, or undone: a new run + * - interrupted: continue it, skipping the users it created + * - partial: continue it, retrying the users that failed or were skipped + * - complete: nothing; the file is already in + * + * @throws UsageError when that run is still running in another process, or + * has an undo that never finished: some of its users are gone and some are + * not, so neither continuing it nor starting over is safe. + */ +export function findResume( + runsDir: string, + match: { + sha256: string; + source: string; + sourceHash?: string; + instanceId: string; + /** The key's stand-in ID, for a run recorded while Clerk could not name the instance. */ + keyInstanceId?: string; + }, +): ResumeCase { + const latest = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + record.sourceHash === match.sourceHash && + (record.target.instanceId === match.instanceId || + record.target.instanceId === match.keyInstanceId), + ); + if (!latest) { + // Clerk did not name the instance (GET /v1/instance failed), so a run of + // this file recorded under a real ID can be neither matched nor ruled out. + // Starting over would reject every user it created and leave it unfinished. + if (match.instanceId.startsWith("key_")) { + const unconfirmed = listRuns(runsDir).find( + (record) => + record.kind === "import" && + record.file?.sha256 === match.sha256 && + record.source === match.source && + record.target.instanceId?.startsWith("ins_") === true, + ); + if (unconfirmed) { + throwUsageError( + `Clerk did not confirm which instance this key addresses, so this import cannot tell whether run ${unconfirmed.id} is for it. ` + + "Nothing was imported. Try again, or pass --new-run to start a new run.", + ); + } + } + return { kind: "new" }; + } + + const state = runState(runsDir, latest); + if (state === "running") { + throwUsageError( + `Run ${latest.id} is importing this file right now in another process (PID ${liveLockPid(runsDir, latest.id)}). ` + + `Wait for it to finish. If that process is not a migrate run, delete ${lockFile(runsDir, latest.id)}.`, + ); + } + if (state === "undone") return { kind: "new" }; + + const undoRun = listRuns(runsDir).find( + (record) => record.kind === "undo" && record.undoes === latest.id, + ); + if (undoRun && runState(runsDir, undoRun) !== "complete") { + throwUsageError( + `Run ${latest.id} has an undo that did not finish (run ${undoRun.id}). ` + + `Finish it with \`clerk migrate undo ${latest.id}\`, or pass --new-run to import into a new run.`, + ); + } + // Undone in full, though the import run was never marked so. + if (undoRun) return { kind: "new" }; + + if (state === "complete") return { kind: "complete", record: latest }; + return { + kind: "continue", + record: latest, + because: state === "interrupted" ? "interrupted" : "partial", + }; +} + +// --- Reporting ------------------------------------------------------------- + +/** How many rejected source IDs to name per reason before summing the rest. */ +const REJECT_SAMPLE = 5; + +function printChecks(checks: ImportChecks): void { + log.blank(); + log.info(bold("Checks")); + log.info(` ${plural(checks.total, "user")} checked`); + + if (checks.rejects.length > 0) { + log.info(` ${red("✗")} ${red(`${plural(checks.rejects.length, "user")} rejected`)}`); + for (const { reason, count } of checks.rejectReasons) { + const ids = checks.rejects + .filter((reject) => reject.reason === reason) + .map((reject) => + reject.keptSourceId + ? `${reject.sourceId} (kept: ${reject.keptSourceId})` + : reject.sourceId, + ); + const sample = ids.slice(0, REJECT_SAMPLE).join(", "); + const more = ids.length > REJECT_SAMPLE ? `, and ${ids.length - REJECT_SAMPLE} more` : ""; + log.info(` ${count}: ${reason}`); + log.info(` ${dim(sample + more)}`); + } + } + + if (checks.warnings.length > 0) { + log.info(` ${yellow("⚠")} ${yellow("Imported, but not everything comes across")}`); + for (const warning of checks.warnings) log.info(` ${dim(warning)}`); + } + + log.info(` ${green("✓")} ${green(`${plural(checks.importable.length, "user")} to import`)}`); + + if (checks.settingsUnavailable) { + log.info( + ` ${yellow("!")} ${dim("Could not read this instance's settings, so required fields were not checked.")}`, + ); + } + + if (checks.fixes.length > 0) { + log.blank(); + log.info(bold("Or change the instance instead")); + for (const fix of checks.fixes) { + log.info(` ${fix.label}`); + log.info(dim(` ${fix.command ?? fix.url}`)); + } + } +} + +function checksJson(checks: ImportChecks) { + return { + total: checks.total, + importable: checks.importable.length, + rejects: checks.rejects, + rejectReasons: checks.rejectReasons, + warnings: checks.warnings, + fixes: checks.fixes, + ...(checks.quota ? { quota: checks.quota } : {}), + settingsUnavailable: checks.settingsUnavailable, + }; +} + +function formatSummary( + summary: ImportSummary, + skipped: number, + record: RunRecord, + runFolder: string, + instanceType: InstanceType, +): string[] { + const lines = [ + `${green("Imported:")} ${summary.successful}`, + `${red("Failed:")} ${summary.failed}`, + ]; + if (skipped > 0) lines.push(`${yellow("Skipped:")} ${skipped}`); + // The user quota stopped the run: these go out on a re-run. + if (summary.notSent > 0) lines.push(`${yellow("Not sent:")} ${summary.notSent}`); + // Printed here, after the progress bar, which would redraw over it mid-run. + if (summary.stopReason) { + lines.push( + "", + yellow( + `${summary.stopReason} No more users were sent: running the import again picks up the rest once the limit is raised.`, + ), + ); + } + + if (summary.errorBreakdown.size > 0) { + lines.push("", bold("Error breakdown:")); + for (const [error, count] of summary.errorBreakdown) { + lines.push(` ${plural(count, "user")}: ${error}`); + } + } + // Imported, so not failures, but the phone is gone: say so, and why. + if (summary.droppedPhones.size > 0) { + lines.push("", bold("Imported without their phone:")); + for (const [reason, count] of summary.droppedPhones) { + lines.push(` ${plural(count, "user")}: ${reason}`); + } + } + const explained = [...summary.errorBreakdown.keys(), ...summary.droppedPhones.keys()]; + for (const note of explainErrors(explained, instanceType)) lines.push("", note); + lines.push("", dim(`Run ${record.id}: ${runFolder}`)); + return lines; +} + +/** + * Once every user is in, the files that hold them are only a liability: the + * export carries user data and password hashes, and the run is only needed + * for `undo`. + */ +function cleanupLines(runsDir: string, record: RunRecord): string[] { + const lines: string[] = []; + if (record.fromExport) { + const exportDir = runDir(runsDir, record.fromExport); + // `export --output` writes the file outside the run folder, and deleting + // the folder alone would leave the password hashes on disk. + const file = record.file?.path; + const outside = file && path.relative(exportDir, file).startsWith(".."); + lines.push( + `The export in run ${record.fromExport} holds your users' data. Once you have checked the import, delete it:`, + dim(` rm -rf ${quoteArg(exportDir)}`), + ...(outside ? [dim(` rm ${quoteArg(file)}`)] : []), + ); + } + lines.push( + `Keep run ${record.id} while you might still undo it. After that:`, + dim(` rm -rf ${quoteArg(runDir(runsDir, record.id))}`), + ); + return lines; +} + +// --- The import ------------------------------------------------------------ + +/** + * The exact command that would carry on from here, for the consent and + * refusal messages. Every value is shell-quoted; secrets are placeholders. + */ +function commandFor(options: MigrateRunOptions, fromExport: string | undefined, extra: string[]) { + const input = fromExport ?? options.input ?? options.file ?? ""; + const parts = ["clerk migrate import", quoteArg(input)]; + const source = options.sourceArg ?? options.source; + if (!fromExport && source) parts.push("--source", quoteArg(source)); + if (options.allowPartial) parts.push("--allow-partial"); + if (options.newRun) parts.push("--new-run"); + if (options.requirePassword) parts.push("--require-password"); + if (options.skipLegalChecks) parts.push("--skip-legal-checks"); + if (options.reserveUnverified) parts.push("--reserve-unverified"); + // Names, not `<…>`: pasted as is, a shell reads `` as a redirect. + if (options.firebaseSignerKey) parts.push("--firebase-signer-key", "SIGNER_KEY"); + if (options.firebaseSaltSeparator !== undefined) + parts.push("--firebase-salt-separator", "SALT_SEPARATOR"); + if (options.firebaseRounds) parts.push("--firebase-rounds", "ROUNDS"); + if (options.firebaseMemCost) parts.push("--firebase-mem-cost", "MEM_COST"); + if (options.secretKey) parts.push("--secret-key", ""); + if (options.app) parts.push("--app", quoteArg(options.app)); + if (options.instance) parts.push("--instance", quoteArg(options.instance)); + if (options.runsDir) parts.push("--runs-dir", quoteArg(options.runsDir)); + if (options.json) parts.push("--json"); + return [...parts, ...extra].join(" "); +} + +/** + * A continued run, linked to the export run it now reads from: one started + * by path and continued by export run ID would otherwise lack the link, and + * the cleanup would not name the export holding the users' data. + */ +function continueWithExport(run: Run, fromExport: string | undefined): Run { + if (fromExport && run.record.fromExport !== fromExport) run.update({ fromExport }); + return run; +} + +/** + * Records the checks' rejects as skipped users, so the run says who they were. + * + * An adopted user keeps its `creating` line: it exists in Clerk, and `undo` + * finds it only through that line. + */ +function recordRejects(run: Run, checks: ImportChecks, adopted: Map): void { + for (const { sourceId, reason, keptSourceId } of checks.rejects) { + if (adopted.has(sourceId)) continue; + run.append({ + sourceId, + status: "skipped", + reason: keptSourceId ? `${reason} (kept: ${keptSourceId})` : reason, + }); + } +} + +export async function run(rawOptions: MigrateRunOptions): Promise { + // Released however the import ends: a return, a refusal or a Ctrl-C. + const lock: ImportLock = {}; + try { + await runImport(rawOptions, lock); + } finally { + lock.release?.(); + } +} + +/** The import's identity lock, taken part-way through and released by {@link run}. */ +type ImportLock = { release?: () => void }; + +async function runImport(rawOptions: MigrateRunOptions, lock: ImportLock): Promise { + await ensureImportTarget(rawOptions); + let options = await applySource(rawOptions); + + const input = await resolveInput(options); + options = { ...options, file: input.file }; + // Hashed before the envelope is read: the envelope and the users are read + // separately, so each later read is checked against this revision. + const readSha256 = fileExists(input.file) + ? sha256File(resolveImportFilePath(input.file)) + : undefined; + const envelope = fileExists(input.file) + ? readEnvelope(resolveImportFilePath(input.file)) + : undefined; + // The revision the import must read throughout: the one an export run + // recorded, or the one the envelope came from. + const expectedSha256 = input.exportSha256 ?? readSha256; + const throwChanged = (filePath: string): never => + input.fromExport ? throwChangedExport(input.fromExport, filePath) : throwChangedFile(filePath); + options = applyEnvelope(options, envelope); + + // Asked only when the file does not say where it came from. + if (!options.source && canPrompt(options) && fileExists(input.file)) { + options = { ...options, source: await promptForSource() }; + } + + const { source, file } = validateRunOptions(options); + // The flags win, so a rotated key can be passed without re-exporting. + const fromFlags = resolveFirebaseHashConfig(options, source); + let firebaseHashConfig = fromFlags ?? (source === "firebase" ? envelope?.firebase : undefined); + let configFrom = fromFlags ? "the --firebase-* flags" : "the export file"; + if (source === "firebase" && !firebaseHashConfig && canPrompt(options)) { + firebaseHashConfig = await promptForFirebaseHashConfig(); + configFrom = "the parameters entered"; + } + // Wherever they came from, each goes into every password digest. + const problem = firebaseHashConfig && firebaseHashConfigProblem(firebaseHashConfig); + if (problem) { + throwUsageError( + `The Firebase hash parameters from ${configFrom} won't work: ${problem}. Nothing was imported.\n` + + "Find all four in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + await withGutter( + "Migrating users to Clerk", + async ({ setNextSteps }) => { + const { secretKey, target } = await resolveClerkTarget(options); + if (!options.json) printTarget(target); + const limits = resolveLimits(secretKey); + + const filePath = resolveImportFilePath(file); + const sha256 = sha256File(filePath); + if (expectedSha256 !== undefined && sha256 !== expectedSha256) throwChanged(filePath); + const runsDir = await resolveRunsDir(options.runsDir); + + // Held from before the checks to the end of the run: the scan below + // alone leaves a gap a second process can pass through before either + // one writes a run. + if (!options.dryRun) { + lock.release = lockImport(runsDir, { sha256, source, instanceId: target.instanceId }); + } + assertNoActiveImport(runsDir, { + sha256, + source, + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); + const resume: ResumeCase = options.newRun + ? { kind: "new" } + : findResume(runsDir, { + sha256, + source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), + instanceId: target.instanceId, + keyInstanceId: keyInstanceId(secretKey), + }); + + // The users this run created carry digests built from its parameters, so + // the rest must be built from the same ones. A run from before this was + // recorded has none to compare, and continues as it did. + const firebaseHash = firebaseHashConfig + ? fingerprintFirebaseHashConfig(firebaseHashConfig) + : undefined; + if ( + resume.kind === "continue" && + resume.record.firebaseHash && + resume.record.firebaseHash !== firebaseHash + ) { + throwUsageError( + `Run ${resume.record.id} imported this file with different Firebase hash parameters, and the users it created carry digests built from them. Nothing was imported.\n` + + "Pass the same --firebase-* flags to continue it, or --new-run to start a new run.", + ); + } + + if (resume.kind === "complete") { + if (options.json) { + log.data(JSON.stringify({ target, run: resume.record, alreadyImported: true }, null, 2)); + } else { + log.success( + `Already imported in run ${resume.record.id}. Pass --new-run to import it again.`, + ); + } + return; + } + + // Users the run being continued already created are done: they are not + // checked or sent again. Those whose extra identifiers never attached + // get just the attaches. + const continued = resume.kind === "continue" ? resume.record : undefined; + // What the plan below is built from; checked again once the run's lock + // is held, since a prompt can wait while an undo or a continue writes. + const plannedLines = continued ? readUserLines(runsDir, continued.id).length : undefined; + const done = new Map(); + const attachOnly: UserLine[] = []; + const inFlight: string[] = []; + const inFlightReserved = new Set(); + if (continued) { + for (const line of latestUserLines(runsDir, continued.id).values()) { + if (line.status === "creating") inFlight.push(line.sourceId); + if (line.status === "creating" && line.reserved) inFlightReserved.add(line.sourceId); + if (line.status !== "created" || !line.clerkId) continue; + done.set(line.sourceId, line.clerkId); + if (line.pending?.length) attachOnly.push(line); + } + } + + // Creates the run stopped with no answer to. Those Clerk holds are + // adopted: checked as usual, but never created again. + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const adopted = new Map(); + if (continued) { + const found = await findInFlight({ + runId: continued.id, + sourceIds: inFlight, + secretKey, + schedule, + }); + for (const user of found) adopted.set(user.sourceId, user.clerkId); + } + + if (!options.json) { + log.info( + resume.kind === "continue" + ? `Continuing run ${resume.record.id}, which ${resume.because === "interrupted" ? "was interrupted" : "finished partial"}: ` + + `${plural(done.size, "user")} already imported ${done.size === 1 ? "is" : "are"} left alone.` + : "Starting a new run.", + ); + if (adopted.size > 0) { + log.info( + `${plural(adopted.size, "user")} whose create was cut off ${adopted.size === 1 ? "is" : "are"} already in the instance, and won't be created again.`, + ); + } + } + + const loaded = await withSpinner(`Loading users from ${file}...`, async () => + loadUsersFromFile(file, source, { context: { firebaseHashConfig } }), + ); + let users = loaded.users.filter((user) => !done.has(user.userId)); + const failures = loaded.failures.filter((failure) => !done.has(failure.userId)); + + // An instruction about this import, not a prediction: users without a + // password are left out of the job rather than recorded as skipped. + let withoutPassword: User[] = []; + if (options.requirePassword) { + withoutPassword = users.filter((user) => !user.password && !adopted.has(user.userId)); + if (withoutPassword.length > 0 && !options.json) { + log.info( + `--require-password: leaving out ${plural(withoutPassword.length, "user")} without a password.`, + ); + } + // A Set, not `includes`: half a large Supabase export can lack a + // password, and a scan per user would stall before any check runs. + const leftOut = new Set(withoutPassword); + users = users.filter((user) => !leftOut.has(user)); + } + + let supabaseRows: Record[] | undefined; + if (source === "supabase") { + try { + supabaseRows = await readSupabaseRows(file); + } catch (error) { + log.debug(`migrate: could not read Supabase providers: ${String(error)}`); + } + } + + // Checked again after the last read of the file, before anything is + // created: the envelope, the users and the provider rows all came from + // one revision, the one an export run recorded if there is one. + if (expectedSha256 !== undefined && sha256File(filePath) !== expectedSha256) { + throwChanged(filePath); + } + + const [settings, existingUsers] = await withSpinner("Checking the instance...", async () => + Promise.all([ + fetchInstanceSettings(secretKey), + limits.instanceType === "dev" ? fetchUserCount(secretKey) : Promise.resolve(null), + ]), + ); + + // Creating a user without legal acceptance needs consent of its own: + // the flag, or a yes at a prompt. Otherwise the checks reject them. + let skipLegalChecks = options.skipLegalChecks ?? false; + const withoutLegal = settings?.sign_up?.legal_consent_enabled + ? users.filter((user) => !user.legalAcceptedAt && !user.skipLegalChecks).length + : 0; + // `-y` imports without prompting, so it does not stop here either: the + // checks reject these users, and --skip-legal-checks is the way through. + if ( + !skipLegalChecks && + withoutLegal > 0 && + !options.dryRun && + !options.yes && + canPrompt(options) + ) { + skipLegalChecks = await confirm({ + message: `${plural(withoutLegal, "user")} ${withoutLegal === 1 ? "has" : "have"} no legal acceptance on record, which this instance requires. Import them without it?`, + default: false, + }); + } + + // Unverified stays the default: reserved makes an address the source never + // confirmed usable for sign-in, which is the operator's call to make. + let reserveUnverified = options.reserveUnverified ?? false; + const withUnverified = users.filter((user) => { + const { unverifiedEmails, unverifiedPhones } = splitIdentifiers(user); + return unverifiedEmails.length > 0 || unverifiedPhones.length > 0; + }).length; + // `-y` is consent to write, not a yes to this, so it does not ask. + if ( + !reserveUnverified && + withUnverified > 0 && + !options.dryRun && + !options.yes && + canPrompt(options) + ) { + reserveUnverified = await confirm({ + message: `${plural(withUnverified, "user")} ${withUnverified === 1 ? "has" : "have"} an email or phone the source never verified. Create them reserved (usable for sign-in, locked to the user) instead of unverified?`, + default: false, + }); + } + // The answers carry into every command printed from here on, so a + // suggested re-run does not quietly drop what the operator chose. + options = { ...options, skipLegalChecks, reserveUnverified }; + + const checks = await withSpinner("Checking users against the instance...", async (spinner) => + checkImport({ + users, + skipLegalChecks, + reserveUnverified, + failures, + unknownFields: loaded.unknownFields, + ...(supabaseRows ? { supabaseRows } : {}), + settings, + existingUsers, + instanceType: limits.instanceType, + target, + secretKey, + schedule, + adoptedClerkIds: new Set(adopted.values()), + adoptedSourceIds: new Set(adopted.keys()), + // Their `creating` line says how the create ran, not this run's flag. + reservedSourceIds: new Set([...adopted.keys()].filter((id) => inFlightReserved.has(id))), + spinner, + }), + ); + + const refused = checks.rejects.length > 0 && !options.allowPartial; + // The users --require-password left out never reach the checks, so the + // JSON says how many, or `checks.total` would not add up to the file. + const leftOut = withoutPassword.length > 0 ? { withoutPassword: withoutPassword.length } : {}; + const preview = (extra: Record) => + log.data( + JSON.stringify( + { + target, + run: continued ?? null, + resume: resume.kind, + checks: checksJson(checks), + ...leftOut, + ...extra, + }, + null, + 2, + ), + ); + + if (!options.json) printChecks(checks); + + if (options.dryRun) { + if (options.json) preview({ dryRun: true }); + else { + log.blank(); + log.info(dim("Dry run: nothing was written.")); + } + // The exit code says what the real run would do. + if (refused) process.exitCode = 2; + return; + } + + if (refused) { + if (options.json) preview({ refused: true }); + throwUsageError( + `${plural(checks.rejects.length, "user")} would be rejected, so nothing was imported. ` + + "Fix them, or pass --allow-partial to import the rest and record them as skipped.", + undefined, + undefined, + [ + { + command: commandFor(options, input.fromExport, ["--allow-partial", "--yes"]), + description: "Import the users that pass", + }, + ], + ); + } + + if ( + checks.importable.length === 0 && + checks.rejects.length === 0 && + withoutPassword.length === 0 && + attachOnly.length === 0 + ) { + // Settled, so a continued run is finished rather than left interrupted. + if (continued) continueRun(runsDir, continued, plannedLines).finish(); + if (options.json) preview({ nothingToImport: true }); + else log.warn("No users left to import."); + return; + } + + // Rule 1: nothing is written without consent — `--yes`, or a yes at a + // prompt. An agent, a non-TTY run and `--json` never prompt. With users + // skipped, the question names them: "Import 0 users?" hides the skips. + const toCreate = plural(checks.importable.length, "user"); + const skipping = checks.rejects.length + withoutPassword.length; + const skips = skipping > 0 ? ` and skip ${plural(skipping, "user")}` : ""; + if (!options.yes) { + if (!canPrompt(options)) { + if (options.json) preview({ consent: "required" }); + throwUsageError( + `\`clerk migrate import\` will create ${toCreate}${skips}. Pass --yes to confirm.`, + undefined, + undefined, + [ + { + command: commandFor(options, input.fromExport, ["--yes"]), + description: "Run the import", + }, + ], + ); + } + log.blank(); + const proceed = await confirm({ + message: skips ? `Create ${toCreate}${skips}?` : `Import ${toCreate}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + // Gitignored only now, once there is consent to write a run. + await resolveRunsDir(options.runsDir, { write: true }); + const run = continued + ? continueWithExport(continueRun(runsDir, continued, plannedLines), input.fromExport) + : startRun(runsDir, { + kind: "import", + target, + source, + ...(options.sourceHash ? { sourceHash: options.sourceHash } : {}), + file: { path: filePath, sha256 }, + ...(input.fromExport ? { fromExport: input.fromExport } : {}), + ...(firebaseHash ? { firebaseHash } : {}), + }); + recordRejects(run, checks, adopted); + for (const user of withoutPassword) { + run.append({ + sourceId: user.userId, + status: "skipped", + reason: "no password (--require-password)", + }); + } + + const summary: ImportSummary = + checks.importable.length > 0 || attachOnly.length > 0 + ? await withProgress( + { total: checks.importable.length, verb: "created" }, + async (progress) => + importUsers({ + users: checks.importable, + secretKey, + limits, + record: run.append, + runId: run.record.id, + attachOnly, + adopted, + adoptedReserved: inFlightReserved, + skipPasswordRequirement: !options.requirePassword, + reserveUnverified, + progress, + }), + ) + : { + totalProcessed: 0, + successful: 0, + failed: 0, + notSent: 0, + droppedPhones: new Map(), + validationFailed: 0, + errorBreakdown: new Map(), + }; + // A Ctrl-C returns the import early, and the users it never sent have + // no line to count. Left unfinished, the run reads as interrupted. The + // signal handler prints nothing of ours, so the run line goes out here, + // and the throw closes the gutter as paused. + if (interruptedExitCode() !== null) { + run.release(); + log.info(`Stopped. Run ${run.record.id} records who was created: ${run.dir}`); + throwUserAbort(); + } + const record = run.finish({ notSent: summary.notSent }); + if (summary.failed > 0) process.exitCode = 1; + + if (options.json) { + log.data( + JSON.stringify( + { + target, + run: record, + resume: resume.kind, + checks: checksJson(checks), + ...leftOut, + result: { + created: summary.successful, + failed: summary.failed, + notSent: summary.notSent, + ...(summary.stopReason ? { stopReason: summary.stopReason } : {}), + skipped: checks.rejects.length + withoutPassword.length, + errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), + warnings: [...summary.droppedPhones].map(([reason, count]) => ({ + warning: `imported without their phone: ${reason}`, + count, + })), + }, + }, + null, + 2, + ), + ); + return; + } + + log.blank(); + for (const line of formatSummary( + summary, + checks.rejects.length + withoutPassword.length, + record, + run.dir, + limits.instanceType, + )) { + log.info(line); + } + if (record.status === "complete") { + log.blank(); + for (const line of cleanupLines(runsDir, record)) log.info(line); + } + + const steps = + summary.failed > 0 + ? NEXT_STEPS.MIGRATE_DONE_WITH_ERRORS(record.id) + : NEXT_STEPS.MIGRATE_DONE(record.id); + setNextSteps(steps); + printAgentNextSteps(steps); + }, + { skip: Boolean(options.json) }, + ); +} diff --git a/packages/cli-core/src/commands/migrate/runs.test.ts b/packages/cli-core/src/commands/migrate/runs.test.ts new file mode 100644 index 000000000..5a751b970 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/runs.test.ts @@ -0,0 +1,110 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { startRun } from "./lib/run-store.ts"; +import { runs } from "./runs.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; + +beforeEach(() => { + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-runs-"))); +}); + +afterEach(() => { + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +function partialImport() { + const run = startRun(runsDir, { + kind: "import", + target: { appLabel: "My App", env: "development", instanceId: "ins_1" }, + source: "clerk", + file: { path: "/tmp/users.json", sha256: "abc" }, + }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "failed", error: "That email address is taken." }); + run.append({ sourceId: "c", status: "failed", error: "That email address is taken." }); + run.append({ sourceId: "d", status: "skipped", reason: "no password (--require-password)" }); + return run.finish(); +} + +describe("runs", () => { + test("names the runs folder, then lists each run", async () => { + const record = partialImport(); + + await runs(undefined, { runsDir }); + + expect(captured.err).toContain(`Runs folder: ${runsDir}`); + expect(captured.err).toContain(record.id); + expect(captured.err).toContain("My App (development, ins_1)"); + expect(captured.err).toContain("1 created, 2 failed, 1 skipped"); + }); + + // Counts are written at the finish, which an interrupted run never reaches. + test("counts an interrupted run from its user lines", async () => { + const run = startRun(runsDir, { + kind: "import", + target: { env: "development", instanceId: "ins_1" }, + source: "clerk", + }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "created", clerkId: "user_b" }); + fs.rmSync(path.join(run.dir, "lock")); + + await runs(undefined, { runsDir }); + + expect(captured.err).toContain("interrupted"); + expect(captured.err).toContain("2 created"); + }); + + test("says so when there are no runs", async () => { + await runs(undefined, { runsDir }); + expect(captured.err).toContain("No migration runs yet."); + }); + + test("--json lists the runs on stdout, with each one's state", async () => { + const record = partialImport(); + + await runs(undefined, { runsDir, json: true }); + + const parsed = JSON.parse(captured.out) as { runsDir: string; runs: { id: string }[] }; + expect(parsed).toMatchObject({ runsDir, runs: [{ id: record.id, state: "partial" }] }); + }); +}); + +describe("runs ", () => { + test("shows the counts, the error breakdown and who did not make it", async () => { + const record = partialImport(); + + await runs(record.id, { runsDir }); + + expect(captured.err).toContain("partial"); + expect(Bun.stripANSI(captured.err)).toContain("2 users: That email address is taken."); + expect(captured.err).toContain("Failed (2)"); + expect(captured.err).toContain("Skipped (1)"); + expect(captured.err).toContain("no password (--require-password)"); + expect(captured.err).toContain(path.join(runsDir, record.id, "users.ndjson")); + }); + + test("--json returns the run, its errors, and the failed and skipped users", async () => { + const record = partialImport(); + + await runs(record.id, { runsDir, json: true }); + + const parsed = JSON.parse(captured.out) as Record; + expect(parsed).toMatchObject({ + run: { id: record.id, state: "partial", counts: { created: 1, failed: 2, skipped: 1 } }, + errors: [{ error: "That email address is taken.", count: 2 }], + failed: [{ sourceId: "b" }, { sourceId: "c" }], + skipped: [{ sourceId: "d" }], + }); + }); + + test("an unknown ID is a usage error", async () => { + await expect(runs("nope", { runsDir })).rejects.toThrow(/No run `nope`/); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/runs.ts b/packages/cli-core/src/commands/migrate/runs.ts new file mode 100644 index 000000000..5c9a70d9f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/runs.ts @@ -0,0 +1,246 @@ +/** + * `clerk migrate runs [run-id]` — what every import, export and undo did. + * + * Reads the run store and nothing else. `runs` alone lists every run, newest + * first. `runs ` shows one: its header, counts, the error breakdown, the + * users that did not make it, and the command to run next. + */ + +import path from "node:path"; +import { bold, dim, green, red, yellow } from "../../lib/color.ts"; +import { throwUsageError } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { + countLines, + latestUserLines, + listRuns, + readRun, + resolveRunsDir, + runDir, + runState, + type RunRecord, + type RunState, + type UserLine, +} from "./lib/run-store.ts"; +import { describeTarget } from "./lib/target.ts"; + +export type RunsOptions = { + json?: boolean; + runsDir?: string; +}; + +/** How many failed or skipped users `runs ` prints before pointing at the file. */ +const USER_LIST_LIMIT = 20; + +function formatDate(iso: string): string { + const date = new Date(iso); + if (Number.isNaN(date.getTime())) return iso; + return date.toLocaleString(undefined, { dateStyle: "medium", timeStyle: "short" }); +} + +function colorState(state: RunState): string { + if (state === "complete") return green(state); + if (state === "partial" || state === "interrupted") return yellow(state); + if (state === "undone") return dim(state); + return state; +} + +/** The counts that matter for a run's kind, in the order they are read. */ +export function formatCounts(record: RunRecord): string { + const counts = record.counts; + const order = + record.kind === "export" + ? (["exported", "skipped"] as const) + : record.kind === "undo" + ? (["deleted", "failed"] as const) + : (["created", "failed", "skipped"] as const); + const parts = order + .filter((status) => (counts[status] ?? 0) > 0 || status === order[0]) + .map((status) => `${counts[status] ?? 0} ${status}`); + return parts.join(", "); +} + +function fileLabel(record: RunRecord): string { + if (!record.file) return ""; + const relative = path.relative(process.cwd(), record.file.path); + return relative.startsWith("..") ? record.file.path : relative; +} + +function pad(rows: string[][]): string[] { + const widths = rows[0]!.map((_, column) => + Math.max(...rows.map((row) => Bun.stringWidth(row[column] ?? ""))), + ); + return rows.map((row) => + row + .map((cell, column) => + column === row.length - 1 + ? cell + : cell + " ".repeat(widths[column]! - Bun.stringWidth(cell)), + ) + .join(" ") + .trimEnd(), + ); +} + +/** Error messages grouped the way the import summary groups them. */ +function errorBreakdown(lines: UserLine[]): { error: string; count: number }[] { + const counts = new Map(); + for (const line of lines) { + if (line.status !== "failed" || !line.error) continue; + const key = normalizeErrorMessage(line.error); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts].map(([error, count]) => ({ error, count })).sort((a, b) => b.count - a.count); +} + +/** What to run after reading about this run. */ +export function nextCommands(record: RunRecord, state: RunState): string[] { + if (state === "running") return []; + if (record.kind === "import" && state !== "undone") { + return [`Run \`clerk migrate undo ${record.id}\` to delete the users it created`]; + } + if (record.kind === "undo" && state !== "complete" && record.undoes) { + return [`Run \`clerk migrate undo ${record.undoes}\` to retry the users that failed`]; + } + return []; +} + +function listJson(runsDir: string, records: RunRecord[]) { + return { + runsDir, + runs: records.map((record) => ({ ...record, state: runState(runsDir, record) })), + }; +} + +function printList(runsDir: string, records: RunRecord[]): void { + log.info(`Runs folder: ${runsDir}`); + log.blank(); + + if (records.length === 0) { + log.info("No migration runs yet."); + return; + } + + const rows = [ + ["RUN", "DATE", "KIND", "STATUS", "TARGET", "FILE", "RESULT"].map((header) => bold(header)), + ...records.map((record) => [ + record.id, + formatDate(record.startedAt), + record.kind, + colorState(runState(runsDir, record)), + describeTarget(record.target), + fileLabel(record), + formatCounts(record), + ]), + ]; + for (const line of pad(rows)) log.info(line); + log.blank(); + log.info(dim(`${records.length} run${records.length === 1 ? "" : "s"}`)); +} + +function showJson(runsDir: string, record: RunRecord) { + const lines = [...latestUserLines(runsDir, record.id).values()]; + return { + runsDir, + run: { ...record, state: runState(runsDir, record) }, + next: nextCommands(record, runState(runsDir, record)), + errors: errorBreakdown(lines), + failed: lines.filter((line) => line.status === "failed"), + skipped: lines.filter((line) => line.status === "skipped"), + }; +} + +function printUsers(title: string, lines: UserLine[], usersFile: string): void { + if (lines.length === 0) return; + log.blank(); + log.info(bold(`${title} (${lines.length})`)); + for (const line of lines.slice(0, USER_LIST_LIMIT)) { + const why = line.reason ?? line.error ?? ""; + log.info(` ${line.sourceId}${why ? dim(` ${why}`) : ""}`); + } + if (lines.length > USER_LIST_LIMIT) { + log.info(dim(` …and ${lines.length - USER_LIST_LIMIT} more in ${usersFile}`)); + } +} + +function printRun(runsDir: string, record: RunRecord): void { + const state = runState(runsDir, record); + const dir = runDir(runsDir, record.id); + const usersFile = path.join(dir, "users.ndjson"); + const lines = [...latestUserLines(runsDir, record.id).values()]; + + log.info(`Runs folder: ${runsDir}`); + log.blank(); + const header: [string, string | undefined][] = [ + ["Run", record.id], + ["Kind", record.kind], + ["Status", colorState(state)], + ["Started", formatDate(record.startedAt)], + ["Finished", record.finishedAt ? formatDate(record.finishedAt) : undefined], + ["Target", describeTarget(record.target)], + ["Key from", record.target.keySource], + ["Source", record.source], + ["File", record.file?.path], + ["From export", record.fromExport], + ["Undoes", record.undoes], + ["Undone by", record.undoneBy], + ["Result", formatCounts(record)], + ]; + const shown = header.filter((entry): entry is [string, string] => Boolean(entry[1])); + const width = Math.max(...shown.map(([label]) => label.length)) + 2; + for (const [label, value] of shown) log.info(`${bold(label.padEnd(width))}${value}`); + + const errors = errorBreakdown(lines); + if (errors.length > 0) { + log.blank(); + log.info(bold("Error breakdown")); + for (const { error, count } of errors) { + log.info(` ${red(String(count))} ${count === 1 ? "user" : "users"}: ${error}`); + } + } + + printUsers( + "Failed", + lines.filter((line) => line.status === "failed"), + usersFile, + ); + printUsers( + "Skipped", + lines.filter((line) => line.status === "skipped"), + usersFile, + ); + + log.blank(); + log.info(dim(`Every outcome: ${usersFile}`)); + for (const step of nextCommands(record, state)) log.info(` → ${step}`); +} + +/** + * Counts are written when a run finishes, so an interrupted one still holds + * its start's zeros. Those are counted from its user lines instead. + */ +function withLiveCounts(runsDir: string, record: RunRecord): RunRecord { + if (record.finishedAt) return record; + return { ...record, counts: countLines(latestUserLines(runsDir, record.id).values()) }; +} + +export async function runs(id: string | undefined, options: RunsOptions = {}): Promise { + const runsDir = await resolveRunsDir(options.runsDir); + + if (id === undefined) { + const records = listRuns(runsDir).map((record) => withLiveCounts(runsDir, record)); + if (options.json) log.data(JSON.stringify(listJson(runsDir, records), null, 2)); + else printList(runsDir, records); + return; + } + + const stored = readRun(runsDir, id); + if (!stored) { + throwUsageError(`No run \`${id}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + const record = withLiveCounts(runsDir, stored); + + if (options.json) log.data(JSON.stringify(showJson(runsDir, record), null, 2)); + else printRun(runsDir, record); +} diff --git a/packages/cli-core/src/commands/migrate/sources/auth0.ts b/packages/cli-core/src/commands/migrate/sources/auth0.ts new file mode 100644 index 000000000..4f6f1fbdd --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/auth0.ts @@ -0,0 +1,65 @@ +import type { SourceEntry } from "../types.ts"; +import { isVerified, routeByVerification, splitName } from "./shared.ts"; + +/** + * Auth0 → Clerk transformer. + * + * Works with Auth0's Export Users API. `user_id` is a `provider|id` string + * (`auth0|abc123`, `github|12345`) and is carried through as the Clerk user's + * `external_id`. + * + * Auth0 does not include password hashes in a standard export — they have to + * be requested from Auth0 support. When present they are bcrypt (`$2a$`/`$2b$`, + * 10 rounds), which is why `passwordHasher` defaults to `bcrypt`. + */ +const auth0Source = { + key: "auth0", + label: "Auth0", + description: + "Works with Auth0's Export Users API. Password hashes require a support request to Auth0.", + carries: { + passwords: { + level: "partial", + note: "Auth0 releases bcrypt hashes only through a support request. Add each as `passwordHash` before importing.", + }, + mfa: { level: "no", note: "Auth0 exports no MFA enrolments. Users enrol again in Clerk." }, + metadata: { + level: "yes", + note: "`user_metadata` → unsafe metadata, which users can edit, as in Auth0. `app_metadata` → private metadata.", + }, + }, + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + username: "username", + name: "name", + blocked: "banned", + given_name: "firstName", + family_name: "lastName", + phone_number: "phone", + phone_verified: "phoneVerified", + passwordHash: "password", + user_metadata: "unsafeMetadata", + app_metadata: "privateMetadata", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + + // given_name/family_name win when present. Auth0 fills `name` with the + // email for database users; splitName drops an email-shaped value. + if (user.firstName || user.lastName) delete user.name; + else splitName(user); + + // Runs before normalizeUserData, so accept CSV's "true" as well. + if (isVerified(user.banned, "boolean")) user.banned = true; + else delete user.banned; + }, + defaults: { + passwordHasher: "bcrypt" as const, + }, +} satisfies SourceEntry; + +export default auth0Source; diff --git a/packages/cli-core/src/commands/migrate/sources/authjs.ts b/packages/cli-core/src/commands/migrate/sources/authjs.ts new file mode 100644 index 000000000..89d814d8b --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/authjs.ts @@ -0,0 +1,49 @@ +import type { SourceEntry } from "../types.ts"; +import { routeByVerification, splitName } from "./shared.ts"; + +/** + * Auth.js (formerly NextAuth) → Clerk transformer. + * + * Auth.js has no export tool and no fixed user table, so this assumes the + * common shape: `SELECT id, name, email, email_verified, created_at FROM users`. + * A different schema means editing the mapping below or supplying a custom + * transformer file. + * + * `email_verified` is a nullable timestamp rather than a boolean — any value + * means verified. + * + * No password default: Auth.js core stores no passwords, so users arrive + * without a digest and are imported with `skip_password_requirement`. An app + * using the Credentials provider keeps its own, which this does not read. + */ +const authjsSource = { + key: "authjs", + label: "Auth.js (NextAuth)", + description: + "Assumes an export of `SELECT id, name, email, email_verified, created_at FROM users`. `name` is split into firstName and lastName.", + carries: { + passwords: { + level: "no", + note: "Auth.js core stores no passwords. An app using the Credentials provider keeps its own, which this does not read: migrate them separately, or have those users reset their password.", + }, + mfa: { level: "no", note: "Auth.js has no MFA of its own." }, + metadata: { + level: "no", + note: "Only `id`, `name`, `email`, `email_verified` and `created_at` are read.", + }, + }, + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + created_at: "createdAt", + updated_at: "updatedAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "timestamp"); + splitName(user); + }, +} satisfies SourceEntry; + +export default authjsSource; diff --git a/packages/cli-core/src/commands/migrate/sources/betterauth.ts b/packages/cli-core/src/commands/migrate/sources/betterauth.ts new file mode 100644 index 000000000..f6ec33b3c --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/betterauth.ts @@ -0,0 +1,146 @@ +import type { SourceEntry } from "../types.ts"; +import { + detectStandardHasher, + isVerified, + routeByVerification, + splitName, + toIsoDate, +} from "./shared.ts"; + +/** + * Better Auth → Clerk source. + * + * Works with `clerk migrate export betterauth`, which joins the user table + * with the credential account row to pick up the stored `password_hash`. + * + * Better Auth plugins add columns Clerk has no equivalent for + * (`display_username`, `role`, `ban_reason`, `two_factor_enabled`). They need + * no handling: the schema strips anything it does not declare. `banned` is the + * exception, because that one *is* a Clerk field. + */ + +/** Better Auth's own scrypt: `<32 hex salt>:<128 hex key>`, N=16384, r=16, p=1. */ +const BETTER_AUTH_SCRYPT = /^([0-9a-f]{32}):([0-9a-f]{128})$/i; + +/** + * Works out which hasher produced a stored Better Auth password, one user at + * a time. + * + * Better Auth hashes with its own scrypt by default and lets an app swap in + * bcrypt or argon2, so one database can hold more than one kind. Better Auth's + * scrypt uses the hex salt string itself as the salt and a 64-byte key, which + * is exactly what `scrypt_werkzeug` verifies once the parameters are written + * inline. + * + * @returns The digest and hasher to send, or `undefined` when the hash is not + * one Clerk can verify. + */ +export function detectBetterAuthHash( + hash: string, +): + | { password: string; passwordHasher: "scrypt_werkzeug" | "bcrypt" | "argon2id" | "argon2i" } + | undefined { + const scrypt = BETTER_AUTH_SCRYPT.exec(hash); + if (scrypt) { + return { + password: `scrypt:16384:16:1$${scrypt[1]}$${scrypt[2]}`, + passwordHasher: "scrypt_werkzeug", + }; + } + const passwordHasher = detectStandardHasher(hash); + return passwordHasher ? { password: hash, passwordHasher } : undefined; +} + +const betterAuthSource = { + key: "betterauth", + label: "Better Auth", + description: + "Works with the Better Auth export. Detects scrypt, bcrypt and argon2 passwords per user, and carries the admin plugin's banned flag while its ban has not expired.", + carries: { + passwords: { + level: "yes", + note: "Better Auth's scrypt, bcrypt and argon2 hashes come across, detected per user. Any other hash is dropped, and that user resets their password.", + }, + mfa: { + level: "no", + note: "The two-factor plugin's secrets are not exported. Users enrol again in Clerk.", + }, + metadata: { + level: "no", + note: "Plugin columns such as `role` have no Clerk equivalent and are left out.", + }, + }, + caveats: [ + "Better Auth's scrypt normalizes a password to NFKC before hashing it; Clerk hashes it as typed. A password whose NFKC form differs — full-width characters, some ligatures — will not verify, and that user resets it.", + ], + transformer: { + user_id: "userId", + email: "email", + email_verified: "emailVerified", + name: "name", + password_hash: "password", + username: "username", + phone_number: "phone", + phone_number_verified: "phoneVerified", + created_at: "createdAt", + }, + postTransform: (user) => { + // Every Better Auth row has one, and Clerk sets its own. + delete user.updated_at; + + routeByVerification(user, "email", "emailVerified", "boolean"); + routeByVerification(user, "phone", "phoneVerified", "boolean"); + splitName(user); + + if (typeof user.password === "string" && user.password) { + const detected = detectBetterAuthHash(user.password); + if (detected) { + user.password = detected.password; + user.passwordHasher = detected.passwordHasher; + } else { + // Imported without it rather than rejected: the user can still sign + // in another way, or reset it. + delete user.password; + user.passwordDropped = true; + } + } + + // Only carry `banned` when it is actually true — Better Auth writes false + // for every user that was never banned, and sending that to Clerk is noise. + // SQLite, libSQL and MySQL hand back 1/0 and CSV hands back "true", so this + // runs before normalizeUserData and must accept those too. + // A ban whose `banExpires` has passed is over: Better Auth lifts it only + // at the user's next sign-in, so the column still says banned. An expiry + // that can't be read keeps the ban. + // Epoch seconds arrive as a number from the database, and as a string of + // digits from a CSV of the same column; both need the seconds read as such. + const rawExpiry = user.banExpires ?? user.ban_expires; + const epoch = + typeof rawExpiry === "number" + ? rawExpiry + : typeof rawExpiry === "string" && /^\d+$/.test(rawExpiry.trim()) + ? Number(rawExpiry) + : undefined; + const expiry = epoch !== undefined && epoch < 1e11 ? epoch * 1000 : (epoch ?? rawExpiry); + const endsAt = Date.parse(String(toIsoDate(expiry, true))); + const expired = endsAt <= Date.now(); + if (isVerified(user.banned, "boolean") && !expired) { + user.banned = true; + // Clerk's ban has no end, so this one's is kept for the checks to warn + // about. No expiry is Better Auth's permanent ban. + if (Number.isFinite(endsAt)) user.banEndsAt = new Date(endsAt).toISOString(); + } else delete user.banned; + delete user.banExpires; + delete user.ban_expires; + + // The anonymous plugin's guests are throwaway accounts with placeholder + // emails (anon-…@…), not people to migrate. + if (isVerified(user.isAnonymous ?? user.is_anonymous, "boolean")) { + user.skipReason = "anonymous Better Auth user"; + } + delete user.isAnonymous; + delete user.is_anonymous; + }, +} satisfies SourceEntry; + +export default betterAuthSource; diff --git a/packages/cli-core/src/commands/migrate/sources/clerk.ts b/packages/cli-core/src/commands/migrate/sources/clerk.ts new file mode 100644 index 000000000..6c63ef062 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/clerk.ts @@ -0,0 +1,85 @@ +import type { SourceEntry } from "../types.ts"; + +/** + * Clerk → Clerk transformer, for moving users between Clerk instances + * (typically development → production). + * + * Maps the Dashboard's user export format onto the import schema. + */ +const clerkSource = { + key: "clerk", + label: "Clerk", + description: + "Migrate between Clerk instances (e.g. development to production, or to another Clerk application). Export your users from the Clerk Dashboard first.", + carries: { + passwords: { + level: "partial", + note: "A Dashboard export carries each digest and its hasher. `clerk migrate export clerk` cannot: the Backend API never returns them.", + }, + mfa: { + level: "partial", + note: "TOTP secrets and backup codes come across from a Dashboard export only.", + }, + metadata: { + level: "partial", + note: "With `clerk migrate export clerk`, public, private and unsafe metadata keep their places; a Dashboard CSV carries no metadata, ban or legal acceptance. `export clerk` moves each user's external_id to private metadata as `clerkExternalId`, because the import uses external_id for the old Clerk ID.", + }, + }, + transformer: { + id: "userId", + primary_email_address: "email", + verified_email_addresses: "emailAddresses", + unverified_email_addresses: "unverifiedEmailAddresses", + first_name: "firstName", + last_name: "lastName", + password_digest: "password", + password_hasher: "passwordHasher", + primary_phone_number: "phone", + verified_phone_numbers: "phoneNumbers", + unverified_phone_numbers: "unverifiedPhoneNumbers", + username: "username", + totp_secret: "totpSecret", + backup_codes_enabled: "backupCodesEnabled", + backup_codes: "backupCodes", + public_metadata: "publicMetadata", + unsafe_metadata: "unsafeMetadata", + private_metadata: "privateMetadata", + // Account state a Dashboard export carries and `POST /v1/users` accepts. + // Unmapped, these survive the export and are then silently stripped at + // validation — losing original signup dates on a dev → prod migration. + created_at: "createdAt", + legal_accepted_at: "legalAcceptedAt", + banned: "banned", + create_organization_enabled: "createOrganizationEnabled", + create_organizations_limit: "createOrganizationsLimit", + delete_self_enabled: "deleteSelfEnabled", + }, + postTransform: (user) => { + // The Dashboard's CSV prefixes a TAB to any value starting with = + - @ + // (or their fullwidth forms), so a spreadsheet won't run it as a formula + // (clerk_go pkg/csvsafe). Undo it, or the TAB is imported. + for (const field of FORMULA_SAFE_FIELDS) { + const value = user[field]; + if (typeof value === "string") user[field] = unprefix(value); + else if (Array.isArray(value)) + user[field] = value.map((v) => (typeof v === "string" ? unprefix(v) : v)); + } + }, +} satisfies SourceEntry; + +const FORMULA_SAFE_FIELDS = [ + "firstName", + "lastName", + "username", + "email", + "emailAddresses", + "unverifiedEmailAddresses", + // A phone always starts with `+`, so a Dashboard CSV always prefixes it. + "phone", + "phoneNumbers", + "unverifiedPhoneNumbers", +] as const; + +const unprefix = (value: string) => value.replace(/^\t(?=[=+\-@\t\r\n＝＋－＠])/, ""); + +export default clerkSource; diff --git a/packages/cli-core/src/commands/migrate/sources/firebase.ts b/packages/cli-core/src/commands/migrate/sources/firebase.ts new file mode 100644 index 000000000..fa574b627 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/firebase.ts @@ -0,0 +1,123 @@ +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import { readJsonFile } from "../lib/export-file.ts"; +import type { PreTransformResult, SourceEntry } from "../types.ts"; +import { isVerified, routeByVerification, splitName, toIsoDate } from "./shared.ts"; + +/** + * Column order of `firebase auth:export --format=csv`, which writes no header + * row. Without these the CSV parser would treat the first user as the header. + */ +const FIREBASE_CSV_HEADERS = + "localId,email,emailVerified,passwordHash,passwordSalt,displayName,photoUrl," + + "googleId,googleEmail,googleDisplayName,googlePhotoUrl," + + "facebookId,facebookEmail,facebookDisplayName,facebookPhotoUrl," + + "twitterId,twitterEmail,twitterDisplayName,twitterPhotoUrl," + + "githubId,githubEmail,githubDisplayName,githubPhotoUrl," + + "createdAt,lastSignedInAt,phoneNumber,disabled,customAttributes,providerUserInfo"; + +/** + * Firebase → Clerk transformer. + * + * Handles both shapes `firebase auth:export` produces: a headerless CSV, and + * JSON wrapped in `{ users: [...] }`. + * + * Firebase's scrypt is a modified variant, so Clerk needs the project's four + * hash parameters alongside each digest. They arrive on the run's + * {@link TransformContext} from `--firebase-*` flags or the matching + * `CLERK_FIREBASE_*` environment variables; they are never persisted. + * + * See https://clerk.com/docs/guides/development/migrating/firebase + */ +const firebaseSource = { + key: "firebase", + label: "Firebase", + description: + "Works with `firebase auth:export` (CSV or JSON). Requires the project's four password hash parameters to migrate passwords.", + + preTransform: (filePath: string, fileType: string): PreTransformResult => { + // The CSV has no header row; name the columns rather than writing a copy + // with one, which would leave the hashes and salts in a temp file. + if (fileType === "text/csv") return { filePath, csvHeaders: FIREBASE_CSV_HEADERS.split(",") }; + + if (fileType === "application/json") { + const parsed = readJsonFile(filePath); + if (Array.isArray(parsed)) return { filePath, data: parsed as Record[] }; + + const users = (parsed as { users?: unknown })?.users; + if (Array.isArray(users)) return { filePath, data: users as Record[] }; + + throw new CliError( + "Invalid Firebase JSON export: expected `{ users: [...] }` or an array of users.", + { code: ERROR_CODE.INVALID_JSON }, + ); + } + + return { filePath }; + }, + + carries: { + passwords: { + level: "yes", + note: "scrypt hashes come across with the project's hash parameters, read by the export or passed as `--firebase-*`.", + }, + mfa: { level: "no", note: "Firebase exports no MFA enrolments. Users enrol again in Clerk." }, + metadata: { level: "no", note: "Custom claims are not exported." }, + }, + transformer: { + localId: "userId", + email: "email", + emailVerified: "emailVerified", + passwordHash: "passwordHash", + passwordSalt: "salt", + phoneNumber: "phone", + displayName: "name", + disabled: "banned", + }, + + postTransform: (user, context) => { + const passwordHash = user.passwordHash; + const salt = user.salt; + + if (passwordHash && salt) { + const config = context.firebaseHashConfig; + if (!config) { + throwUsageError( + "This export contains Firebase password hashes, which need the project's hash parameters to import.\n" + + "Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters, then pass:\n" + + " --firebase-signer-key --firebase-salt-separator --firebase-rounds --firebase-mem-cost", + "https://clerk.com/docs/guides/development/migrating/firebase", + ); + } + + // Clerk's scrypt_firebase hasher expects every parameter inline: + // hash$salt$signerKey$saltSeparator$rounds$memCost + user.password = [ + passwordHash, + salt, + config.base64_signer_key, + config.base64_salt_separator, + config.rounds, + config.mem_cost, + ].join("$"); + + delete user.passwordHash; + delete user.salt; + } + + routeByVerification(user, "email", "emailVerified", "boolean"); + // Firebase exports timestamps as Unix milliseconds, often as strings. + user.createdAt = toIsoDate(user.createdAt, true); + splitName(user); + + // A disabled Firebase account is Clerk's banned. Runs before + // normalizeUserData, so accept CSV's "true" as well. + if (isVerified(user.banned, "boolean")) user.banned = true; + else delete user.banned; + }, + + defaults: { + passwordHasher: "scrypt_firebase" as const, + }, +} satisfies SourceEntry; + +export default firebaseSource; diff --git a/packages/cli-core/src/commands/migrate/sources/list.test.ts b/packages/cli-core/src/commands/migrate/sources/list.test.ts new file mode 100644 index 000000000..8d6108adf --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.test.ts @@ -0,0 +1,174 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError, EXIT_CODE } from "../../../lib/errors.ts"; +import { getMode, setMode, type Mode } from "../../../mode.ts"; +import { useCaptureLog } from "../../../test/lib/stubs.ts"; +import { list, wrapText } from "./list.ts"; +import { ACCOUNT_LINKING_URL, sources } from "./registry.ts"; + +const captured = useCaptureLog(); + +/** `log.info` highlights backticked spans and the levels are coloured. */ +const plain = () => Bun.stripANSI(captured.err).replace(/\s+/g, " "); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); + fs.writeFileSync( + path.join(workDir, "custom.ts"), + `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", mail: "email" }, + carries: { + passwords: { level: "partial", note: "Only for accounts made after 2020." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "yes", note: "All of it." }, + }, + };`, + ); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe("sources", () => { + test.each([...sources])("lists $key with what it carries", async (entry) => { + await list(undefined); + const row = plain().match(new RegExp(`${entry.key} (\\w+) (\\w+) (\\w+)`)); + expect(row?.slice(1)).toEqual([ + entry.carries.passwords.level, + entry.carries.mfa.level, + entry.carries.metadata.level, + ]); + }); + + // No social column: every source shares one note. + test("prints the account-linking note once", async () => { + await list(undefined); + expect(captured.err.split(ACCOUNT_LINKING_URL)).toHaveLength(2); + }); + + // A compiled binary has no source tree to grep, so the way to extend it has + // to be discoverable from the list itself. + test("says how to add one", async () => { + await list(undefined); + expect(captured.err).toContain("--source ./my-source.ts"); + }); + + test("--json lists every built-in with its carries on stdout", async () => { + await list(undefined, { json: true }); + + const parsed = JSON.parse(captured.out) as { + sources: { key: string; carries: unknown }[]; + account_linking: string; + }; + expect(parsed.sources.map((entry) => entry.key)).toEqual(sources.map((entry) => entry.key)); + expect(parsed.sources[0]?.carries).toEqual(sources[0]?.carries); + expect(parsed.account_linking).toContain(ACCOUNT_LINKING_URL); + }); +}); + +describe("sources ", () => { + test("shows where each field lands, how to export, and what comes across", async () => { + await list("supabase"); + + expect(plain()).toContain("Export with `clerk migrate export supabase`"); + expect(plain()).toContain("encrypted_password → password"); + expect(plain()).toContain("Passwords yes"); + // Supabase's hasher is read from each digest, so it lists no fixed one. + expect(plain()).not.toContain("passwordHasher is always"); + expect(captured.err).toContain(ACCOUNT_LINKING_URL); + }); + + test("lists a source's fixed defaults", async () => { + await list("auth0"); + + expect(plain()).toContain('passwordHasher is always "bcrypt"'); + }); + + test("shows a source you wrote, by its path", async () => { + await list("./custom.ts"); + + expect(plain()).toContain("myplatform My Platform (custom — ./custom.ts)"); + expect(plain()).toContain("mail → email"); + expect(plain()).toContain("Only for accounts made after 2020."); + }); + + test("--json returns the detail on stdout", async () => { + await list("auth0", { json: true }); + + expect(JSON.parse(captured.out)).toMatchObject({ + key: "auth0", + export_command: "clerk migrate export auth0", + fields: { user_id: "userId" }, + carries: { passwords: { level: "partial" } }, + }); + }); + + test("an unknown source is a usage error naming the valid ones", async () => { + const error = (await list("okta").catch((caught: unknown) => caught)) as CliError; + expect(error).toBeInstanceOf(CliError); + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Valid sources: clerk, auth0"); + }); +}); + +describe("human-mode frame", () => { + let originalMode: Mode; + + beforeAll(() => { + originalMode = getMode(); + setMode("human"); + }); + + afterAll(() => { + setMode(originalMode); + }); + + // Reading a static registry is not a run: there is no progress to bracket, + // and the gutter's `│` would sit in front of every wrapped line. + test("prints no intro/outro gutter", async () => { + await list(undefined); + + expect(captured.err).not.toContain("┌"); + expect(captured.err).not.toContain("└"); + }); + + test("--json stays on stdout only", async () => { + await list(undefined, { json: true }); + + expect(() => JSON.parse(captured.out)).not.toThrow(); + expect(captured.err).toBe(""); + }); +}); + +describe("wrapText", () => { + test("breaks on whitespace within the width", () => { + expect(wrapText("one two three four", 9)).toEqual(["one two", "three", "four"]); + }); + + // `log.info` pairs backticks per line, so a span split across two lines + // leaves one unmatched backtick on each and colours the wrong half of both. + test("never breaks inside a backticked span", () => { + const lines = wrapText("Assumes an export of `SELECT id, name FROM users`.", 30); + + expect(lines).toContain("`SELECT id, name FROM users`."); + for (const line of lines) { + expect((line.match(/`/g) ?? []).length % 2).toBe(0); + } + }); + + test("gives an over-long word its own line rather than dropping it", () => { + expect(wrapText("short supercalifragilistic", 8)).toEqual(["short", "supercalifragilistic"]); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/list.ts b/packages/cli-core/src/commands/migrate/sources/list.ts new file mode 100644 index 000000000..7991bb05a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/list.ts @@ -0,0 +1,202 @@ +/** + * `clerk migrate sources [source]` — which platforms an import can read, and + * what each one brings across. + * + * A compiled binary's users have no source tree to read the mappings in, so + * this is where they are shown. `sources` alone lists every source with its + * passwords, MFA and metadata at a glance. `sources ` shows one in + * full: what each field becomes, how to export from it, and its caveats. + */ + +import { bold, cyan, dim, green, red, yellow } from "../../../lib/color.ts"; +import { log } from "../../../lib/log.ts"; +import { exportPlatforms } from "../export/registry.ts"; +import type { Carry, CarryLevel, SourceEntry } from "../types.ts"; +import { ACCOUNT_LINKING_NOTE, resolveSource, sources as builtIns } from "./registry.ts"; + +export type SourcesOptions = { + json?: boolean; +}; + +const KINDS = [ + ["passwords", "Passwords"], + ["mfa", "MFA"], + ["metadata", "Metadata"], +] as const; + +/** + * Capped, not just measured: a description that rewrapped differently on every + * terminal makes two runs of the same command look like different output. 80 is + * the same width `--help` lays itself out at. + */ +const MAX_WIDTH = 80; + +function outputWidth(): number { + return Math.min(process.stderr.columns || MAX_WIDTH, MAX_WIDTH); +} + +/** + * A run of non-space characters, except that a backticked span counts as one + * character run even when it contains spaces. Keeps `SELECT a, b FROM users` + * whole: `log.info` pairs backticks per line, so a span broken across two lines + * leaves an unmatched backtick on each and colours the wrong half of both. + */ +const WORD = /(?:`[^`]*`|\S)+/g; + +/** + * Wraps on whitespace. Safe to measure raw because the backtick spans + * `log.info` highlights keep their backticks — the colour it adds is invisible + * to width, and nothing here is coloured before wrapping. + */ +export function wrapText(text: string, width: number): string[] { + const lines: string[] = []; + let line = ""; + + for (const word of text.match(WORD) ?? []) { + if (!line) line = word; + else if (line.length + 1 + word.length <= width) line += ` ${word}`; + else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + + return lines; +} + +function levelMark(level: CarryLevel): string { + if (level === "yes") return green("yes"); + if (level === "partial") return yellow("partial"); + return red("no"); +} + +/** The command that writes a file this source reads, when there is one. */ +function exportCommand(entry: SourceEntry): string | undefined { + const platform = exportPlatforms.find((candidate) => candidate.sourceKey === entry.key); + return platform ? `clerk migrate export ${platform.key}` : undefined; +} + +function toJson(entry: SourceEntry, custom?: string) { + return { + key: entry.key, + label: entry.label, + description: entry.description, + ...(custom ? { path: custom } : {}), + carries: entry.carries, + export_command: exportCommand(entry) ?? null, + fields: entry.transformer, + caveats: entry.caveats ?? [], + }; +} + +function printList(width: number): void { + for (const line of wrapText( + "A source maps one platform's export onto the fields Clerk imports. A file from " + + "`clerk migrate export` names its own; anything else takes `--source `.", + width, + )) { + log.info(line); + } + log.blank(); + + const keyWidth = Math.max(...builtIns.map((entry) => entry.key.length)) + 2; + log.info( + bold( + `${"SOURCE".padEnd(keyWidth)}${"PASSWORDS".padEnd(11)}${"MFA".padEnd(9)}${"METADATA".padEnd(10)}`, + ), + ); + for (const entry of builtIns) { + const cell = (carry: Carry, pad: number) => + levelMark(carry.level) + " ".repeat(Math.max(1, pad - carry.level.length)); + log.info( + `${cyan(entry.key.padEnd(keyWidth))}${cell(entry.carries.passwords, 11)}` + + `${cell(entry.carries.mfa, 9)}${levelMark(entry.carries.metadata.level)}`, + ); + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); + log.blank(); + log.info("Run `clerk migrate sources ` for what each field becomes."); + log.info("Migrating from something else? Write a source and pass --source ./my-source.ts."); +} + +function printDetail(entry: SourceEntry, width: number, custom?: string): void { + log.info(`${cyan(bold(entry.key))} ${entry.label}${custom ? dim(` (custom — ${custom})`) : ""}`); + for (const line of wrapText(entry.description, width)) log.info(line); + + const command = exportCommand(entry); + if (command) { + log.blank(); + log.info(`${bold("Export with")} \`${command}\``); + } + + log.blank(); + log.info(bold("What comes across")); + for (const [kind, label] of KINDS) { + const carry = entry.carries[kind]; + log.info(` ${label.padEnd(10)}${levelMark(carry.level)}`); + for (const line of wrapText(carry.note, width - 4)) log.info(` ${dim(line)}`); + } + + const fields = Object.entries(entry.transformer); + const fromWidth = Math.max(...fields.map(([from]) => from.length)) + 2; + log.blank(); + log.info(bold("Where each field lands")); + for (const [from, to] of fields) log.info(` ${from.padEnd(fromWidth)}→ ${to}`); + if (entry.defaults) { + for (const [field, value] of Object.entries(entry.defaults)) { + log.info(dim(` ${field} is always ${JSON.stringify(value)}`)); + } + } + + if (entry.caveats?.length) { + log.blank(); + log.info(bold("Caveats")); + for (const caveat of entry.caveats) { + const [first, ...rest] = wrapText(caveat, width - 4); + log.info(` - ${first ?? ""}`); + for (const line of rest) log.info(` ${line}`); + } + } + + log.blank(); + for (const line of wrapText(ACCOUNT_LINKING_NOTE, width)) log.info(dim(line)); +} + +export async function list(key: string | undefined, options: SourcesOptions = {}): Promise { + const width = outputWidth(); + + if (key === undefined) { + if (options.json) { + log.data( + JSON.stringify( + { + sources: builtIns.map((entry) => toJson(entry)), + account_linking: ACCOUNT_LINKING_NOTE, + }, + null, + 2, + ), + ); + return; + } + printList(width); + return; + } + + // No gutter: this reads a static registry, it does not run anything. + const resolved = await resolveSource(key); + if (options.json) { + log.data( + JSON.stringify( + { ...toJson(resolved.entry, resolved.path), account_linking: ACCOUNT_LINKING_NOTE }, + null, + 2, + ), + ); + return; + } + printDetail(resolved.entry, width, resolved.path ? key : undefined); +} diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts new file mode 100644 index 000000000..513457375 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.test.ts @@ -0,0 +1,250 @@ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadCustomSource, validateSource } from "./load-custom.ts"; +import { __resetCustomSourcesForTesting } from "./registry.ts"; + +let workDir: string; +let originalCwd: string; +let counter = 0; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-custom-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +afterEach(() => { + __resetCustomSourcesForTesting(); +}); + +/** + * Writes a source file with a unique name. + * + * Names must not repeat: a dynamic `import()` caches by URL, so reusing one + * would silently return the previous test's module. + */ +function writeSource(source: string, ext = "ts"): string { + const name = `custom-${counter++}.${ext}`; + fs.writeFileSync(path.join(workDir, name), source); + return `./${name}`; +} + +const VALID = `export default { + key: "myplatform", + label: "My Platform", + description: "Exports from My Platform.", + transformer: { account_ref: "userId", contact_email: "email" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, +};`; + +describe("loadCustomSource", () => { + test("loads a user-authored TypeScript source", async () => { + const entry = await loadCustomSource(writeSource(VALID)); + + expect(entry).toMatchObject({ + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId", contact_email: "email" }, + }); + }); + + test("loads plain JavaScript too", async () => { + const entry = await loadCustomSource(writeSource(VALID, "js")); + expect(entry.key).toBe("myplatform"); + }); + + // The file is the user's own code; the CLI must transpile whatever they wrote. + test("transpiles TypeScript syntax the runtime has to strip", async () => { + const entry = await loadCustomSource( + writeSource(` + interface Entry { key: string; label: string; transformer: Record; carries: unknown } + const mapping = { my_id: "userId" } as const; + const custom: Entry = { key: "tsplatform", label: "TS", transformer: { ...mapping }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } }; + export default custom satisfies Entry; + `), + ); + expect(entry.key).toBe("tsplatform"); + }); + + test("carries the optional hooks through", async () => { + const entry = await loadCustomSource( + writeSource(`export default { + key: "hooked", label: "Hooked", + transformer: { id: "userId" }, + carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } }, + defaults: { passwordHasher: "bcrypt" }, + postTransform: (user) => { user.firstName = "set"; }, + };`), + ); + + expect(entry.defaults).toEqual({ passwordHasher: "bcrypt" }); + const user: Record = {}; + entry.postTransform?.(user, {}); + expect(user.firstName).toBe("set"); + }); + + test("supplies a description when the author omitted one", async () => { + const entry = await loadCustomSource( + writeSource( + `export default { key: "bare", label: "Bare", transformer: { id: "userId" }, carries: { passwords: { level: "no", note: "-" }, mfa: { level: "no", note: "-" }, metadata: { level: "no", note: "-" } } };`, + ), + ); + expect(entry.description).toBe("Custom source"); + }); + + test("reports a path that is not there", async () => { + await expect(loadCustomSource("./nope.ts")).rejects.toThrow(/No source file at/); + }); + + test("reports a directory given instead of a file", async () => { + fs.mkdirSync(path.join(workDir, "adir"), { recursive: true }); + await expect(loadCustomSource("./adir")).rejects.toThrow(/is a directory/); + }); + + test("reports a file that does not parse, quoting the syntax error", async () => { + await expect(loadCustomSource(writeSource("export default { key: ,,, }"))).rejects.toThrow( + /Could not load/, + ); + }); + + test("reports a file that throws while loading", async () => { + await expect( + loadCustomSource(writeSource(`throw new Error("boom"); export default {};`)), + ).rejects.toThrow(/Could not load .*boom/s); + }); + + test("points at a named export when the default is missing", async () => { + const file = writeSource( + `export const myPlatform = { key: "x", label: "X", transformer: { a: "userId" } };`, + ); + + await expect(loadCustomSource(file)).rejects.toThrow( + /has no default export.*`myPlatform`.*did you mean `export default`/s, + ); + }); + + test("reports a missing default with no named exports to suggest", async () => { + await expect(loadCustomSource(writeSource("const unused = 1;"))).rejects.toThrow( + /has no default export\.$/m, + ); + }); +}); + +describe("validateSource", () => { + const valid = { + key: "myplatform", + label: "My Platform", + transformer: { account_ref: "userId" }, + carries: { + passwords: { level: "no", note: "None." }, + mfa: { level: "no", note: "None." }, + metadata: { level: "no", note: "None." }, + }, + }; + + test("accepts a minimal valid entry", () => { + expect(validateSource(valid, "f.ts").key).toBe("myplatform"); + }); + + test.each([ + ["a null default export", null, /is null, not an object/], + ["a number default export", 42, /is number, not an object/], + ["a string default export", "nope", /is string, not an object/], + ])("rejects %s", (_label, value, expected) => { + expect(() => validateSource(value, "f.ts")).toThrow(expected); + }); + + test.each([ + ["key", { ...valid, key: undefined }], + ["key", { ...valid, key: "" }], + ["key", { ...valid, key: " " }], + ["key", { ...valid, key: 7 }], + ["label", { ...valid, label: undefined }], + ["label", { ...valid, label: "" }], + ])("rejects a bad %s naming the field", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test("rejects a non-string description", () => { + expect(() => validateSource({ ...valid, description: 7 }, "f.ts")).toThrow( + /`description` must be a string/, + ); + }); + + test.each([ + ["missing", { ...valid, transformer: undefined }], + ["null", { ...valid, transformer: null }], + ["an array", { ...valid, transformer: [] }], + ["a string", { ...valid, transformer: "id" }], + ])("rejects a transformer mapping that is %s", (_label, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(/`transformer`|`transformer\./); + }); + + test("names the offending entry when a mapping target is not a field name", () => { + expect(() => + validateSource({ ...valid, transformer: { account_ref: "userId", bad: 7 } }, "f.ts"), + ).toThrow(/`transformer.bad` must map to a Clerk field name, got number/); + }); + + // Without it the import runs to completion and creates every user with no + // external_id — which is what makes a migration reversible. + test("rejects a mapping with no userId target", () => { + expect(() => validateSource({ ...valid, transformer: { a: "email" } }, "f.ts")).toThrow( + /no source field maps to `userId`/, + ); + }); + + test.each([ + ["defaults", { ...valid, defaults: "nope" }], + ["preTransform", { ...valid, preTransform: "nope" }], + ["postTransform", { ...valid, postTransform: 7 }], + ])("rejects a %s of the wrong type", (field, value) => { + expect(() => validateSource(value, "f.ts")).toThrow(new RegExp(`\`${field}\``)); + }); + + test.each([["clerk"], ["auth0"], ["supabase"]])( + "rejects %s, which would shadow a built-in", + (key) => { + expect(() => + validateSource({ ...valid, key }, "f.ts", ["clerk", "auth0", "supabase"]), + ).toThrow(/already a built-in source/); + }, + ); + + // What a source brings across is the first thing `sources` shows, and only + // its author knows it. + test("requires carries", () => { + expect(() => validateSource({ ...valid, carries: undefined }, "f.ts")).toThrow( + /`carries` must say what the source brings across/, + ); + }); + + test.each([ + ["a level that is not yes, no or partial", { level: "maybe", note: "x" }], + ["no note", { level: "yes" }], + ])("rejects a carries entry with %s", (_label, mfa) => { + expect(() => validateSource({ ...valid, carries: { ...valid.carries, mfa } }, "f.ts")).toThrow( + /`carries.mfa` must be/, + ); + }); + + test("names the file in every message, so the author knows which one", () => { + expect(() => validateSource({}, "./their-file.ts")).toThrow(/\.\/their-file\.ts/); + }); + + test("raises CliError, so the global handler formats it", () => { + expect(() => validateSource({}, "f.ts")).toThrow(CliError); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/sources/load-custom.ts b/packages/cli-core/src/commands/migrate/sources/load-custom.ts new file mode 100644 index 000000000..cec25c327 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/load-custom.ts @@ -0,0 +1,198 @@ +/** + * Loading a user-authored source at runtime. + * + * In the standalone migration-tool, supporting a new platform meant adding a + * file to `src/transformers/` and one line to the registry — the user had the + * source tree. A compiled binary has neither a source tree to edit nor a way + * for an end user to rebuild it, so `--source ` restores that + * extensibility by importing a file from the user's own project instead. + * + * **Verified before this was built on:** a `bun build --compile` executable can + * `import()` an arbitrary external `.ts` file at runtime, including TypeScript + * that needs transpiling. Bun's transpiler is part of the runtime, not only the + * bundler. Confirmed with a throwaway compiled binary on darwin-arm64, + * linux-arm64 (glibc), linux-arm64-musl and linux-x64. + * + * The file is user-supplied code the CLI executes, so its shape is validated + * up front and rejected with a specific message rather than crashing deep in + * the transform pipeline on a missing field. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { CliError, ERROR_CODE, throwUsageError } from "../../../lib/errors.ts"; +import type { CarryLevel, SourceEntry } from "../types.ts"; + +const DOCS_URL = "https://clerk.com/docs/guides/development/migrating/overview"; + +const LEVELS: readonly CarryLevel[] = ["yes", "no", "partial"]; + +function invalid(problem: string, file: string): never { + throwUsageError(`${file} is not a valid source: ${problem}`, DOCS_URL); +} + +/** + * Checks a loaded value against the registry entry shape. + * + * Every failure names the specific field and what was wrong with it — the + * author is writing this file by hand against a shape they cannot see. + * + * @param file - Path as the user typed it, for the error message. + * @param reservedKeys - The built-in keys, which a custom source may not reuse. + */ +export function validateSource( + value: unknown, + file: string, + reservedKeys: readonly string[] = [], +): SourceEntry { + if (value === null || typeof value !== "object") { + invalid(`the default export is ${value === null ? "null" : typeof value}, not an object`, file); + } + + const entry = value as Record; + + for (const field of ["key", "label"] as const) { + if (typeof entry[field] !== "string" || entry[field].trim().length === 0) { + invalid(`\`${field}\` must be a non-empty string`, file); + } + } + + if (entry.description !== undefined && typeof entry.description !== "string") { + invalid("`description` must be a string when present", file); + } + + // Arrays are objects, and an author who wrote `transformer: []` should hear + // that rather than the downstream "no field maps to userId". + if ( + entry.transformer === null || + typeof entry.transformer !== "object" || + Array.isArray(entry.transformer) + ) { + invalid("`transformer` must be an object mapping source fields to Clerk fields", file); + } + + const mapping = entry.transformer as Record; + for (const [source, target] of Object.entries(mapping)) { + if (typeof target !== "string" || target.length === 0) { + invalid( + `\`transformer.${source}\` must map to a Clerk field name, got ${typeof target}`, + file, + ); + } + } + + // Without this the import runs to completion and creates every user with no + // external_id, which is what makes a migration re-runnable and reversible. + if (!Object.values(mapping).includes("userId")) { + invalid( + "no source field maps to `userId`. Every user needs one — it becomes the Clerk user's external_id", + file, + ); + } + + if ( + entry.defaults !== undefined && + (entry.defaults === null || typeof entry.defaults !== "object" || Array.isArray(entry.defaults)) + ) { + invalid("`defaults` must be an object when present", file); + } + + // What a source brings across is the first thing `sources` shows, and the + // author is the only one who knows it. + const carries = entry.carries as Record | undefined; + if (!carries || typeof carries !== "object" || Array.isArray(carries)) { + invalid( + "`carries` must say what the source brings across: { passwords, mfa, metadata }, each { level, note }", + file, + ); + } + for (const kind of ["passwords", "mfa", "metadata"] as const) { + const carry = carries[kind] as { level?: unknown; note?: unknown } | undefined; + if (!carry || !LEVELS.includes(carry.level as CarryLevel) || typeof carry.note !== "string") { + invalid( + `\`carries.${kind}\` must be { level: "yes" | "no" | "partial", note: string }`, + file, + ); + } + } + + for (const hook of ["preTransform", "postTransform"] as const) { + if (entry[hook] !== undefined && typeof entry[hook] !== "function") { + invalid(`\`${hook}\` must be a function when present`, file); + } + } + + if (reservedKeys.includes(entry.key as string)) { + invalid( + `\`key\` is "${String(entry.key)}", which is already a built-in source. Choose another key`, + file, + ); + } + + return { + ...(entry as unknown as SourceEntry), + description: (entry.description as string | undefined) ?? "Custom source", + }; +} + +/** + * Imports and validates a user-authored source. + * + * @throws CliError when the path is missing, the module fails to load, or the + * exported value does not match the registry entry shape. + */ +export async function loadCustomSource( + file: string, + reservedKeys: readonly string[] = [], + /** + * The file's content hash. The module loader caches by URL, so an edited + * file at the same path would load the old module; the hash in the URL + * loads the version that was hashed. + */ + version?: string, +): Promise { + const resolved = path.resolve(process.cwd(), file); + + if (!fs.existsSync(resolved)) { + throw new CliError(`No source file at ${resolved}.`, { + code: ERROR_CODE.FILE_NOT_FOUND, + docsUrl: DOCS_URL, + }); + } + if (fs.statSync(resolved).isDirectory()) { + throwUsageError(`${resolved} is a directory, not a source file.`); + } + + let module: Record; + try { + // A file URL rather than a bare path: an absolute POSIX path happens to + // work, but a Windows path (`C:\...`) is not a valid import specifier. + // Bun keys its module cache by specifier and keeps a `file://` URL's + // module whatever its query, but a plain path's query loads afresh. + // ponytail: Windows keeps the URL form, so an edit loaded twice in one + // process reuses the first; each CLI run is a process of its own. + const specifier = + version && process.platform !== "win32" + ? `${resolved}?v=${version}` + : Bun.pathToFileURL(resolved).href; + module = (await import(specifier)) as Record; + } catch (error) { + throwUsageError( + `Could not load ${file}: ${(error as Error).message}\n` + + "The file must be valid JavaScript or TypeScript that this CLI can import.", + DOCS_URL, + ); + } + + if (module.default === undefined) { + // Point at what they probably meant rather than just restating the rule. + const named = Object.keys(module).filter((key) => key !== "default"); + const hint = + named.length > 0 + ? ` Found named export${named.length === 1 ? "" : "s"} ${named.map((n) => `\`${n}\``).join(", ")} — did you mean \`export default\`?` + : ""; + throwUsageError(`${file} has no default export.${hint}`, DOCS_URL); + } + + return validateSource(module.default, file, reservedKeys); +} diff --git a/packages/cli-core/src/commands/migrate/sources/registry.ts b/packages/cli-core/src/commands/migrate/sources/registry.ts new file mode 100644 index 000000000..257732d9a --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/registry.ts @@ -0,0 +1,135 @@ +/** + * Source registry. + * + * `migrate import` resolves `--source` here, and `migrate sources` lists what + * each one brings across. + * + * To add a platform: create `sources/.ts` exporting a `SourceEntry`, + * then add it to the array below. + */ + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { throwUsageError } from "../../../lib/errors.ts"; +import type { SourceEntry } from "../types.ts"; +import auth0Source from "./auth0.ts"; +import authjsSource from "./authjs.ts"; +import betterAuthSource from "./betterauth.ts"; +import clerkSource from "./clerk.ts"; +import firebaseSource from "./firebase.ts"; +import { loadCustomSource } from "./load-custom.ts"; +import supabaseSource from "./supabase.ts"; +import workosSource from "./workos.ts"; + +export const sources: SourceEntry[] = [ + clerkSource, + auth0Source, + authjsSource, + betterAuthSource, + firebaseSource, + supabaseSource, + workosSource, +]; + +export const ACCOUNT_LINKING_URL = + "https://clerk.com/docs/guides/configure/auth-strategies/social-connections/account-linking"; + +/** + * What happens to social sign-ins, the same for every source. + * + * No export carries a user's OAuth connections, and none needs to: once the + * provider is enabled in Clerk, the user signs in with it and Clerk links the + * account to the imported user by verified email. + */ +export const ACCOUNT_LINKING_NOTE = + "Social sign-ins are not copied. Enable the same providers in Clerk, and a user who signs in " + + `with one is linked to their imported account by verified email. See ${ACCOUNT_LINKING_URL}`; + +/** + * A source loaded from a user's `--source ` for this invocation. + * + * Kept beside the built-ins rather than pushed into them, so the shipped list + * is never mutated. One invocation loads at most one, so this holding a + * single entry is the normal case; the array shape just avoids a special case + * in the lookups. + */ +const customSources: SourceEntry[] = []; + +export function registerCustomSource(entry: SourceEntry): void { + // The latest load of a key wins: an edited source replaces its old self. + const earlier = customSources.findIndex((source) => source.key === entry.key); + if (earlier >= 0) customSources.splice(earlier, 1); + customSources.push(entry); +} + +/** Test-only: drops anything a previous test registered. */ +export function __resetCustomSourcesForTesting(): void { + customSources.length = 0; +} + +/** Built-ins plus whatever `--source ` loaded. */ +export function allSources(): SourceEntry[] { + return [...sources, ...customSources]; +} + +/** The built-in keys, for tab-completion and error messages. */ +export function sourceKeys(): string[] { + return sources.map((entry) => entry.key); +} + +/** + * Looks up a source by key, custom ones included. + * + * @throws Error when no source is registered under that key. + */ +export function getSource(key: string): SourceEntry { + const source = allSources().find((entry) => entry.key === key); + if (!source) { + throw new Error(`Source not found for key: ${key}`); + } + return source; +} + +/** A `--source` value that names a file rather than a built-in key. */ +export function isSourcePath(value: string): boolean { + return /^(\.\.?\/|\/)/.test(value) || /\.(ts|js|mjs)$/.test(value); +} + +/** A resolved `--source`: its key, and for a custom one, the file's content hash. */ +export type ResolvedSource = { key: string; entry: SourceEntry; path?: string; hash?: string }; + +/** + * Resolves `--source`: a built-in key, or a path to a source you wrote. + * + * A custom source is keyed by a hash of its file as well as its key, so an + * edited source is a different source when a re-run decides whether to + * continue an earlier import. + * + * @throws UsageError for an unknown key, listing the valid ones. + */ +export async function resolveSource(value: string): Promise { + if (isSourcePath(value)) { + const resolved = path.resolve(process.cwd(), value); + // Hashed first, and loaded by that hash: the record names the very code + // that maps the users, even if the file was edited since a last load. + const hash = fs.existsSync(resolved) + ? createHash("sha256").update(fs.readFileSync(resolved)).digest("hex") + : undefined; + const entry = await loadCustomSource(value, sourceKeys(), hash); + registerCustomSource(entry); + return { key: entry.key, entry, path: resolved, hash }; + } + + const entry = sources.find((candidate) => candidate.key === value); + if (!entry) { + throwUsageError( + `Unknown source "${value}". Valid sources: ${sourceKeys().join(", ")}.\n` + + "For a platform with no built-in, pass the path to a source you wrote, e.g. --source ./my-source.ts.", + undefined, + undefined, + [{ command: "clerk migrate sources", description: "List the built-in sources" }], + ); + } + return { key: entry.key, entry }; +} diff --git a/packages/cli-core/src/commands/migrate/sources/shared.ts b/packages/cli-core/src/commands/migrate/sources/shared.ts new file mode 100644 index 000000000..64405b279 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/shared.ts @@ -0,0 +1,137 @@ +/** + * Helpers shared by more than one transformer. + * + * Every source platform records verification as a sibling field of the + * identifier, and several ship a single `name` string where Clerk wants a + * first/last pair — so both live here rather than being copied five times. + */ + +/** + * How a platform records that an identifier is verified. + * + * - `boolean` — a true/false flag (Auth0, Better Auth, Firebase). A CSV export + * turns these into the *strings* `"true"`/`"false"`, so `"false"` must not + * be mistaken for a truthy value. + * - `timestamp` — a nullable confirmation time (Auth.js `email_verified`, + * Supabase `email_confirmed_at`). Any real value means verified. + */ +export type VerificationStyle = "boolean" | "timestamp"; + +/** CSV exports write SQL NULL as one of these rather than an empty cell. */ +const NULLISH_STRINGS = new Set(["", "null", "nil", "undefined", "\\n"]); + +/** + * True for a string a CSV export writes in place of SQL NULL. A first name + * that really is "Null" reads as one too; the spellings tools write matter more. + */ +export function isNullish(value: unknown): boolean { + return typeof value === "string" && NULLISH_STRINGS.has(value.trim().toLowerCase()); +} + +/** + * A boolean as a CSV or a database writes it: `TRUE` from a spreadsheet, `t` + * and `f` from psql, `yes`/`no`. Anything else is returned unchanged, so the + * schema can reject it. + */ +export function normalizeBooleanField(value: unknown): unknown { + if (typeof value === "boolean") return value; + if (typeof value === "number") { + if (value === 1) return true; + if (value === 0) return false; + return value; + } + if (typeof value !== "string") return value; + + const normalized = value.trim().toLowerCase(); + if (["true", "1", "yes", "y", "t"].includes(normalized)) return true; + if (["false", "0", "no", "n", "f"].includes(normalized)) return false; + return value; +} + +export function isVerified(value: unknown, style: VerificationStyle): boolean { + if (value === null || value === undefined) return false; + + if (style === "boolean") return normalizeBooleanField(value) === true; + + if (value instanceof Date) return !Number.isNaN(value.getTime()); + if (typeof value === "number") return true; + return typeof value === "string" && !isNullish(value); +} + +/** + * Routes an identifier to its verified or unverified field, then drops the + * platform's verification marker. + * + * An unverified identifier must not go on `POST /v1/users`'s primary field: + * Clerk creates those verified, which would silently promote an address the + * source platform never confirmed. + */ +export function routeByVerification( + user: Record, + field: "email" | "phone", + verifiedField: string, + style: VerificationStyle, +): void { + const value = user[field]; + if (value && !isVerified(user[verifiedField], style)) { + user[field === "email" ? "unverifiedEmailAddresses" : "unverifiedPhoneNumbers"] = value; + delete user[field]; + } + delete user[verifiedField]; +} + +/** + * Splits a single display name into `firstName` and `lastName`: the first word, + * then the rest. A one-word name (`Cher`) becomes the first name alone. + * + * A one-word value that is an email address is dropped instead: Auth0 fills + * `name` with the email for database users, and that is not anyone's name. + */ +export function splitName(user: Record, field = "name"): void { + const name = user[field]; + if (!name || typeof name !== "string") return; + + const parts = name.trim().split(/\s+/); + if (parts.length > 1) { + user.firstName = parts[0]; + user.lastName = parts.slice(1).join(" "); + } else if (parts[0] && !parts[0].includes("@")) { + user.firstName = parts[0]; + } + delete user[field]; +} + +/** + * Converts a source timestamp to ISO 8601, leaving it untouched when it does + * not parse so the schema reports it as a validation failure with the original + * value visible in the log. + * + * @param epochMillis - Treat a bare number (or numeric string) as Unix + * milliseconds, which is how Firebase exports timestamps. + */ +export function toIsoDate(value: unknown, epochMillis = false): unknown { + if (value === undefined || value === null || value === "") return value; + + const parsed = + epochMillis && (typeof value === "number" || /^\d+$/.test(String(value))) + ? new Date(Number(value)) + : new Date(String(value)); + + return Number.isNaN(parsed.getTime()) ? value : parsed.toISOString(); +} + +/** + * The hasher for a self-describing bcrypt or argon2 digest, or undefined. + * Platforms that accept imported hashes (Supabase, Better Auth) can hold + * either, so the hasher is read per user rather than assumed per source. + */ +export function detectStandardHasher(hash: string): "bcrypt" | "argon2id" | "argon2i" | undefined { + // The whole digest, not the prefix: a malformed one is dropped like any + // other unusable hash, where a prefix match would see the user rejected. + // clerk_go caps the cost at 15 (pkg/hash/bcrypt.go), and Go's bcrypt + // refuses any cost under 4 at sign-in, so such a digest could never be used. + if (/^\$2[aby]\$(0[4-9]|1[0-5])\$[./A-Za-z0-9]{53}$/.test(hash)) return "bcrypt"; + if (hash.startsWith("$argon2id$")) return "argon2id"; + if (hash.startsWith("$argon2i$")) return "argon2i"; + return undefined; +} diff --git a/packages/cli-core/src/commands/migrate/sources/sources.test.ts b/packages/cli-core/src/commands/migrate/sources/sources.test.ts new file mode 100644 index 000000000..7e920c157 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/sources.test.ts @@ -0,0 +1,894 @@ +import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { CliError } from "../../../lib/errors.ts"; +import { loadUsersFromFile, transformUsers } from "../lib/transform.ts"; +import type { FirebaseHashConfig } from "../types.ts"; +import { getSource, isSourcePath, sourceKeys, sources } from "./registry.ts"; +import { isVerified } from "./shared.ts"; + +const FIREBASE_HASH: FirebaseHashConfig = { + base64_signer_key: "SIGNERKEY==", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, +}; + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-sources-"))); + process.chdir(workDir); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +/** Writes `records` to a uniquely-named file and loads it through `key`. */ +async function load(key: string, records: unknown, ext = "json", context = {}) { + const file = `${key}-${Math.abs(JSON.stringify(records).length)}-${ext}.${ext}`; + fs.writeFileSync( + path.join(workDir, file), + typeof records === "string" ? records : JSON.stringify(records), + ); + return loadUsersFromFile(file, key, { context }); +} + +const one = (key: string, record: Record, context = {}) => + transformUsers([record], key, { validate: false, context }).transformedData[0] as + | Record + | undefined; + +describe("registry", () => { + test("registers all seven platforms", () => { + expect(sourceKeys()).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + "workos", + ]); + }); + + test.each([...sources])("$key maps a source field to userId", (source) => { + expect(Object.values(source.transformer)).toContain("userId"); + }); + + test.each([...sources])("$key carries a label and description", (source) => { + expect(source.label.length).toBeGreaterThan(0); + expect(source.description.length).toBeGreaterThan(0); + }); + + test.each([...sources])("$key says what it carries, with a note for each", (source) => { + for (const carry of Object.values(source.carries)) { + expect(["yes", "no", "partial"]).toContain(carry.level); + expect(carry.note.length).toBeGreaterThan(0); + } + }); + + test("throws for an unregistered key", () => { + expect(() => getSource("okta")).toThrow(/Source not found/); + }); +}); + +describe("isSourcePath", () => { + test.each([["./mine.ts"], ["../up/mine.js"], ["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/abs/mine.mjs"], ["mine.ts"], ["dir/mine.js"]])( + "%s is a path", + (value) => { + expect(isSourcePath(value)).toBe(true); + }, + ); + + test.each([["clerk"], ["betterauth"], ["okta"]])("%s is a key", (value) => { + expect(isSourcePath(value)).toBe(false); + }); +}); + +describe("isVerified", () => { + // A CSV export stringifies everything, so the boolean style must read + // "false" as false. Treating it as truthy would mark unconfirmed addresses + // verified on import — the exact thing the routing exists to prevent. + test.each([ + [true, true], + ["true", true], + [1, true], + ["1", true], + [false, false], + ["false", false], + [0, false], + ["0", false], + ["", false], + [null, false], + [undefined, false], + // A CSV re-saved by a spreadsheet, or written by psql. + ["TRUE", true], + ["FALSE", false], + ["t", true], + ["f", false], + ["Yes", true], + ])("boolean style: %p -> %p", (value, expected) => { + expect(isVerified(value, "boolean")).toBe(expected); + }); + + // The timestamp style is presence-based: any real confirmation time counts, + // and SQL NULL arrives from a CSV export as one of several spellings. + test.each([ + ["2024-06-29 20:25:06+00", true], + ["2024-01-15T10:30:00.000Z", true], + ["", false], + [" ", false], + ["null", false], + ["NULL", false], + ["\\N", false], + [null, false], + [undefined, false], + ])("timestamp style: %p -> %p", (value, expected) => { + expect(isVerified(value, "timestamp")).toBe(expected); + }); +}); + +describe("auth0", () => { + const base = { user_id: "auth0|abc", email: "a@x.dev", passwordHash: "$2b$10$hash" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("auth0", [ + { ...base, email_verified: true, given_name: "Ada", family_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "auth0|abc", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + password: "$2b$10$hash", + passwordHasher: "bcrypt", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("auth0", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("routes an unverified phone away from the primary field", () => { + const user = one("auth0", { ...base, phone_number: "+15555550100", phone_verified: false }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test("drops the platform's verification markers", () => { + const user = one("auth0", { ...base, email_verified: true, phone_verified: true }); + expect("emailVerified" in (user ?? {})).toBe(false); + expect("phoneVerified" in (user ?? {})).toBe(false); + }); + + // `user_metadata` is the user's own to edit in Auth0, which is what Clerk's + // unsafe metadata is; public metadata is read-only to the user. + test("sends user_metadata to unsafe metadata and app_metadata to private", async () => { + const { users } = await load("auth0", [ + { + ...base, + email_verified: true, + user_metadata: { theme: "dark" }, + app_metadata: { plan: "pro" }, + }, + ]); + expect(users[0]?.unsafeMetadata).toEqual({ theme: "dark" }); + expect(users[0]?.publicMetadata).toBeUndefined(); + expect(users[0]?.privateMetadata).toEqual({ plan: "pro" }); + }); + + test.each([ + [true, true], + ["true", true], + [false, undefined], + [undefined, undefined], + ])("blocked=%p is carried as banned=%p", (blocked, expected) => { + expect(one("auth0", { ...base, blocked })?.banned).toBe(expected as boolean | undefined); + }); + + test("splits name when given_name and family_name are absent", () => { + const user = one("auth0", { ...base, name: "Ada King Lovelace" }); + expect(user).toMatchObject({ firstName: "Ada", lastName: "King Lovelace" }); + }); + + test("prefers given_name/family_name over name", () => { + const user = one("auth0", { ...base, name: "a@x.dev", given_name: "Ada", family_name: "L" }); + expect(user).toMatchObject({ firstName: "Ada", lastName: "L" }); + }); + + test("leaves Auth0's email-default name unset", () => { + const user = one("auth0", { ...base, name: "a@x.dev" }); + expect(user?.firstName).toBeUndefined(); + }); +}); + +describe("workos", () => { + const base = { id: "user_01ABC", email: "a@x.dev" }; + + test("maps identity, name and metadata onto the Clerk schema", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, first_name: "Ada", last_name: "Lovelace" }, + ]); + expect(users[0]).toMatchObject({ + userId: "user_01ABC", + email: "a@x.dev", + firstName: "Ada", + lastName: "Lovelace", + }); + }); + + test.each([ + [true, "email", undefined], + [false, undefined, "a@x.dev"], + [undefined, undefined, "a@x.dev"], + ])("email_verified=%p routes the address correctly", (verified, kept, unverified) => { + const user = one("workos", { ...base, email_verified: verified }); + expect(user?.email).toBe(kept ? "a@x.dev" : undefined); + expect(user?.unverifiedEmailAddresses).toBe(unverified); + }); + + test("sends metadata to unsafe metadata", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, metadata: { plan: "pro" } }, + ]); + expect(users[0]?.unsafeMetadata).toEqual({ plan: "pro" }); + }); + + // The WorkOS id stays the Clerk external_id; the tenant's own ID is kept + // where users cannot edit it. + test("puts WorkOS's external_id in private metadata", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, external_id: "cust_1" }, + ]); + expect(users[0]?.userId).toBe("user_01ABC"); + expect(users[0]?.privateMetadata).toEqual({ workosExternalId: "cust_1" }); + expect("workosExternalId" in (users[0] ?? {})).toBe(false); + }); + + // No other transformer omits it. WorkOS never returns a digest, so naming a + // hasher would imply a password column that cannot exist. + test("names no password hasher, because WorkOS returns no hashes", () => { + expect(getSource("workos").defaults).toBeUndefined(); + }); + + // The export carries these so whoever runs the migration can see who used + // social sign-in; Clerk's import has no field for them. + test("drops OAuth identities carried through from the export", async () => { + const { users } = await load("workos", [ + { ...base, email_verified: true, identities: [{ provider: "GoogleOAuth", idp_id: "1" }] }, + ]); + expect(users[0]?.userId).toBe("user_01ABC"); + expect("identities" in (users[0] ?? {})).toBe(false); + }); +}); + +describe("authjs", () => { + const base = { id: "cuid1", email: "a@x.dev" }; + + test("treats a confirmation timestamp as verified", () => { + const user = one("authjs", { ...base, email_verified: "2024-01-15T10:30:00.000Z" }); + expect(user?.email).toBe("a@x.dev"); + expect(user?.unverifiedEmailAddresses).toBeUndefined(); + }); + + test.each([[null], [""], [undefined]])("treats email_verified=%p as unverified", (value) => { + const user = one("authjs", { ...base, email_verified: value }); + expect(user?.unverifiedEmailAddresses).toBe("a@x.dev"); + }); + + test.each([ + ["Jane Doe", "Jane", "Doe"], + ["Mary Jane Watson", "Mary", "Jane Watson"], + [" Ada Lovelace ", "Ada", "Lovelace"], + ])("splits %p into %p / %p", (name, firstName, lastName) => { + const user = one("authjs", { ...base, name }); + expect(user?.firstName).toBe(firstName); + expect(user?.lastName).toBe(lastName); + }); + + test("keeps a single-word name as the first name, without inventing a last name", () => { + const user = one("authjs", { ...base, name: "Prince" }); + expect(user?.firstName).toBe("Prince"); + expect(user?.lastName).toBeUndefined(); + expect("name" in (user ?? {})).toBe(false); + }); + + test("imports without a password, since Auth.js core is passwordless", async () => { + const { users } = await load("authjs", [{ ...base, email_verified: "2024-01-01" }]); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + }); +}); + +describe("betterauth", () => { + const base = { user_id: "ba1", email: "a@x.dev", email_verified: true }; + + test.each([ + [1, "anonymous Better Auth user"], + [true, "anonymous Better Auth user"], + [0, undefined], + [undefined, undefined], + ])("isAnonymous=%p sets skipReason %p", (isAnonymous, expected) => { + const user = one("betterauth", { ...base, isAnonymous }); + expect(user?.skipReason).toBe(expected); + expect("isAnonymous" in (user ?? {})).toBe(false); + }); + + // Better Auth's own scrypt: a 16-byte hex salt, a colon, a 64-byte hex key. + const SALT = "a".repeat(32); + const KEY = "b".repeat(128); + + test.each([ + [`${SALT}:${KEY}`, `scrypt:16384:16:1$${SALT}$${KEY}`, "scrypt_werkzeug"], + [ + "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "bcrypt", + ], + [ + "$2b$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "$2b$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "bcrypt", + ], + [ + "$2y$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "$2y$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + "bcrypt", + ], + [ + "$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", + "$argon2id$v=19$m=65536,t=3,p=4$c2FsdA$aGFzaA", + "argon2id", + ], + [ + "$argon2i$v=19$m=4096,t=3,p=1$c2FsdA$aGFzaA", + "$argon2i$v=19$m=4096,t=3,p=1$c2FsdA$aGFzaA", + "argon2i", + ], + ])("detects the hasher per user: %s", async (stored, password, passwordHasher) => { + const { users } = await load("betterauth", [{ ...base, password_hash: stored }]); + expect(users[0]).toMatchObject({ password, passwordHasher }); + expect(users[0]?.passwordDropped).toBeUndefined(); + }); + + // Imported without the password rather than rejected: the user can still + // sign in another way, or reset it. + test.each([["plaintext"], ["$pbkdf2$abc"], [`${SALT}:short`]])( + "drops a password it cannot verify (%s) and imports the user", + async (stored) => { + const { users, validationFailed } = await load("betterauth", [ + { ...base, password_hash: stored }, + ]); + expect(validationFailed).toBe(0); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordHasher).toBeUndefined(); + expect(users[0]?.passwordDropped).toBe(true); + }, + ); + + test("names no hasher for a user without a password", async () => { + const { users } = await load("betterauth", [base]); + expect(users[0]?.passwordHasher).toBeUndefined(); + expect(users[0]?.passwordDropped).toBeUndefined(); + }); + + test("routes an unverified phone", () => { + const user = one("betterauth", { + ...base, + phone_number: "+15555550100", + phone_number_verified: false, + }); + expect(user?.unverifiedPhoneNumbers).toBe("+15555550100"); + }); + + test.each([ + [true, true], + [1, true], + ["true", true], + [false, undefined], + [0, undefined], + [undefined, undefined], + ])("banned=%p is carried through as %p", (banned, expected) => { + expect(one("betterauth", { ...base, banned })?.banned).toBe(expected as boolean | undefined); + }); + + // The prefix alone said bcrypt, so the user was rejected instead of dropped. + test("drops a malformed bcrypt hash, and imports the user", async () => { + const { users } = await load("betterauth", [{ ...base, password_hash: "$2a$10$hash" }]); + expect(users[0]?.password).toBeUndefined(); + expect(users[0]?.passwordDropped).toBe(true); + }); + + // Clerk's ban has no end: the checks warn using the one Better Auth set. + test("a ban with an expiry keeps its end date; one without is permanent", () => { + expect( + one("betterauth", { ...base, banned: true, banExpires: "2999-01-01T00:00:00.000Z" }), + ).toMatchObject({ banned: true, banEndsAt: "2999-01-01T00:00:00.000Z" }); + const permanent = one("betterauth", { ...base, banned: true, banExpires: null }); + expect(permanent?.banned).toBe(true); + expect(permanent).not.toHaveProperty("banEndsAt"); + }); + + // Better Auth lifts an expired ban only at the next sign-in, so the column + // can still say banned long after the ban ended. + test.each([ + ["an expiry in the past", "2020-01-01T00:00:00.000Z", undefined], + ["an expiry in the future", "2999-01-01T00:00:00.000Z", true], + ["epoch seconds in the past", 1_577_836_800, undefined], + ["epoch milliseconds in the future", 32_472_144_000_000, true], + // A CSV of a Drizzle SQLite `integer({ mode: "timestamp" })` column. + ["epoch seconds in the future, as a string", "32472144000", true], + ["epoch seconds in the past, as a string", "1577836800", undefined], + ["epoch milliseconds in the future, as a string", "32472144000000", true], + ["an unreadable expiry", "soon", true], + ])("a ban with %s -> banned %p", (_label, banExpires, expected) => { + const user = one("betterauth", { ...base, banned: true, banExpires }); + expect(user?.banned).toBe(expected as boolean | undefined); + }); + + test("drops plugin-only columns during validation", async () => { + const { users } = await load("betterauth", [ + { ...base, role: "admin", display_username: "ADA", two_factor_enabled: true }, + ]); + const user = users[0] as Record; + expect("role" in user).toBe(false); + expect("display_username" in user).toBe(false); + expect("two_factor_enabled" in user).toBe(false); + }); +}); + +describe("firebase", () => { + const base = { localId: "fb1", email: "a@x.dev", emailVerified: true }; + + test.each([ + [true, true], + ["true", true], + [false, undefined], + [undefined, undefined], + ])("disabled=%p carries as banned=%p", (disabled, expected) => { + expect(one("firebase", { ...base, disabled })?.banned).toBe(expected as boolean | undefined); + }); + const withHash = { ...base, passwordHash: "SGFzaA==", salt: "U2FsdA==" }; + + test("builds the scrypt digest Clerk expects, parameters inline", async () => { + const { users } = await load("firebase", { users: [withHash] }, "json", { + firebaseHashConfig: FIREBASE_HASH, + }); + expect(users[0]?.password).toBe("SGFzaA==$U2FsdA==$SIGNERKEY==$Bw==$8$14"); + expect(users[0]?.passwordHasher).toBe("scrypt_firebase"); + }); + + test("refuses to import hashes without the project's hash parameters", async () => { + await expect(load("firebase", { users: [withHash] })).rejects.toThrow( + /Firebase password hashes/, + ); + }); + + test("imports a passwordless export with no hash parameters at all", async () => { + const { users } = await load("firebase", { users: [base] }); + expect(users).toHaveLength(1); + expect(users[0]?.password).toBeUndefined(); + }); + + test("unwraps the { users: [...] } export shape", async () => { + const { users } = await load("firebase", { users: [base, { ...base, localId: "fb2" }] }); + expect(users.map((u) => u.userId)).toEqual(["fb1", "fb2"]); + }); + + test("accepts a bare array too", async () => { + const { users } = await load("firebase", [base]); + expect(users).toHaveLength(1); + }); + + test("rejects a JSON export that is neither", async () => { + await expect(load("firebase", { records: [] })).rejects.toThrow(CliError); + }); + + test("rejects a wrapped row that is not a user object", async () => { + await expect(load("firebase", { users: [base, null] })).rejects.toThrow( + /row 2 is not a user object/, + ); + }); + + // Named, not prepended: a copy with a header row would leave the hashes and + // salts in a temp file nobody deletes. + test("names the columns of a headerless CSV export, writing no copy", async () => { + const csv = "fb9,a@x.dev,true,,,Ada Lovelace,,,,,,,,,,,,,,,,,,1704067200000,,,,,\n"; + const mkdtemp = spyOn(fs, "mkdtempSync"); + try { + const { users } = await load("firebase", csv, "csv"); + expect(users[0]).toMatchObject({ userId: "fb9", email: "a@x.dev", firstName: "Ada" }); + expect(mkdtemp).not.toHaveBeenCalled(); + } finally { + mkdtemp.mockRestore(); + } + }); + + test.each([ + ["1704067200000", "2024-01-01T00:00:00.000Z"], + [1704067200000, "2024-01-01T00:00:00.000Z"], + ])("converts the Unix-millisecond createdAt %p", (createdAt, expected) => { + expect(one("firebase", { ...base, createdAt })?.createdAt).toBe(expected); + }); + + test.each([ + [true, true], + ["true", true], + [false, false], + ["false", false], + ])("emailVerified=%p keeps the address primary: %p", (emailVerified, verified) => { + const user = one("firebase", { ...base, emailVerified }); + expect(user?.email !== undefined).toBe(verified); + }); +}); + +describe("supabase", () => { + // What `export supabase` hands over: first_name only when the metadata has + // one, the rest left in the metadata for this source to split. + test.each([ + ["an OAuth name", { name: "Jane Doe" }, "Jane", "Doe"], + [ + "a display name beside a real last name", + { display_name: "Jane Doe", last_name: "Doe" }, + "Jane", + "Doe", + ], + ])("splits %s from the metadata", (_label, meta, firstName, lastName) => { + const user = one("supabase", { + id: "s1", + email: "a@x.dev", + last_name: (meta as { last_name?: string }).last_name, + raw_user_meta_data: meta, + }); + expect(user).toMatchObject({ firstName, lastName }); + }); + + // `NULL` is how SQL tools write an empty column into a CSV. + test("reads NULL cells in a CSV as empty, not as values", async () => { + const { users, failures } = await load( + "supabase", + [ + "id,email,email_confirmed_at,encrypted_password,phone,raw_user_meta_data,deleted_at,created_at", + "s1,a@x.dev,2024-01-01,NULL,NULL,NULL,NULL,NULL", + ].join("\n"), + "csv", + ); + + expect(failures).toEqual([]); + expect(users).toHaveLength(1); + expect(users[0]).toMatchObject({ userId: "s1", email: "a@x.dev" }); + for (const field of [ + "skipReason", + "phone", + "unverifiedPhoneNumbers", + "passwordDropped", + "createdAt", + ]) { + expect(users[0]).not.toHaveProperty(field); + } + }); + + test("keeps a name that reads as NULL", async () => { + const { users } = await load( + "supabase", + ["id,email,email_confirmed_at,last_name,deleted_at", "s2,b@x.dev,2024-01-01,Null,NULL"].join( + "\n", + ), + "csv", + ); + + expect(users[0]).toMatchObject({ userId: "s2", lastName: "Null" }); + expect(users[0]).not.toHaveProperty("skipReason"); + }); + + const base = { id: "sb1", email: "a@x.dev", email_confirmed_at: "2024-06-29 20:25:06.126079+00" }; + + // Supabase accepts argon2 hashes on import, so the hasher is read per user. + test.each([ + ["$2a$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", "bcrypt"], + ["$argon2id$v=19$m=19456,t=2,p=1$c2FsdHNhbHQ$aGFzaGhhc2hoYXNo", "argon2id"], + ["$argon2i$v=19$m=4096,t=3,p=1$c2FsdHNhbHQ$aGFzaGhhc2hoYXNo", "argon2i"], + ])("detects the hasher for %p", (encrypted_password, hasher) => { + const user = one("supabase", { ...base, encrypted_password }); + expect(user?.passwordHasher).toBe(hasher); + expect(user?.password).toBe(encrypted_password); + }); + + // The prefix alone said bcrypt, so the user was rejected instead of dropped. + // Go's bcrypt refuses a cost under 4 at sign-in, so that digest is unusable. + test.each([ + ["$2a$10$hash"], + ["$2a$31$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"], + ["$2a$03$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy"], + ])("drops a malformed bcrypt hash (%p), and imports the user", (encrypted_password) => { + const user = one("supabase", { ...base, encrypted_password }); + expect(user?.password).toBeUndefined(); + expect(user?.passwordDropped).toBe(true); + }); + + test("drops a hash no hasher fits, and imports the user", () => { + const user = one("supabase", { ...base, encrypted_password: "md5:abc" }); + expect(user?.password).toBeUndefined(); + expect(user?.passwordDropped).toBe(true); + }); + + test("skips a soft-deleted user", () => { + const user = one("supabase", { ...base, deleted_at: "2026-01-01 00:00:00+00" }); + expect(user?.skipReason).toBe("deleted in Supabase"); + expect("deletedAt" in (user ?? {})).toBe(false); + }); + + test.each([ + ["14165550123", "+14165550123"], + ["+14165550123", "+14165550123"], + ])("phone %p imports as %p", (phone, expected) => { + const user = one("supabase", { ...base, phone, phone_confirmed_at: "2024-06-29 20:25:06+00" }); + expect(user?.phone).toBe(expected); + }); + + test("adds the + to an unconfirmed phone too", () => { + expect(one("supabase", { ...base, phone: "14165550123" })?.unverifiedPhoneNumbers).toBe( + "+14165550123", + ); + }); + + // An expired ban means the user is active again in Supabase. + test.each([ + ["2999-01-01 00:00:00+00", true], + ["2020-01-01 00:00:00+00", undefined], + [undefined, undefined], + ])("banned_until=%p carries as banned=%p", (bannedUntil, expected) => { + const user = one("supabase", { ...base, banned_until: bannedUntil }); + expect(user?.banned).toBe(expected as boolean | undefined); + expect("bannedUntil" in (user ?? {})).toBe(false); + }); + + // Clerk's ban has no end: a temporary one keeps its end date for the checks + // to warn about, and a far-future "permanent" one does not. + test("a temporary ban keeps its end date; a permanent one does not", () => { + const soon = new Date(Date.now() + 2 * 24 * 60 * 60_000).toISOString(); + expect(one("supabase", { ...base, banned_until: soon })).toMatchObject({ + banned: true, + banEndsAt: soon, + }); + const permanent = one("supabase", { ...base, banned_until: "2999-01-01 00:00:00+00" }); + expect(permanent?.banned).toBe(true); + expect(permanent).not.toHaveProperty("banEndsAt"); + }); + + // The hasher comes from each digest, so a user without one gets none. Loaded + // from a file, because that is where source defaults are applied. + test("gives a passwordless user no password hasher", async () => { + const { users } = await load("supabase", [ + { id: "s1", email: "a@x.dev", encrypted_password: "" }, + ]); + expect(users[0]).not.toHaveProperty("passwordHasher"); + }); + + test("maps the bcrypt password and converts the PostgreSQL timestamp", async () => { + const { users } = await load("supabase", [ + { + ...base, + encrypted_password: "$2b$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + created_at: "2024-06-29 20:25:06.126079+00", + }, + ]); + expect(users[0]).toMatchObject({ + password: "$2b$10$N9qo8uLOickgx2ZMRZoMyeIjZAgcfl7p92ldGxad68LJZdL17lhWy", + passwordHasher: "bcrypt", + createdAt: "2024-06-29T20:25:06.126Z", + }); + }); + + test.each([ + ["2024-06-29 20:25:06+00", true], + [null, false], + ["", false], + ])("email_confirmed_at=%p means verified: %p", (confirmedAt, verified) => { + const user = one("supabase", { ...base, email_confirmed_at: confirmedAt }); + expect(user?.email !== undefined).toBe(verified); + }); + + test("falls back to user metadata for a missing first name", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + + // A CSV carries the metadata as JSON text. + test("falls back to metadata given as JSON text, as a CSV carries it", () => { + const user = one("supabase", { + ...base, + raw_user_meta_data: JSON.stringify({ display_name: "Ada Lovelace" }), + }); + expect(user?.firstName).toBe("Ada"); + }); + + test.each([ + ["an object", { first_name: "Mary Ann", last_name: "Doe" }], + ["JSON text", JSON.stringify({ first_name: "Mary Ann", last_name: "Doe" })], + ])("maps separate metadata name fields as-is, given as %s", (_, raw_user_meta_data) => { + const user = one("supabase", { ...base, raw_user_meta_data }); + expect(user?.firstName).toBe("Mary Ann"); + expect(user?.lastName).toBe("Doe"); + }); + + // Supabase's own social logins write `full_name`. + test("splits a metadata full_name", () => { + const user = one("supabase", { ...base, raw_user_meta_data: { full_name: "Ada Lovelace" } }); + expect(user?.firstName).toBe("Ada"); + expect(user?.lastName).toBe("Lovelace"); + }); + + test("prefers explicit name columns over metadata", () => { + const user = one("supabase", { + ...base, + first_name: "Grace", + raw_user_meta_data: { display_name: "Ada Lovelace" }, + }); + expect(user?.firstName).toBe("Grace"); + }); + + test.each([ + ["ada#0", "ada"], + ["ada#1234", "ada"], + ])("strips the Discord discriminator from %p", (displayName, expected) => { + const user = one("supabase", { ...base, raw_user_meta_data: { display_name: displayName } }); + expect(user?.firstName).toBe(expected); + }); + + test("drops a name that was nothing but a discriminator", () => { + const user = one("supabase", { ...base, first_name: "#0" }); + expect(user?.firstName).toBeUndefined(); + }); +}); + +// The Dashboard's CSV prefixes a TAB to a value a spreadsheet would run as a +// formula (clerk_go pkg/csvsafe). +describe("clerk", () => { + test.each([ + ["\t=Ada", "=Ada"], + ["\t@ada", "@ada"], + ["\t＋Ada", "＋Ada"], + ["\tAda", "\tAda"], + ])("first name %p imports as %p", (firstName, expected) => { + expect( + one("clerk", { id: "u1", primary_email_address: "a@x.dev", first_name: firstName }) + ?.firstName, + ).toBe(expected); + }); + + test("unprefixes each address in a list", () => { + const user = one("clerk", { + id: "u1", + primary_email_address: "a@x.dev", + unverified_email_addresses: "\t-b@x.dev", + }); + expect(user?.unverifiedEmailAddresses).toEqual(["-b@x.dev"]); + }); + + // The TAB has to come off before the primary is matched against the lists, + // or a phone is sent twice, or an unverified one is created verified. + test("a prefixed phone that is primary and verified is one phone", () => { + const user = one("clerk", { + id: "u1", + primary_phone_number: "\t+15555550100", + verified_phone_numbers: "\t+15555550100", + }); + expect(user?.phone).toEqual(["+15555550100"]); + }); + + test("a prefixed primary phone listed as unverified stays unverified", () => { + const user = one("clerk", { + id: "u1", + primary_email_address: "a@x.dev", + primary_phone_number: "\t+15555550100", + unverified_phone_numbers: "\t+15555550100", + }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toEqual(["+15555550100"]); + }); + + // The schema accepts an array for a primary identifier, so it must not throw. + test("reads a primary email given as an array", () => { + const user = one("clerk", { + id: "u1", + primary_email_address: [" a@x.dev ", "b@x.dev"], + verified_email_addresses: ["b@x.dev"], + }); + expect(user?.email).toEqual(["a@x.dev", "b@x.dev"]); + }); + + test("an array primary listed as unverified stays unverified", () => { + const user = one("clerk", { + id: "u1", + primary_phone_number: ["+15555550100"], + primary_email_address: "a@x.dev", + unverified_phone_numbers: [" +15555550100 "], + }); + expect(user?.phone).toBeUndefined(); + expect(user?.unverifiedPhoneNumbers).toEqual(["+15555550100"]); + }); + + test("a prefixed primary email listed as unverified stays unverified", () => { + const user = one("clerk", { + id: "u1", + primary_email_address: "\t+a@x.dev", + unverified_email_addresses: "\t+a@x.dev", + }); + expect(user?.email).toBeUndefined(); + expect(user?.unverifiedEmailAddresses).toEqual(["+a@x.dev"]); + }); +}); + +// The CLI's own exports add these; reporting them as "Clerk won't store" on +// every import would be noise about the CLI itself. +describe("fields the CLI's own export adds", () => { + test.each([ + [ + "supabase", + { + id: "s1", + email: "a@x.dev", + email_confirmed_at: "2024-01-01", + raw_app_meta_data: { providers: ["email"] }, + }, + ], + [ + "betterauth", + { user_id: "b1", email: "a@x.dev", email_verified: true, updated_at: "2024-01-01" }, + ], + ])("%s reports no unknown fields", async (key, record) => { + const { unknownFields } = await load(key, [record]); + expect(unknownFields).toEqual({}); + }); +}); + +describe("invalid records", () => { + const INVALID: [string, Record][] = [ + ["auth0", { user_id: "a1" }], + ["authjs", { id: "a2" }], + ["betterauth", { user_id: "a3" }], + ["firebase", { localId: "a4" }], + ["supabase", { id: "a5" }], + ]; + + test.each(INVALID)( + "%s reports a user with no identifier instead of crashing", + async (key, record) => { + const { users, validationFailed, failures } = await load(key, [ + record, + { ...record, ...identifierFor(key) }, + ]); + + expect(validationFailed).toBe(1); + expect(users).toHaveLength(1); + + expect(failures).toHaveLength(1); + }, + ); +}); + +/** The per-platform source field that becomes a Clerk identifier. */ +function identifierFor(key: string, email = "ok@x.dev"): Record { + if (key === "auth0") return { email, email_verified: true }; + if (key === "authjs") return { email, email_verified: "2024-01-01" }; + if (key === "betterauth") return { email, email_verified: true }; + if (key === "firebase") return { email, emailVerified: true }; + return { email, email_confirmed_at: "2024-01-01 00:00:00+00" }; +} diff --git a/packages/cli-core/src/commands/migrate/sources/supabase.ts b/packages/cli-core/src/commands/migrate/sources/supabase.ts new file mode 100644 index 000000000..1b75f2b85 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/supabase.ts @@ -0,0 +1,165 @@ +import type { SourceEntry } from "../types.ts"; +import { detectStandardHasher, isNullish, routeByVerification, toIsoDate } from "./shared.ts"; + +/** + * Supabase Auth → Clerk transformer. + * + * Works with a `auth.users` export, per + * https://supabase.com/docs/guides/auth/managing-user-data#exporting-users + * + * Supabase records verification as a nullable confirmation timestamp + * (`email_confirmed_at`) rather than a boolean, and stores timestamps in + * PostgreSQL's format (`2024-06-29 20:25:06.126079+00`). + */ + +/** Discord writes display names as `name#0`; the suffix reads as a URL to Clerk. */ +const DISCORD_DISCRIMINATOR = /#\d+$/; + +function stripDiscriminator(value: unknown): string | undefined { + if (typeof value !== "string") return undefined; + return value.replace(DISCORD_DISCRIMINATOR, "").trim() || undefined; +} + +/** How far out a Supabase ban may end and still be a temporary one. */ +const TEMPORARY_BAN_MS = 10 * 365 * 24 * 60 * 60_000; + +/** The nullable `auth.users` columns, by the field each maps to. */ +const NULLABLE_FIELDS = [ + "email", + "emailConfirmedAt", + "password", + "phone", + "phoneConfirmedAt", + "unsafeMetadata", + "bannedUntil", + "deletedAt", + "createdAt", +] as const; + +const supabaseSource = { + key: "supabase", + label: "Supabase", + description: + "Works with a Supabase `auth.users` export. Users whose only social provider is not enabled in Clerk are rejected by the import's checks, unless they can sign in by email or phone code.", + carries: { + passwords: { + level: "yes", + note: "bcrypt and argon2 `encrypted_password` hashes come across, detected per user. Any other hash is dropped, and that user signs in another way.", + }, + mfa: { + level: "no", + note: "Supabase MFA factors are not exported. Users enrol again in Clerk.", + }, + metadata: { + level: "partial", + note: "`raw_user_meta_data` → unsafe metadata, which users can edit, as in Supabase. `raw_app_meta_data` is not carried.", + }, + }, + transformer: { + id: "userId", + email: "email", + email_confirmed_at: "emailConfirmedAt", + first_name: "firstName", + last_name: "lastName", + encrypted_password: "password", + phone: "phone", + phone_confirmed_at: "phoneConfirmedAt", + raw_user_meta_data: "unsafeMetadata", + banned_until: "bannedUntil", + deleted_at: "deletedAt", + created_at: "createdAt", + }, + postTransform: (user) => { + // A CSV export writes SQL NULL as text. Left in, `deleted_at: "NULL"` + // skips every user, and `raw_user_meta_data: "NULL"` fails the row. Only + // the nullable `auth.users` columns are cleared: a last name of "Null" stays. + for (const field of NULLABLE_FIELDS) { + if (isNullish(user[field])) delete user[field]; + } + + user.createdAt = toIsoDate(user.createdAt); + + // Supabase accepts bcrypt and argon2 hashes on import, so `bcrypt` (the + // default it hashes with) is only right for most users, not all. + if (typeof user.password === "string" && user.password) { + const hasher = detectStandardHasher(user.password); + if (hasher) { + user.passwordHasher = hasher; + } else { + delete user.password; + user.passwordDropped = true; + } + } + + // `export supabase` selects this for the provider checks, which read the + // raw rows; Clerk has no place for it, so it isn't a field "Clerk won't store". + delete user.raw_app_meta_data; + + // A soft-deleted user is gone from the app; Supabase scrambles its email. + if (user.deletedAt) user.skipReason = "deleted in Supabase"; + delete user.deletedAt; + + // Supabase stores E.164 without the leading + (14165550123); Clerk needs it. + if (typeof user.phone === "string" && /^\d+$/.test(user.phone)) user.phone = `+${user.phone}`; + + // Supabase bans until a time; a "permanent" ban is just a far-future one. + // Clerk's ban has no end, so only a ban still in force carries, and it then + // stays until someone lifts it in Clerk; a temporary one keeps its end + // date, so the checks can say so. + const bannedUntil = Date.parse(String(toIsoDate(user.bannedUntil))); + if (bannedUntil > Date.now()) { + user.banned = true; + // ponytail: Supabase writes "permanent" as about 100 years out, so a ban + // ending within 10 years reads as temporary; an odd 20-year ban reads + // as permanent. + if (bannedUntil - Date.now() < TEMPORARY_BAN_MS) { + user.banEndsAt = new Date(bannedUntil).toISOString(); + } + } + delete user.bannedUntil; + routeByVerification(user, "email", "emailConfirmedAt", "timestamp"); + routeByVerification(user, "phone", "phoneConfirmedAt", "timestamp"); + + // A basic SQL export has no first_name/last_name columns; the name lives in + // user metadata instead, under whichever key the provider happened to use. + // A CSV carries the metadata as JSON text, still unparsed at this point. + const meta = parseObject(user.unsafeMetadata); + if (meta) { + // Separate name fields map as-is and are never split. + if (!user.firstName && typeof meta.first_name === "string") user.firstName = meta.first_name; + if (!user.lastName && typeof meta.last_name === "string") user.lastName = meta.last_name; + if (!user.firstName) { + const displayName = stripDiscriminator(meta.display_name ?? meta.full_name ?? meta.name); + if (displayName) { + const parts = displayName.split(/\s+/); + user.firstName = parts[0]; + if (parts.length > 1 && !user.lastName) user.lastName = parts.slice(1).join(" "); + } + } + } + + for (const field of ["firstName", "lastName"] as const) { + if (typeof user[field] === "string") { + const cleaned = stripDiscriminator(user[field]); + if (cleaned) user[field] = cleaned; + else delete user[field]; + } + } + }, +} satisfies SourceEntry; + +/** A metadata value as an object, parsing it when a CSV left it as JSON text. */ +function parseObject(value: unknown): Record | undefined { + if (typeof value === "string") { + try { + value = JSON.parse(value); + } catch { + return undefined; + } + } + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +export default supabaseSource; diff --git a/packages/cli-core/src/commands/migrate/sources/workos.ts b/packages/cli-core/src/commands/migrate/sources/workos.ts new file mode 100644 index 000000000..beab06bd8 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/sources/workos.ts @@ -0,0 +1,59 @@ +import type { SourceEntry } from "../types.ts"; +import { routeByVerification } from "./shared.ts"; + +/** + * WorkOS → Clerk transformer. + * + * Works with WorkOS's User Management API. `id` is a `user_…` string and is + * carried through as the Clerk user's `external_id`. WorkOS's own `external_id` + * — the tenant's app ID, when they set one — has nowhere else to go, so it lands + * in private metadata as `workosExternalId`: private because users can edit + * unsafe metadata, and an ID other records key on must not be editable. + * + * **There is no `passwordHasher` default here, and that is deliberate.** Every + * other source names the hasher its platform ships so a digest can be + * verified; WorkOS returns no digest to verify. It accepts password hashes on + * import and never gives them back, and its TOTP secrets are returned on enrol + * only — so a WorkOS migration moves identities, not credentials. Naming a + * hasher here would imply a password column that cannot exist. + * + * WorkOS has no phone number and no username, which is why the map is short: + * those fields have nothing to come from. + */ +const workosSource = { + key: "workos", + label: "WorkOS", + description: + "Works with WorkOS's User Management API. WorkOS returns no password hashes, so imported users sign in by reset or SSO.", + carries: { + passwords: { + level: "no", + note: "WorkOS never returns password hashes. Users reset their password, or sign in with SSO.", + }, + mfa: { level: "no", note: "WorkOS returns TOTP secrets at enrolment only." }, + metadata: { + level: "yes", + note: "`metadata` → unsafe metadata. WorkOS's `external_id` → private metadata `workosExternalId`; the WorkOS `id` becomes the Clerk external ID.", + }, + }, + transformer: { + id: "userId", + email: "email", + email_verified: "emailVerified", + first_name: "firstName", + last_name: "lastName", + metadata: "unsafeMetadata", + external_id: "workosExternalId", + created_at: "createdAt", + }, + postTransform: (user) => { + routeByVerification(user, "email", "emailVerified", "boolean"); + + if (user.workosExternalId) { + user.privateMetadata = { workosExternalId: String(user.workosExternalId) }; + } + delete user.workosExternalId; + }, +} satisfies SourceEntry; + +export default workosSource; diff --git a/packages/cli-core/src/commands/migrate/types.ts b/packages/cli-core/src/commands/migrate/types.ts new file mode 100644 index 000000000..7c8e2d9f2 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/types.ts @@ -0,0 +1,146 @@ +/** + * Shared types for `clerk migrate`. + * + * Ported from the standalone migration-tool's `src/types.ts`. + */ + +import type * as z from "zod"; +import type { userSchema } from "./validator.ts"; + +/** + * Password hashing algorithms Clerk can verify on import. + * + * When migrating users with existing passwords, the source platform's hasher + * must be named so Clerk can validate the digest instead of rejecting it. + */ +export const PASSWORD_HASHERS = [ + "argon2i", + "argon2id", + "awscognito", + "bcrypt", + "bcrypt_peppered", + "bcrypt_sha256_django", + "hmac_sha256_utf16_b64", + "md5", + "md5_salted", + "pbkdf2_sha1", + "pbkdf2_sha256", + "pbkdf2_sha256_django", + "pbkdf2_sha512", + "pbkdf2_sha512_hex", + "scrypt_firebase", + "scrypt_werkzeug", + "sha256", + "sha256_salted", + "md5_phpass", + "phpass", + "ldap_ssha", + "sha512_symfony", +] as const; + +/** A user that has passed schema validation and is ready to import. */ +export type User = z.infer; + +/** Totals for a completed import run. */ +export type ImportSummary = { + totalProcessed: number; + successful: number; + failed: number; + /** + * Users not created because a Ctrl-C or the instance's user quota stopped + * the run first. Most have no line in the run; one whose first create got a + * 429 keeps its `creating` line. A re-run picks up both. + */ + notSent: number; + /** + * Clerk's message when the user quota stopped the run, for the caller to + * print once any progress display is done. + */ + stopReason?: string; + /** + * Users created without the phone Clerk refused, by Clerk's reason. They + * count as imported, so these are warnings, not failures. + */ + droppedPhones: Map; + validationFailed: number; + errorBreakdown: Map; +}; + +/** + * Firebase's scrypt parameters, needed to rebuild a password hash Clerk can + * verify. + * + * Found in the Firebase console under Authentication → Users → (⋮) → Password + * hash parameters. All four are required together; a partial set produces a + * digest that silently fails every sign-in. + */ +export type FirebaseHashConfig = { + base64_signer_key: string; + base64_salt_separator: string; + rounds: number; + mem_cost: number; +}; + +/** + * Per-run values a source may need but cannot read from the user record. + * + * Passed to `postTransform` rather than held in module state so two runs in one + * process — or two test files — cannot see each other's configuration. + */ +export type TransformContext = { + firebaseHashConfig?: FirebaseHashConfig; +}; + +/** + * Result of a transformer's `preTransform` hook. + * + * @property filePath - Path to read from; may differ from the input (e.g. a + * temp file with generated CSV headers). + * @property data - Users already extracted from a wrapper object, when the + * source format nests them. + */ +export type PreTransformResult = { + filePath: string; + data?: Record[]; + /** Column names for a CSV with no header row of its own. */ + csvHeaders?: string[]; +}; + +/** How much of one kind of data a source brings across. */ +export type CarryLevel = "yes" | "no" | "partial"; + +export type Carry = { level: CarryLevel; note: string }; + +/** + * What a source brings across, per kind of data that is easy to lose without + * noticing. Social sign-ins are not listed: no source carries them, and every + * source shares one account-linking note instead. + */ +export type SourceCarries = { passwords: Carry; mfa: Carry; metadata: Carry }; + +/** + * A source: how to get from one platform's export shape to Clerk's import + * shape. + * + * @property transformer - Source field path → Clerk field name. + * @property carries - What comes across: passwords, MFA and metadata. + * @property caveats - Anything else worth knowing before importing. + * @property defaults - Values merged into every user from this platform. + * @property preTransform - Runs before field mapping. + * @property postTransform - Mutates a user after field mapping, given the + * run's {@link TransformContext}. + */ +export type SourceEntry = { + key: string; + label: string; + description: string; + transformer: Record; + carries: SourceCarries; + caveats?: string[]; + defaults?: Record; + preTransform?: ( + filePath: string, + fileType: string, + ) => PreTransformResult | Promise; + postTransform?: (user: Record, context: TransformContext) => void; +}; diff --git a/packages/cli-core/src/commands/migrate/undo.test.ts b/packages/cli-core/src/commands/migrate/undo.test.ts new file mode 100644 index 000000000..83da210bf --- /dev/null +++ b/packages/cli-core/src/commands/migrate/undo.test.ts @@ -0,0 +1,443 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { EXIT_CODE, type CliError } from "../../lib/errors.ts"; +import { useCaptureLog } from "../../test/lib/stubs.ts"; +import { + continueRun, + latestUserLines, + listRuns, + lockFile, + patchRun, + readRun, + startRun, + type RunRecord, +} from "./lib/run-store.ts"; +import { keyInstanceId } from "./lib/target.ts"; +import { undo } from "./undo.ts"; + +const captured = useCaptureLog(); + +let runsDir: string; +let originalFetch: typeof globalThis.fetch; +let requests: { method: string; url: string }[]; +/** Clerk IDs the stubbed instance still holds, with each one's last sign-in. */ +let instanceUsers: Map; +/** Clerk IDs whose DELETE fails with a 500. */ +let failing: Set; +/** external_id → Clerk ID, for users whose create was in flight. */ +let inFlight: Map; +/** external_id → the run whose marker Clerk holds on that in-flight user. */ +let inFlightMarker: Map; + +const IMPORT_STARTED = "2026-09-01T00:00:00.000Z"; +const AFTER_IMPORT = Date.parse("2026-09-02T00:00:00.000Z"); + +beforeAll(() => { + originalFetch = globalThis.fetch; +}); + +afterAll(() => { + globalThis.fetch = originalFetch; +}); + +beforeEach(() => { + runsDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-undo-"))); + requests = []; + failing = new Set(); + inFlight = new Map(); + inFlightMarker = new Map(); + instanceUsers = new Map([ + ["user_a", null], + ["user_b", AFTER_IMPORT], + ]); + + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(input.toString()); + const method = init?.method ?? "GET"; + requests.push({ method, url: url.toString() }); + + if (url.pathname === "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/v1/instance") { + return Response.json({ object: "instance", id: "ins_1", environment_type: "development" }); + } + if (method === "GET" && url.pathname === "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/v1/users" && url.searchParams.has("external_id")) { + return Response.json( + url.searchParams + .getAll("external_id") + .map((externalId) => externalId.replace(/^\+/, "")) + .filter((externalId) => inFlight.has(externalId)) + .map((externalId) => ({ + id: inFlight.get(externalId), + external_id: externalId, + private_metadata: { clerkMigrateRun: inFlightMarker.get(externalId) }, + })), + ); + } + if (method === "GET" && url.pathname === "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/v1/users") { + const ids = url.searchParams.getAll("user_id"); + return Response.json( + ids + .filter((id) => instanceUsers.has(id)) + .map((id) => ({ id, last_sign_in_at: instanceUsers.get(id) })), + ); + } + if (method === "DELETE") { + const id = url.pathname.split("/").pop() as string; + if (failing.has(id)) { + return Response.json({ errors: [{ code: "boom", message: "boom" }] }, { status: 500 }); + } + instanceUsers.delete(id); + return Response.json({ id, deleted: true }); + } + return new Response("unexpected", { status: 500 }); + }) as typeof fetch; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + process.exitCode = 0; + fs.rmSync(runsDir, { recursive: true, force: true }); +}); + +/** An import into `ins_1` that created two users and failed one. */ +function importRun(overrides: Partial = {}): RunRecord { + const run = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + run.update({ startedAt: IMPORT_STARTED }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "b", status: "created", clerkId: "user_b" }); + run.append({ sourceId: "c", status: "failed", error: "taken" }); + const record = run.finish(); + if (Object.keys(overrides).length > 0) run.update(overrides); + return { ...record, ...overrides }; +} + +const options = { secretKey: "sk_test_x", runsDir: "" }; +const withDir = (extra: Record = {}) => ({ ...options, runsDir, ...extra }); +const deletes = () => requests.filter((request) => request.method === "DELETE"); + +describe("refusals, all exit 2 and delete nothing", () => { + const exitCodeOf = async (promise: Promise) => { + const error = (await promise.catch((caught: unknown) => caught)) as CliError; + return error.exitCode; + }; + + test("an unknown run", async () => { + expect(await exitCodeOf(undo("nope", withDir({ yes: true })))).toBe(EXIT_CODE.USAGE); + }); + + test("a run that is not an import", async () => { + const run = startRun(runsDir, { kind: "export", target: { platform: "auth0" } }); + run.finish(); + await expect(undo(run.record.id, withDir({ yes: true }))).rejects.toThrow( + /is an export run\. Only an import run can be undone/, + ); + expect(deletes()).toHaveLength(0); + }); + + test("a run already undone", async () => { + const record = importRun({ status: "undone", undoneBy: "20260901-000000-beef" }); + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /already undone by run 20260901-000000-beef/, + ); + }); + + test("a key that addresses a different instance, naming both", async () => { + const record = importRun({ target: { instanceId: "ins_other", env: "production" } }); + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /imported into instance \(production, ins_other\).*addresses instance \(development, ins_1\)/s, + ); + expect(deletes()).toHaveLength(0); + }); + + // `key_` is the identity fallback when GET /v1/instance failed, e.g. rate + // limited straight after a large import. That is unknown, not different. + test("an unconfirmed instance is not reported as a different one", async () => { + const record = importRun({ + target: { instanceId: "key_0123456789abcdef", env: "development" }, + }); + const error = (await undo(record.id, withDir({ yes: true })).catch((e: unknown) => e)) as Error; + expect(error.message).toContain("Could not confirm"); + expect(error.message).not.toContain("addresses instance"); + expect(deletes()).toHaveLength(0); + }); + + // The import ran while Clerk could not name the instance; the same key is + // the same instance once it can. + test("accepts a run recorded under this key's stand-in ID", async () => { + const record = importRun({ + target: { instanceId: keyInstanceId("sk_test_x"), env: "development" }, + }); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes()).toHaveLength(2); + }); + + test("no consent where nobody can be asked: the preview, then the command", async () => { + const record = importRun(); + + const error = (await undo(record.id, withDir()).catch((caught: unknown) => caught)) as CliError; + + expect(error.exitCode).toBe(EXIT_CODE.USAGE); + expect(error.message).toContain("Pass --yes to confirm"); + expect(captured.err).toContain("Will delete 2 users"); + expect(deletes()).toHaveLength(0); + }); + + // Another process continues the import while the undo previews it. + describe("the run changing during the preview", () => { + const duringPreview = (act: () => void) => { + const stubbed = globalThis.fetch; + let acted = false; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (!acted && new URL(input.toString()).searchParams.has("user_id")) { + acted = true; + act(); + } + return stubbed(input, init); + }) as typeof fetch; + }; + + test("still running: refused by the import run's lock", async () => { + const record = importRun(); + duringPreview(() => fs.writeFileSync(lockFile(runsDir, record.id), String(process.ppid))); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /in use by another process/, + ); + expect(deletes()).toHaveLength(0); + expect(readRun(runsDir, record.id)?.status).not.toBe("undone"); + }); + + test("finished: refused because the preview is out of date", async () => { + const record = importRun(); + duringPreview(() => { + const run = continueRun(runsDir, record); + run.append({ sourceId: "c", status: "created", clerkId: "user_c" }); + run.finish(); + }); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /preview is out of date/, + ); + expect(deletes()).toHaveLength(0); + expect(fs.existsSync(lockFile(runsDir, record.id))).toBe(false); + }); + + test("undone by another undo: refused, and its undo is kept", async () => { + const record = importRun(); + duringPreview(() => + patchRun(runsDir, record.id, { status: "undone", undoneBy: "20260901-000000-beef" }), + ); + + await expect(undo(record.id, withDir({ yes: true }))).rejects.toThrow( + /already undone by run 20260901-000000-beef/, + ); + expect(deletes()).toHaveLength(0); + expect(readRun(runsDir, record.id)?.undoneBy).toBe("20260901-000000-beef"); + expect(listRuns(runsDir).filter((run) => run.kind === "undo")).toHaveLength(0); + }); + }); +}); + +describe("--dry-run", () => { + test("previews the deletes and how many users signed in since, and deletes nothing", async () => { + const record = importRun(); + + await undo(record.id, withDir({ dryRun: true })); + + expect(captured.err).toContain("Target: development instance ins_1"); + expect(captured.err).toContain("Will delete 2 users"); + expect(captured.err).toContain("1 of them has signed in since the import"); + expect(deletes()).toHaveLength(0); + expect(listRuns(runsDir)).toHaveLength(1); + }); + + test("--json returns the target, the run and the preview", async () => { + const record = importRun(); + + await undo(record.id, withDir({ dryRun: true, json: true })); + + expect(JSON.parse(captured.out)).toMatchObject({ + target: { instanceId: "ins_1", keySource: "--secret-key" }, + run: { id: record.id }, + preview: { toDelete: 2, signedInSince: 1, alreadyGone: 0, alreadyDeleted: 0 }, + dryRun: true, + }); + }); +}); + +describe("deleting", () => { + // The run stopped with this user's POST /v1/users in flight: Clerk created + // it, but its ID never reached the run record. + test("finds a user whose create was in flight by external_id, and deletes it", async () => { + const run = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + run.update({ startedAt: IMPORT_STARTED }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "d", status: "creating" }); + run.append({ sourceId: "e", status: "creating" }); + const record = run.finish(); + inFlight.set("d", "user_d"); + inFlightMarker.set("d", record.id); + instanceUsers.set("user_d", null); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes().map((request) => request.url.split("/").pop())).toEqual( + expect.arrayContaining(["user_a", "user_d"]), + ); + expect(deletes()).toHaveLength(2); + }); + + // Same source ID, but no marker from this run: an app or another tool's user. + test("leaves an in-flight match that does not carry the run's marker", async () => { + const run = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + run.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + run.append({ sourceId: "d", status: "creating" }); + const record = run.finish(); + inFlight.set("d", "user_d"); + instanceUsers.set("user_d", null); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes().map((request) => request.url.split("/").pop())).toEqual(["user_a"]); + }); + + // Run A stopped with d's create in flight; a later run B then created d. + // The user Clerk holds is B's, so undoing A leaves it alone. + test("leaves an in-flight user that another import run records as created", async () => { + const runA = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + runA.append({ sourceId: "a", status: "created", clerkId: "user_a" }); + runA.append({ sourceId: "d", status: "creating" }); + const recordA = runA.finish(); + const runB = startRun(runsDir, { + kind: "import", + target: { instanceId: "ins_1", env: "development" }, + source: "clerk", + }); + runB.append({ sourceId: "d", status: "created", clerkId: "user_d" }); + runB.finish(); + inFlight.set("d", "user_d"); + inFlightMarker.set("d", runB.record.id); + instanceUsers.set("user_d", null); + + await undo(recordA.id, withDir({ yes: true })); + + expect(deletes().map((request) => request.url.split("/").pop())).toEqual(["user_a"]); + }); + + // The instance is over its limit for every delete, not just the one that hit it. + test("holds every other delete through a 429's wait", async () => { + const record = importRun(); + const stub = globalThis.fetch; + const started = performance.now(); + const deletedAt: number[] = []; + globalThis.fetch = (async (input: string | URL | Request, init?: RequestInit) => { + if (init?.method !== "DELETE") return stub(input, init); + deletedAt.push(performance.now() - started); + if (deletedAt.length === 1) { + requests.push({ method: "DELETE", url: input.toString() }); + return new Response(JSON.stringify({ errors: [{ code: "x", message: "slow down" }] }), { + status: 429, + headers: { "retry-after": "1" }, + }); + } + return stub(input, init); + }) as typeof fetch; + process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT = "1"; + try { + await undo(record.id, withDir({ yes: true })); + } finally { + delete process.env.CLERK_MIGRATE_CONCURRENCY_LIMIT; + } + + expect(deletedAt).toHaveLength(3); + for (const at of deletedAt.slice(1)) expect(at - deletedAt[0]!).toBeGreaterThanOrEqual(900); + }); + + test("deletes what the import created, records an undo run, and marks the import undone", async () => { + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect( + deletes() + .map((request) => new URL(request.url).pathname) + .sort(), + ).toEqual(["/v1/users/user_a", "/v1/users/user_b"]); + const undoRun = listRuns(runsDir).find((candidate) => candidate.kind === "undo"); + expect(undoRun).toMatchObject({ + status: "complete", + undoes: record.id, + counts: { deleted: 2 }, + }); + expect(readRun(runsDir, record.id)).toMatchObject({ + status: "undone", + undoneBy: undoRun?.id, + }); + expect(process.exitCode).toBe(0); + }); + + test("counts a user already gone from the instance as deleted, without asking Clerk", async () => { + instanceUsers.delete("user_a"); + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect(deletes()).toHaveLength(1); + expect(readRun(runsDir, record.id)?.status).toBe("undone"); + }); + + test("a failed delete leaves the undo partial, exits 1, and a re-run continues it", async () => { + failing.add("user_b"); + const record = importRun(); + + await undo(record.id, withDir({ yes: true })); + + expect(process.exitCode).toBe(1); + const [partial] = listRuns(runsDir).filter((candidate) => candidate.kind === "undo"); + expect(partial).toMatchObject({ status: "partial", counts: { deleted: 1, failed: 1 } }); + expect(readRun(runsDir, record.id)?.status).toBe("partial"); + + failing.clear(); + requests = []; + process.exitCode = 0; + await undo(record.id, withDir({ yes: true })); + + // Only the user that failed is retried, in the same undo run. + expect(deletes().map((request) => new URL(request.url).pathname)).toEqual(["/v1/users/user_b"]); + const undoRuns = listRuns(runsDir).filter((candidate) => candidate.kind === "undo"); + expect(undoRuns).toHaveLength(1); + expect(undoRuns[0]).toMatchObject({ id: partial?.id, status: "complete" }); + expect(latestUserLines(runsDir, partial!.id).get("b")?.status).toBe("deleted"); + expect(readRun(runsDir, record.id)?.status).toBe("undone"); + }); + + test("--json --yes returns the undo run and the result", async () => { + const record = importRun(); + + await undo(record.id, withDir({ yes: true, json: true })); + + expect(JSON.parse(captured.out)).toMatchObject({ + run: { kind: "undo", undoes: record.id, status: "complete" }, + result: { deleted: 2, failed: 0, errors: [] }, + }); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/undo.ts b/packages/cli-core/src/commands/migrate/undo.ts new file mode 100644 index 000000000..211edec2e --- /dev/null +++ b/packages/cli-core/src/commands/migrate/undo.ts @@ -0,0 +1,485 @@ +/** + * `clerk migrate undo ` — delete the users an import created. + * + * The import run is the whole record of what to delete: every source ID whose + * latest line is `created`, by the Clerk ID recorded next to it. Nothing is + * matched by searching the instance, so nothing the import did not create is + * ever in scope. + * + * Refuses (exit 2) rather than guessing when: + * - the resolved key addresses a different instance than the run imported into + * - the run is not an import + * - the run has already been undone + * + * The undo is a run of its own (`kind: "undo"`, `undoes: `). The import is + * marked `undone` only once every user is gone. A partial undo exits 1, and + * running `undo` again continues the same undo run. + */ + +import { bapiRequest } from "../../lib/bapi.ts"; +import { bold, dim, green, red } from "../../lib/color.ts"; +import { BapiError, throwUsageError, throwUserAbort } from "../../lib/errors.ts"; +import { log } from "../../lib/log.ts"; +import { confirm } from "../../lib/prompts.ts"; +import { withSpinner, type SpinnerControls } from "../../lib/spinner.ts"; +import { isAgent, isHuman } from "../../mode.ts"; +import { normalizeErrorMessage } from "./import-users.ts"; +import { resolveLimits, type ResolvedLimits } from "./lib/instance.ts"; +import { RateLimitExceededError, retryOn429 } from "./lib/retry.ts"; +import { + continueRun, + latestUserLines, + listRuns, + liveLockPid, + lockFile, + lockRun, + patchRun, + readRun, + readUserLines, + resolveRunsDir, + runState, + startRun, + type Run, + type RunRecord, +} from "./lib/run-store.ts"; +import { withProgress, type ProgressUpdate } from "./lib/progress.ts"; +import { createApiScheduler, type ApiScheduler } from "./lib/scheduler.ts"; +import { + describeTarget, + keyInstanceId, + printTarget, + resolveClerkTarget, + type ClerkTarget, +} from "./lib/target.ts"; +import { findInFlight, lookupUsers } from "./lib/user-lookup.ts"; + +export type UndoOptions = { + dryRun?: boolean; + yes?: boolean; + json?: boolean; + secretKey?: string; + app?: string; + instance?: string; + runsDir?: string; +}; + +/** One user the import created, still to be deleted. */ +export type UndoUser = { sourceId: string; clerkId: string }; + +export type UndoPreview = { + /** Users that will be deleted. */ + toDelete: number; + /** Of those, how many have signed in since the import started. */ + signedInSince: number; + /** Users the import created that are already gone from the instance. */ + alreadyGone: number; + /** Users an earlier attempt at this undo already deleted. */ + alreadyDeleted: number; +}; + +const plural = (count: number, word: string) => `${count} ${word}${count === 1 ? "" : "s"}`; + +/** Loads the import run, refusing anything that cannot be undone. */ +function readImportRun(runsDir: string, runId: string): RunRecord { + const record = readRun(runsDir, runId); + if (!record) { + throwUsageError(`No run \`${runId}\` in ${runsDir}. Run \`clerk migrate runs\` to list them.`); + } + if (record.kind !== "import") { + throwUsageError( + `Run ${runId} is an ${record.kind} run. Only an import run can be undone.`, + undefined, + undefined, + [{ command: "clerk migrate runs", description: "Find the import run to undo" }], + ); + } + if (record.status === "undone") { + throwUsageError(`Run ${runId} was already undone by run ${record.undoneBy ?? "(unknown)"}.`); + } + if (runState(runsDir, record) === "running") { + throwUsageError( + `Run ${runId} is still running (PID ${liveLockPid(runsDir, runId)}). Wait for it to finish, then undo it. ` + + `If that process is not a migrate run, delete ${lockFile(runsDir, runId)}.`, + ); + } + return record; +} + +/** Refuses to delete from an instance the import did not write to. */ +function assertSameInstance(record: RunRecord, target: ClerkTarget, secretKey: string): void { + if (record.target.instanceId === target.instanceId) return; + // Recorded under the key's stand-in ID while Clerk could not name the + // instance: the same key is the same instance. + if (record.target.instanceId === keyInstanceId(secretKey)) return; + // A `key_` ID is the fallback for an instance lookup that failed. It cannot + // be compared with an `ins_` ID, so this is "unknown", not "different". + const unconfirmed = [record.target.instanceId, target.instanceId].some((id) => + id?.startsWith("key_"), + ); + if (unconfirmed) { + throwUsageError( + `Could not confirm that the resolved key addresses the instance run ${record.id} imported into: ` + + "Clerk did not answer GET /v1/instance (often rate limiting straight after a large import). " + + "Nothing was deleted. Try again in a minute; --verbose shows the response.", + ); + } + throwUsageError( + `Run ${record.id} imported into ${describeTarget(record.target)}, but the resolved key ` + + `addresses ${describeTarget(target)}. Nothing was deleted.\n` + + "Target the instance the import used with --secret-key, --app or --instance.", + ); +} + +/** The latest undo of this import that has not finished its job, if any. */ +function findOpenUndo(runsDir: string, importId: string): RunRecord | undefined { + return listRuns(runsDir).find( + (candidate) => + candidate.kind === "undo" && + candidate.undoes === importId && + runState(runsDir, candidate) !== "complete", + ); +} + +/** + * Every user the import created, minus those an earlier undo already deleted. + * + * `unconfirmed` holds the source IDs whose create was in flight when the run + * stopped: they may exist in Clerk with no ID on record. + */ +function usersToDelete( + runsDir: string, + record: RunRecord, + openUndo: RunRecord | undefined, +): { users: UndoUser[]; unconfirmed: string[]; alreadyDeleted: number } { + const deleted = new Set(); + if (openUndo) { + for (const line of latestUserLines(runsDir, openUndo.id).values()) { + if (line.status === "deleted") deleted.add(line.sourceId); + } + } + + const users: UndoUser[] = []; + const unconfirmed: string[] = []; + for (const line of latestUserLines(runsDir, record.id).values()) { + if (deleted.has(line.sourceId)) continue; + if (line.status === "creating") unconfirmed.push(line.sourceId); + if (line.status !== "created" || !line.clerkId) continue; + users.push({ sourceId: line.sourceId, clerkId: line.clerkId }); + } + return { users, unconfirmed, alreadyDeleted: deleted.size }; +} + +/** + * Reads each user back from the instance, for the preview. + * + * @returns The users still there, and how many of them have signed in since + * the import started — the ones whose deletion someone will notice. + */ +async function inspectUsers(options: { + users: UndoUser[]; + since: string; + secretKey: string; + schedule: ApiScheduler; + spinner?: SpinnerControls; +}): Promise<{ present: UndoUser[]; gone: UndoUser[]; signedInSince: number }> { + const found = await lookupUsers({ + filter: "user_id", + values: options.users.map((user) => user.clerkId), + secretKey: options.secretKey, + schedule: options.schedule, + spinner: options.spinner, + label: "Checking the imported users", + }); + + const byId = new Map(found.map((user) => [user.id, user])); + const since = Date.parse(options.since); + const present = options.users.filter((user) => byId.has(user.clerkId)); + const gone = options.users.filter((user) => !byId.has(user.clerkId)); + const signedInSince = present.filter((user) => { + const lastSignIn = byId.get(user.clerkId)?.last_sign_in_at; + return typeof lastSignIn === "number" && lastSignIn > since; + }).length; + + return { present, gone, signedInSince }; +} + +export type UndoSummary = { + deleted: number; + failed: number; + errorBreakdown: Map; +}; + +/** Deletes each user, rate-limited and 429-retried exactly as the import is. */ +async function deleteUsers(options: { + users: UndoUser[]; + secretKey: string; + limits: ResolvedLimits; + run: Run; + progress?: ProgressUpdate; +}): Promise { + const { users, secretKey, limits, run, progress: report } = options; + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const errorBreakdown = new Map(); + let processed = 0; + let deleted = 0; + let failed = 0; + + const progress = () => report?.({ done: processed, ok: deleted, failed }); + + // A failure on one user must not stop the rest: a half-undone import with + // no record of which half is far worse than a reported failure. + const deleteOne = async (user: UndoUser): Promise => { + const retries: string[] = []; + try { + await retryOn429( + async () => + schedule(async () => + bapiRequest({ method: "DELETE", path: `/v1/users/${user.clerkId}`, secretKey }), + ), + { + // A 429 pauses every delete still queued, as on import. + onRetry: ({ message, delaySeconds }) => { + retries.push(message); + schedule.pause(delaySeconds * 1000); + }, + }, + ); + deleted++; + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "deleted", + ...(retries.length > 0 ? { error: retries.join("; ") } : {}), + }); + } catch (error) { + // Gone already, by the dashboard or another undo: the goal is met. + if (error instanceof BapiError && error.status === 404) { + deleted++; + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "deleted", + reason: "already deleted", + }); + } else { + failed++; + const message = + error instanceof RateLimitExceededError + ? error.message + : ((error as BapiError).longMessage ?? (error as Error).message ?? "Unknown error"); + const code = + error instanceof RateLimitExceededError + ? "429" + : String((error as BapiError).status ?? "unknown"); + const normalized = normalizeErrorMessage(message); + errorBreakdown.set(normalized, (errorBreakdown.get(normalized) ?? 0) + 1); + run.append({ + sourceId: user.sourceId, + clerkId: user.clerkId, + status: "failed", + error: [message, ...retries].join("; "), + code, + }); + } + } + processed++; + progress(); + }; + + progress(); + await Promise.all(users.map(deleteOne)); + return { deleted, failed, errorBreakdown }; +} + +function printPreview(record: RunRecord, preview: UndoPreview): void { + const are = (count: number) => (count === 1 ? "is" : "are"); + log.blank(); + log.info( + `${bold(`Will delete ${plural(preview.toDelete, "user")}`)} created by import run ${record.id}.`, + ); + if (preview.signedInSince > 0) { + const who = + preview.signedInSince === 1 ? "1 of them has" : `${preview.signedInSince} of them have`; + log.warn(`${who} signed in since the import. Deleting removes their accounts and sessions.`); + } + if (preview.alreadyGone > 0) { + log.info( + dim( + `${plural(preview.alreadyGone, "user")} the import created ${are(preview.alreadyGone)} already gone.`, + ), + ); + } + if (preview.alreadyDeleted > 0) { + log.info( + dim( + `${plural(preview.alreadyDeleted, "user")} ${preview.alreadyDeleted === 1 ? "was" : "were"} deleted by an earlier attempt.`, + ), + ); + } +} + +/** The run header shared by every outcome, human and JSON alike. */ +function jsonResult( + target: ClerkTarget, + record: RunRecord, + preview: UndoPreview, + extra: Record = {}, +) { + return { target, run: record, preview, ...extra }; +} + +export async function undo(runId: string, options: UndoOptions = {}): Promise { + const runsDir = await resolveRunsDir(options.runsDir); + const record = readImportRun(runsDir, runId); + const importLines = readUserLines(runsDir, record.id).length; + + const { secretKey, target } = await resolveClerkTarget(options); + if (!options.json) printTarget(target); + assertSameInstance(record, target, secretKey); + + const limits = resolveLimits(secretKey); + const openUndo = findOpenUndo(runsDir, record.id); + const recorded = usersToDelete(runsDir, record, openUndo); + const { alreadyDeleted } = recorded; + + const schedule = createApiScheduler(limits.concurrencyLimit, limits.rateLimit); + const users = [ + ...recorded.users, + ...(await findInFlight({ + runId: record.id, + sourceIds: recorded.unconfirmed, + secretKey, + schedule, + })), + ]; + const { present, gone, signedInSince } = + users.length > 0 + ? await withSpinner("Checking the imported users...", async (spinner) => + inspectUsers({ users, since: record.startedAt, secretKey, schedule, spinner }), + ) + : { present: [], gone: [], signedInSince: 0 }; + + const preview: UndoPreview = { + toDelete: present.length, + signedInSince, + alreadyGone: gone.length, + alreadyDeleted, + }; + + if (!options.json) printPreview(record, preview); + + const command = `clerk migrate undo ${record.id} --yes`; + const nothingLeft = present.length === 0 && gone.length === 0; + + if (options.dryRun) { + if (options.json) { + log.data(JSON.stringify(jsonResult(target, record, preview, { dryRun: true }), null, 2)); + } else { + log.blank(); + log.info(dim(`Dry run: nothing was deleted. Run \`${command}\` to delete them.`)); + } + return; + } + + // Rule 1: nothing is deleted without consent — `--yes`, or a yes at a + // prompt. `--json` never prompts. + if (!nothingLeft && !options.yes) { + const canAsk = !options.json && isHuman() && !isAgent(); + if (!canAsk) { + if (options.json) { + log.data( + JSON.stringify(jsonResult(target, record, preview, { consent: "required" }), null, 2), + ); + } + throwUsageError( + `\`clerk migrate undo\` permanently deletes ${plural(present.length, "user")} and needs consent. Pass --yes to confirm.`, + undefined, + undefined, + [{ command, description: "Delete the users this import created" }], + ); + } + const proceed = await confirm({ + message: `Permanently delete ${plural(present.length, "user")}?`, + default: false, + }); + if (!proceed) throwUserAbort(); + } + + // Gitignored only now, once there is consent to write a run. + await resolveRunsDir(options.runsDir, { write: true }); + + // Held until the import is marked undone, so a re-import cannot continue + // the run while its users are deleted and overwrite the `undone` mark. + const releaseImport = lockRun(runsDir, record.id); + let run: Run; + let summary: UndoSummary; + let undoRecord: RunRecord; + try { + // Another undo that finished during the preview has already marked it. + readImportRun(runsDir, record.id); + // A re-import that continued the run during the preview created users + // the preview never listed. + if (readUserLines(runsDir, record.id).length !== importLines) { + throwUsageError( + `Run ${record.id} changed while this undo was waiting, so the preview is out of date. ` + + `Nothing was deleted. Run \`clerk migrate undo ${record.id}\` again.`, + ); + } + + run = openUndo + ? continueRun(runsDir, openUndo) + : startRun(runsDir, { kind: "undo", target, undoes: record.id, source: record.source }); + + // Users the import created that are no longer there: the undo's goal for + // them is already met, so they count as deleted without another request. + for (const user of gone) { + run.append({ ...user, status: "deleted", reason: "not found in the instance" }); + } + + summary = + present.length > 0 + ? await withProgress({ total: present.length, verb: "deleted" }, async (progress) => + deleteUsers({ users: present, secretKey, limits, run, progress }), + ) + : { deleted: 0, failed: 0, errorBreakdown: new Map() }; + + undoRecord = run.finish(); + if (undoRecord.status === "complete") { + patchRun(runsDir, record.id, { status: "undone", undoneBy: undoRecord.id }); + } + } finally { + releaseImport(); + } + + if (options.json) { + log.data( + JSON.stringify( + jsonResult(target, undoRecord, preview, { + result: { + deleted: summary.deleted, + failed: summary.failed, + errors: [...summary.errorBreakdown].map(([error, count]) => ({ error, count })), + }, + }), + null, + 2, + ), + ); + } else { + log.blank(); + log.info(`${bold("Deleted:")} ${green(String(summary.deleted))}`); + log.info(`${bold("Failed:")} ${red(String(summary.failed))}`); + for (const [error, count] of summary.errorBreakdown) { + log.info(` ${plural(count, "user")}: ${error}`); + } + log.blank(); + log.info(dim(`Undo run ${undoRecord.id}: ${run.dir}`)); + if (undoRecord.status === "complete") { + log.success(`Import run ${record.id} is undone.`); + } else { + log.info(`Run \`clerk migrate undo ${record.id}\` again to retry the users that failed.`); + } + } + + if (undoRecord.status !== "complete") process.exitCode = 1; +} diff --git a/packages/cli-core/src/commands/migrate/validator.test.ts b/packages/cli-core/src/commands/migrate/validator.test.ts new file mode 100644 index 000000000..7cda5b1a3 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import { PASSWORD_HASHERS } from "./types.ts"; +import { userSchema } from "./validator.ts"; + +const base = { userId: "user_1", email: "a@example.com" }; + +describe("userSchema identifiers", () => { + const IDENTIFIER_CASES = [ + ["email", { email: "a@example.com" }, true], + ["emailAddresses array", { emailAddresses: ["a@example.com"] }, true], + ["unverified email", { unverifiedEmailAddresses: ["a@example.com"] }, true], + ["phone", { phone: "+15555550100" }, true], + ["unverified phone", { unverifiedPhoneNumbers: ["+15555550100"] }, true], + ["username", { username: "alice" }, true], + ["nothing", {}, false], + ["empty email array", { email: [] }, false], + ["empty username", { username: "" }, false], + ] as const; + + test.each([...IDENTIFIER_CASES])( + "accepts a user identified by %s: %p -> %p", + (_label, fields, ok) => { + expect(userSchema.safeParse({ userId: "user_1", ...fields }).success).toBe(ok); + }, + ); + + test("reports the identifier failure against the email path", () => { + const result = userSchema.safeParse({ userId: "user_1" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["email"]); + }); +}); + +describe("userSchema passwords", () => { + test("rejects a password without a hasher", () => { + const result = userSchema.safeParse({ ...base, password: "digest" }); + expect(result.success).toBe(false); + if (result.success) return; + expect(result.error.issues[0]?.path).toEqual(["passwordHasher"]); + }); + + test("accepts a password with a valid hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "bcrypt" }).success, + ).toBe(true); + }); + + test("rejects an unknown hasher", () => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: "rot13" }).success, + ).toBe(false); + }); + + test.each([...PASSWORD_HASHERS])("accepts the %s hasher", (hasher) => { + expect( + userSchema.safeParse({ ...base, password: "digest", passwordHasher: hasher }).success, + ).toBe(true); + }); +}); + +describe("userSchema field types", () => { + const FIELD_CASES = [ + ["valid email", { email: "a@example.com" }, true], + // The checks drop an address Clerk would refuse; the schema only wants a string. + ["malformed email", { email: "not-an-email" }, true], + ["email array with one bad entry", { email: ["a@example.com", "nope"] }, true], + ["email that is not a string", { email: 42 }, false], + ["userId missing", { userId: undefined }, false], + ["valid createdAt", { createdAt: "2024-01-01T00:00:00Z" }, true], + ["unparseable createdAt", { createdAt: "yesterday" }, false], + ["integer org limit", { createOrganizationsLimit: 3 }, true], + ["fractional org limit", { createOrganizationsLimit: 1.5 }, false], + ["metadata object", { publicMetadata: { plan: "pro" } }, true], + ["metadata string", { publicMetadata: "pro" }, false], + ["backupCodes array", { backupCodes: ["a", "b"] }, true], + ] as const; + + test.each([...FIELD_CASES])("%s -> %p", (_label, fields, ok) => { + expect(userSchema.safeParse({ ...base, ...fields }).success).toBe(ok); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/validator.ts b/packages/cli-core/src/commands/migrate/validator.ts new file mode 100644 index 000000000..58e08c83f --- /dev/null +++ b/packages/cli-core/src/commands/migrate/validator.ts @@ -0,0 +1,112 @@ +/** + * Zod schema every user is validated against before it reaches BAPI. + * + * Ported from the standalone migration-tool's `src/migrate/validator.ts`. + * + * ============================================================================ + * ONLY EDIT THIS IF YOU ARE ADDING A NEW FIELD. + * Adding support for a new source platform means adding a source, not + * touching the schema. + * ============================================================================ + */ + +import * as z from "zod"; +import { PASSWORD_HASHERS } from "./types.ts"; + +const metadataSchema = z.record(z.string(), z.unknown()); + +const dateStringSchema = z.string().refine((value) => !Number.isNaN(new Date(value).getTime()), { + message: "Expected a valid date string", +}); + +/** Zod enum of the password hashers Clerk accepts on import. */ +export const passwordHasherEnum = z.enum(PASSWORD_HASHERS, { + error: (issue) => + `Unknown password hasher ${JSON.stringify(issue.input)}. Expected one of: ${PASSWORD_HASHERS.join(", ")}`, +}); + +/** + * Validates user data before sending it to Clerk. + * + * Everything is optional except: + * - `userId`, required for tracking, re-runs and `undo` + * - `passwordHasher`, required whenever `password` is present + * - at least one identifier (email, phone or username) + * + * Identifier fields accept either a single value or an array. + */ +export const userSchema = z + .object({ + userId: z.string(), + // Email fields + // Strings, not z.email(): an address Clerk would refuse is dropped by the + // checks, with a warning, rather than taking the whole user down. + email: z.union([z.string(), z.array(z.string())]).optional(), + emailAddresses: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedEmailAddresses: z.union([z.string(), z.array(z.string())]).optional(), + // Phone fields + phone: z.union([z.string(), z.array(z.string())]).optional(), + phoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + unverifiedPhoneNumbers: z.union([z.string(), z.array(z.string())]).optional(), + // User info + username: z.string().optional(), + firstName: z.string().optional(), + lastName: z.string().optional(), + // Password + password: z.string().optional(), + passwordHasher: passwordHasherEnum.optional(), + /** Set by a source that found a password Clerk cannot verify, and left it out. */ + passwordDropped: z.boolean().optional(), + // Set by a source for a user that should not be created at all; the checks + // reject it with this reason. Never sent to Clerk. + skipReason: z.string().optional(), + // 2FA + totpSecret: z.string().optional(), + backupCodesEnabled: z.boolean().optional(), + backupCodes: z.array(z.string()).optional(), + // Metadata + unsafeMetadata: metadataSchema.optional(), + publicMetadata: metadataSchema.optional(), + privateMetadata: metadataSchema.optional(), + // Additional Clerk API fields + banned: z.boolean().optional(), + // When a source's ban on this user was due to end (ISO 8601). Clerk's ban + // has no end, so the checks warn that it stays until lifted. Never sent. + banEndsAt: z.string().optional(), + bypassClientTrust: z.boolean().optional(), + createOrganizationEnabled: z.boolean().optional(), + createOrganizationsLimit: z.number().int().optional(), + createdAt: dateStringSchema.optional(), + deleteSelfEnabled: z.boolean().optional(), + legalAcceptedAt: dateStringSchema.optional(), + skipLegalChecks: z.boolean().optional(), + skipPasswordChecks: z.boolean().optional(), + }) + .refine((data) => !data.password || data.passwordHasher, { + message: "passwordHasher is required when password is provided", + path: ["passwordHasher"], + }) + .refine( + (data) => { + const hasValue = (field: unknown): boolean => { + if (!field) return false; + if (typeof field === "string") return field.length > 0; + if (Array.isArray(field)) return field.length > 0; + return false; + }; + return ( + hasValue(data.email) || + hasValue(data.emailAddresses) || + hasValue(data.unverifiedEmailAddresses) || + hasValue(data.phone) || + hasValue(data.phoneNumbers) || + hasValue(data.unverifiedPhoneNumbers) || + hasValue(data.username) + ); + }, + { + message: + "User must have at least one identifier (email, phone, unverified email, unverified phone, or username)", + path: ["email"], + }, + ); diff --git a/packages/cli-core/src/commands/migrate/wizard.test.ts b/packages/cli-core/src/commands/migrate/wizard.test.ts new file mode 100644 index 000000000..824e0ebb7 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.test.ts @@ -0,0 +1,131 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { listageStubs, useCaptureLog } from "../../test/lib/stubs.ts"; + +useCaptureLog(); + +type Prompt = { message: string; default?: string; validate?: (v?: string) => string | undefined }; +type SelectPrompt = Prompt & { choices: { name: string; value: string }[] }; + +// Registered at file top, before the wizard (or anything it imports) loads. +// This file is the only consumer of the mocked prompt modules. +const mockSelect = mock(async (_config: SelectPrompt) => undefined as unknown); +const mockText = mock(async (_config: Prompt) => "" as unknown); + +mock.module("../../lib/listage.ts", () => ({ + ...listageStubs, + select: (config: SelectPrompt) => mockSelect(config), +})); + +mock.module("../../lib/prompts.ts", () => ({ + confirm: async () => true, + text: (config: Prompt) => mockText(config), + password: async () => "", + editor: async () => "{}", +})); + +const { promptForFile, promptForFirebaseHashConfig, promptForSource } = await import("./wizard.ts"); + +let workDir: string; +let originalCwd: string; + +beforeAll(() => { + originalCwd = process.cwd(); + workDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "clerk-migrate-wizard-"))); + process.chdir(workDir); + fs.writeFileSync(path.join(workDir, "users.json"), "[]"); + fs.writeFileSync(path.join(workDir, "other.csv"), ""); + fs.writeFileSync(path.join(workDir, "notes.txt"), ""); +}); + +afterAll(() => { + process.chdir(originalCwd); + fs.rmSync(workDir, { recursive: true, force: true }); +}); + +beforeEach(() => { + mockSelect.mockReset(); + mockText.mockReset(); +}); + +/** The config object passed to the Nth `text`/`select` prompt. */ +const textCall = (index: number): Prompt | undefined => mockText.mock.calls[index]?.[0]; +const selectCall = (index: number): SelectPrompt | undefined => mockSelect.mock.calls[index]?.[0]; + +describe("promptForSource", () => { + test("is built from the registry, so every platform appears", async () => { + mockSelect.mockResolvedValue("auth0"); + + expect(await promptForSource()).toBe("auth0"); + expect(selectCall(0)?.choices.map((choice) => choice.value)).toEqual([ + "clerk", + "auth0", + "authjs", + "betterauth", + "firebase", + "supabase", + "workos", + ]); + }); + + test("labels each choice with the source's display name", async () => { + mockSelect.mockResolvedValue("clerk"); + await promptForSource(); + expect(selectCall(0)?.choices.map((choice) => choice.name)).toContain("Better Auth"); + }); +}); + +describe("promptForFile", () => { + const validate = async () => { + mockText.mockResolvedValue("users.json"); + await promptForFile(); + return textCall(0)?.validate as (value?: string) => string | undefined; + }; + + test("returns the trimmed path", async () => { + mockText.mockResolvedValue(" users.json "); + expect(await promptForFile()).toBe("users.json"); + }); + + test("accepts an existing JSON or CSV file", async () => { + const check = await validate(); + expect(check("users.json")).toBeUndefined(); + expect(check("other.csv")).toBeUndefined(); + }); + + test.each([ + ["", /required/], + ["nope.json", /File not found/], + ["notes.txt", /\.json or \.csv/], + ])("rejects %p", async (value, message) => { + const check = await validate(); + expect(check(value)).toMatch(message); + }); +}); + +describe("promptForFirebaseHashConfig", () => { + test("collects all four parameters as a set", async () => { + mockText + .mockResolvedValueOnce("SIGNER") + .mockResolvedValueOnce("Bw==") + .mockResolvedValueOnce("8") + .mockResolvedValueOnce("14"); + + expect(await promptForFirebaseHashConfig()).toEqual({ + base64_signer_key: "SIGNER", + base64_salt_separator: "Bw==", + rounds: 8, + mem_cost: 14, + }); + }); + + // An export with no password hashes needs none of them. + test("stops when the signer key is left blank", async () => { + mockText.mockResolvedValueOnce(""); + + expect(await promptForFirebaseHashConfig()).toBeUndefined(); + expect(mockText).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli-core/src/commands/migrate/wizard.ts b/packages/cli-core/src/commands/migrate/wizard.ts new file mode 100644 index 000000000..cb89f03b5 --- /dev/null +++ b/packages/cli-core/src/commands/migrate/wizard.ts @@ -0,0 +1,94 @@ +/** + * The prompts behind an interactive `clerk migrate import`. + * + * Each one fills in exactly one thing the command was not given: the file, + * the source when the file does not name its own, and Firebase's hash + * parameters when neither the flags nor the export carry them. + * + * Nothing here runs for an agent, a non-TTY run or `--json`: `run` raises a + * usage error naming what to pass instead. + */ + +import { select } from "../../lib/listage.ts"; +import { log } from "../../lib/log.ts"; +import { text } from "../../lib/prompts.ts"; +import { fileExists, getFileType } from "./lib/transform.ts"; +import { sources } from "./sources/registry.ts"; +import type { FirebaseHashConfig } from "./types.ts"; + +/** Trims a description down to a single readable hint line. */ +function hint(description: string): string { + const firstSentence = description.split(". ")[0] ?? description; + return firstSentence.length > 96 ? `${firstSentence.slice(0, 93)}...` : firstSentence; +} + +/** Asks which platform the file came from. Built from the registry. */ +export async function promptForSource(): Promise { + return select({ + message: "Which platform did this file come from?", + choices: sources.map((entry) => ({ + name: entry.label, + value: entry.key, + description: hint(entry.description), + })), + }); +} + +/** Asks for the file to import. */ +export async function promptForFile(): Promise { + const answer = await text({ + message: "Path to the exported user file (JSON or CSV)", + validate: (value) => { + const file = value?.trim(); + if (!file) return "A file path is required"; + if (!fileExists(file)) return `File not found: ${file}`; + if (!getFileType(file)) return "Provide a .json or .csv file"; + return undefined; + }, + }); + return answer.trim(); +} + +async function askNumber(label: string): Promise { + const answer = await text({ + message: label, + validate: (value) => { + const parsed = Number(value?.trim()); + return Number.isInteger(parsed) && parsed > 0 ? undefined : "Enter a positive whole number"; + }, + }); + return Number(answer.trim()); +} + +/** + * Collects Firebase's four hash parameters. + * + * Asked as a set because a partial set produces a digest that verifies against + * nothing. Pressing enter at the first leaves the config unset, which is + * correct for an export with no password hashes. + */ +export async function promptForFirebaseHashConfig(): Promise { + log.info( + "Firebase password hashes need the project's hash parameters. Find them in the Firebase console under Authentication → Users → (⋮) → Password hash parameters.", + ); + log.info("Pass the four --firebase-* flags to skip these prompts on the next run."); + + const signerKey = ( + await text({ + message: "base64 signer key (leave blank if this export has no passwords)", + }) + ).trim(); + if (!signerKey) return undefined; + + // Blank is a valid separator: some projects have none. + const saltSeparator = ( + await text({ message: "base64 salt separator (leave blank if the project has none)" }) + ).trim(); + + return { + base64_signer_key: signerKey, + base64_salt_separator: saltSeparator, + rounds: await askNumber("rounds"), + mem_cost: await askNumber("mem cost"), + }; +} diff --git a/packages/cli-core/src/commands/update/index.test.ts b/packages/cli-core/src/commands/update/index.test.ts new file mode 100644 index 000000000..243cedf49 --- /dev/null +++ b/packages/cli-core/src/commands/update/index.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { CliError, ERROR_CODE } from "../../lib/errors.ts"; +import { asRegistryError } from "./index.ts"; + +// Offline, `loggedFetch` throws NETWORK_UNREACHABLE; `clerk update` keeps +// reporting the registry, as it did before that existed. +test.each([ + [ + "a refused connection", + new CliError("Could not reach x", { code: ERROR_CODE.NETWORK_UNREACHABLE }), + ], + ["a timeout", new Error("The operation was aborted.")], +])("%s is registry_unreachable", (_label, error) => { + expect((asRegistryError(error) as CliError).code).toBe(ERROR_CODE.REGISTRY_UNREACHABLE); +}); + +test("a registry that answered keeps its own code", () => { + const error = new CliError("Registry returned HTTP 500.", { code: ERROR_CODE.UPDATE_FAILED }); + expect(asRegistryError(error)).toBe(error); +}); diff --git a/packages/cli-core/src/commands/update/index.ts b/packages/cli-core/src/commands/update/index.ts index b83fb2dfe..72ffbb8ae 100644 --- a/packages/cli-core/src/commands/update/index.ts +++ b/packages/cli-core/src/commands/update/index.ts @@ -259,6 +259,24 @@ async function confirmUpdate(currentVersion: string, latestVersion: string): Pro // ── Main ───────────────────────────────────────────────────────────────────── +/** + * What a failed registry lookup throws. + * + * A registry that answered — badly, or without the requested channel — + * already carries its own code, and a Ctrl-C is the user's decision, not the + * network's. Only transport and timeout failures are genuinely "unreachable", + * and retrying is only right for those. `loggedFetch` reports a refused + * connection as NETWORK_UNREACHABLE, naming the host; this command has always + * called it the registry. + */ +export function asRegistryError(error: unknown): unknown { + const unreachable = error instanceof CliError && error.code === ERROR_CODE.NETWORK_UNREACHABLE; + if ((error instanceof CliError && !unreachable) || isCancelled(error)) return error; + return new CliError("Could not reach npm registry. Check your network connection.", { + code: ERROR_CODE.REGISTRY_UNREACHABLE, + }); +} + export async function update(options: UpdateOptions): Promise { if (IS_DEV_BUILD) { log.info(`Running development build (${CURRENT_VERSION}); update not applicable.`); @@ -272,14 +290,7 @@ export async function update(options: UpdateOptions): Promise { const [latest, installDirs] = await Promise.all([ withSpinner("Checking for updates...", async () => fetchLatestVersion(channel)).catch( (error: unknown) => { - // A registry that answered — badly, or without the requested channel — - // already carries its own code, and a Ctrl-C is the user's decision, - // not the network's. Only transport and timeout failures are genuinely - // "unreachable", and retrying is only right for those. - if (error instanceof CliError || isCancelled(error)) throw error; - throw new CliError("Could not reach npm registry. Check your network connection.", { - code: ERROR_CODE.REGISTRY_UNREACHABLE, - }); + throw asRegistryError(error); }, ), getInstallerPackageDirs(), diff --git a/packages/cli-core/src/lib/bapi-command.test.ts b/packages/cli-core/src/lib/bapi-command.test.ts index b2c67e5ca..50e7c9da7 100644 --- a/packages/cli-core/src/lib/bapi-command.test.ts +++ b/packages/cli-core/src/lib/bapi-command.test.ts @@ -324,7 +324,7 @@ describe("bapi-command", () => { expect(fetchApplicationSpy).not.toHaveBeenCalled(); }); - test("describes the resolved app and instance target", async () => { + test("describes the app and instance --app resolves, and says so", async () => { resolveAppContextSpy.mockResolvedValue({ appId: "app_123", appLabel: "My App", @@ -333,7 +333,7 @@ describe("bapi-command", () => { }); await expect(describeBapiTarget({ app: "app_123", instance: "prod" })).resolves.toBe( - "My App (production)", + "My App (production) via --app", ); expect(resolveAppContextSpy).toHaveBeenCalledWith({ @@ -342,14 +342,41 @@ describe("bapi-command", () => { }); }); - test("returns no target description when only a secret key is available", async () => { - resolveAppContextSpy.mockRejectedValue( - new CliError("linked profile missing", { - code: ERROR_CODE.NOT_LINKED, - }), + test("names the linked profile as the key's source", async () => { + resolveAppContextSpy.mockResolvedValue({ + appId: "app_123", + appLabel: "My App", + instanceId: "ins_dev", + instanceLabel: "development", + }); + + await expect(describeBapiTarget({})).resolves.toBe( + "My App (development) via the linked profile", ); + }); + + test("names --secret-key rather than describing nothing", async () => { + await expect(describeBapiTarget({ secretKey: "sk_test_123" })).resolves.toBe( + "the instance behind --secret-key", + ); + expect(resolveAppContextSpy).not.toHaveBeenCalled(); + }); + + // resolveBapiSecretKey takes an exported key before the linked profile, so + // naming the linked app here would point at an instance the key may not be for. + test("an exported CLERK_SECRET_KEY wins over the linked profile, matching resolveBapiSecretKey", async () => { + process.env.CLERK_SECRET_KEY = "sk_test_env"; + resolveAppContextSpy.mockResolvedValue({ + appId: "app_123", + appLabel: "Linked App", + instanceId: "ins_dev", + instanceLabel: "development", + }); - await expect(describeBapiTarget({ secretKey: "sk_test_123" })).resolves.toBeUndefined(); + await expect(describeBapiTarget({})).resolves.toBe( + "the instance behind the CLERK_SECRET_KEY env var", + ); + expect(resolveAppContextSpy).not.toHaveBeenCalled(); }); test("describes an unclaimed keyless target without querying the account", async () => { @@ -365,11 +392,21 @@ describe("bapi-command", () => { test("an explicit --secret-key wins over a keyless target on disk, matching resolveBapiSecretKey", async () => { resolveKeylessTargetSpy.mockResolvedValue({ secretKey: "sk_test_disk", source: ".env.local" }); - await expect(describeBapiTarget({ secretKey: "sk_test_explicit" })).resolves.toBeUndefined(); + await expect(describeBapiTarget({ secretKey: "sk_test_explicit" })).resolves.toBe( + "the instance behind --secret-key", + ); expect(resolveKeylessTargetSpy).not.toHaveBeenCalled(); }); + test("describes nothing when nothing is linked, leaving the error to the key lookup", async () => { + resolveAppContextSpy.mockRejectedValue( + new CliError("linked profile missing", { code: ERROR_CODE.NOT_LINKED }), + ); + + await expect(describeBapiTarget({})).resolves.toBeUndefined(); + }); + test("throws instance-not-found when the resolved instance is missing from the application", async () => { resolveAppContextSpy.mockResolvedValue({ appId: "app_123", diff --git a/packages/cli-core/src/lib/bapi-command.ts b/packages/cli-core/src/lib/bapi-command.ts index 5581358af..8eb18b489 100644 --- a/packages/cli-core/src/lib/bapi-command.ts +++ b/packages/cli-core/src/lib/bapi-command.ts @@ -19,39 +19,44 @@ interface ResolveBapiSecretKeyOptions { cwd?: string; } +/** + * Names the instance {@link resolveBapiSecretKey} will reach, and where its key + * comes from, for prose ("… for My App (development) via the linked profile"). + * + * Walks the same chain in the same order. An exported `CLERK_SECRET_KEY` wins + * over a linked profile there, so it wins here too: naming the linked app for + * a key that may belong to any app at all would point the reader at the wrong + * instance. + */ export async function describeBapiTarget( options: ResolveBapiSecretKeyOptions, ): Promise { - // An explicit --secret-key wins in resolveBapiSecretKey, so it has no - // app/instance context to describe. - if (options.secretKey) return undefined; - - // Mirrors resolveBapiSecretKey's precedence: an unclaimed keyless project has - // no app/instance to describe, only the key's own source. - const keyless = await resolveKeylessTarget({ - app: options.app, - instance: options.instance, - cwd: options.cwd, - }); - if (keyless) { - return `this accountless application (secret key from ${keyless.source})`; - } + if (options.secretKey) return "the instance behind --secret-key"; - try { + if (options.app) { const ctx = await resolveAppContext({ app: options.app, instance: options.instance, cwd: options.cwd, }); - return `${ctx.appLabel} (${ctx.instanceLabel})`; + return `${ctx.appLabel} (${ctx.instanceLabel}) via --app`; + } + + if (process.env.CLERK_SECRET_KEY) return "the instance behind the CLERK_SECRET_KEY env var"; + + // An unclaimed keyless project has no app/instance to describe, only the + // key's own source. + const keyless = await resolveKeylessTarget({ instance: options.instance, cwd: options.cwd }); + if (keyless) { + return `this accountless application (secret key from ${keyless.source})`; + } + + try { + const ctx = await resolveAppContext({ instance: options.instance, cwd: options.cwd }); + return `${ctx.appLabel} (${ctx.instanceLabel}) via the linked profile`; } catch (error) { - if ( - error instanceof CliError && - error.code === ERROR_CODE.NOT_LINKED && - (options.secretKey || process.env.CLERK_SECRET_KEY) - ) { - return undefined; - } + // Nothing to name; resolveBapiSecretKey raises the error worth reading. + if (error instanceof CliError && error.code === ERROR_CODE.NOT_LINKED) return undefined; throw error; } } diff --git a/packages/cli-core/src/lib/config.ts b/packages/cli-core/src/lib/config.ts index 41943b990..a7e1f085b 100644 --- a/packages/cli-core/src/lib/config.ts +++ b/packages/cli-core/src/lib/config.ts @@ -308,7 +308,7 @@ export async function resolveProfile(cwd: string): Promise< return undefined; } -const INSTANCE_ALIASES: Record = { +export const INSTANCE_ALIASES: Record = { dev: "development", development: "development", prod: "production", diff --git a/packages/cli-core/src/lib/errors.ts b/packages/cli-core/src/lib/errors.ts index 6d7eb9209..1f8e94532 100644 --- a/packages/cli-core/src/lib/errors.ts +++ b/packages/cli-core/src/lib/errors.ts @@ -104,6 +104,8 @@ export const ERROR_CODE = { INSTALLER_NOT_FOUND: "installer_not_found", /** The npm registry was unreachable. */ REGISTRY_UNREACHABLE: "registry_unreachable", + /** A request never reached the server — DNS, refused connection, no route. */ + NETWORK_UNREACHABLE: "network_unreachable", /** Production instance was created but came back without a domain. */ DEPLOY_DOMAIN_MISSING: "deploy_domain_missing", /** diff --git a/packages/cli-core/src/lib/fetch.test.ts b/packages/cli-core/src/lib/fetch.test.ts index 505131b9e..9ad2432d6 100644 --- a/packages/cli-core/src/lib/fetch.test.ts +++ b/packages/cli-core/src/lib/fetch.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { _resetUserAgentCache, loggedFetch } from "./fetch.ts"; import { _resetInterruptState, abortInFlight, beginInterrupt, interruptSignal } from "./signals.ts"; import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from "./config.ts"; +import { CliError } from "./errors.ts"; const originalFetch = globalThis.fetch; @@ -34,6 +35,31 @@ describe("loggedFetch", () => { expect(init.headers.get("User-Agent")).toBe("Custom/1.0"); }); + test("reports a connection failure as a CliError naming the host", async () => { + globalThis.fetch = mock(async () => { + // Bun's own shape for DNS failures, refused connections and no-route. + const error: NodeJS.ErrnoException = new Error( + "Unable to connect. Is the computer able to access the url?", + ); + error.code = "ConnectionRefused"; + throw error; + }) as unknown as typeof fetch; + + const failure = loggedFetch("https://example.test/x", { tag: "test" }); + await expect(failure).rejects.toThrow(/Could not reach example\.test/); + await expect(failure).rejects.toBeInstanceOf(CliError); + }); + + test("leaves a non-connection failure alone", async () => { + globalThis.fetch = mock(async () => { + throw new DOMException("The operation was aborted.", "AbortError"); + }) as unknown as typeof fetch; + + await expect(loggedFetch("https://example.test/x", { tag: "test" })).rejects.toThrow( + /operation was aborted/, + ); + }); + test("preserves other caller-provided headers", async () => { globalThis.fetch = mock( async () => new Response("ok", { status: 200 }), diff --git a/packages/cli-core/src/lib/fetch.ts b/packages/cli-core/src/lib/fetch.ts index 1d73ea9ff..36289e164 100644 --- a/packages/cli-core/src/lib/fetch.ts +++ b/packages/cli-core/src/lib/fetch.ts @@ -8,6 +8,7 @@ * every network error. See `.claude/rules/debug-logging.md`. */ +import { CliError, ERROR_CODE } from "./errors.ts"; import { log } from "./log.ts"; import { interruptSignal } from "./signals.ts"; import { withNetworkAccess } from "./host-execution.ts"; @@ -74,6 +75,23 @@ function interruptSignalFor( return own ? AbortSignal.any([own, interruptSignal()]) : interruptSignal(); } +/** + * Bun reports every connection-level failure — DNS, refused, no route — as a + * bare `Error` reading "Unable to connect. Is the computer able to access the + * url?", which names neither the host nor what wanted it, and which the global + * handler can only render as `unexpected_error`. Everything else, an aborted + * request included, is left exactly as thrown. + */ +function asConnectionError(error: unknown, url: string): unknown { + if ((error as NodeJS.ErrnoException | null)?.code !== "ConnectionRefused") return error; + + const host = URL.parse(url)?.host ?? url; + return new CliError( + `Could not reach ${host}. Check your network connection (or VPN) and try again.`, + { code: ERROR_CODE.NETWORK_UNREACHABLE }, + ); +} + export async function loggedFetch(url: URL | string, options: LoggedFetchInit): Promise { const { tag, bestEffort, ignoreInterrupt, ...init } = options; const method = init.method ?? "GET"; @@ -85,7 +103,9 @@ export async function loggedFetch(url: URL | string, options: LoggedFetchInit): const response = await withNetworkAccess( { operation: "connect", target: urlStr, label: tag, bestEffort }, async () => fetch(url, { ...init, headers, signal }), - ); + ).catch((error: unknown) => { + throw asConnectionError(error, urlStr); + }); if (!response.ok) { // Clone so the caller can still consume the body for error construction. const body = await response.clone().text(); diff --git a/packages/cli-core/src/lib/git.ts b/packages/cli-core/src/lib/git.ts index da4f4e955..34fced039 100644 --- a/packages/cli-core/src/lib/git.ts +++ b/packages/cli-core/src/lib/git.ts @@ -1,4 +1,4 @@ -import { resolve } from "node:path"; +import { join, resolve } from "node:path"; import { log } from "./log.ts"; const $ = Bun.$; @@ -100,3 +100,22 @@ export function normalizeGitRemoteUrl(raw: string): string { return url.toLowerCase(); } + +/** + * Adds `entry` to the project's `.gitignore` unless it is already listed. + * + * The CLI writes files into a user's repository that must not be committed, + * such as the keyless breadcrumb. Creating one without this is how a live + * credential ends up in a tracked file. + */ +export async function ensureGitignoreEntry(cwd: string, entry: string): Promise { + const gitignorePath = join(cwd, ".gitignore"); + const content = await Bun.file(gitignorePath) + .text() + .catch(() => ""); + const lines = content.split("\n").map((l) => l.trim()); + if (lines.includes(entry)) return; + const separator = content && !content.endsWith("\n") ? "\n" : ""; + await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); + log.debug(`git: added ${entry} to .gitignore`); +} diff --git a/packages/cli-core/src/lib/json-body.ts b/packages/cli-core/src/lib/json-body.ts index f5ff5a7d2..31b18cc5f 100644 --- a/packages/cli-core/src/lib/json-body.ts +++ b/packages/cli-core/src/lib/json-body.ts @@ -85,7 +85,7 @@ function preview(raw: string): string { * literal `"` inside doubled; everywhere else single quotes pass the value * through untouched. */ -function quoteArg(value: string): string { +export function quoteArg(value: string): string { if (/^[A-Za-z0-9_\-./:@]+$/.test(value)) return value; if (process.platform === "win32") return `"${value.replace(/"/g, '""')}"`; return `'${value.replace(/'/g, `'\\''`)}'`; diff --git a/packages/cli-core/src/lib/keyless-target.ts b/packages/cli-core/src/lib/keyless-target.ts index 99a63edbb..0ae9492b9 100644 --- a/packages/cli-core/src/lib/keyless-target.ts +++ b/packages/cli-core/src/lib/keyless-target.ts @@ -47,6 +47,12 @@ const ENV_FILES = [".env", ".env.local"]; */ const SDK_KEYLESS_FILE = [".clerk", ".tmp", "keyless.json"]; +/** + * The `source` a key found in the SDK's own keyless file carries. Exported so + * a caller can tell that source apart without restating the path. + */ +export const SDK_KEYLESS_SOURCE = SDK_KEYLESS_FILE.join("/"); + /** * Reads the SDK's own keyless file, ignoring a partially-written one. * Exported for `init`, whose keep-the-existing-app guard must recognise an @@ -130,7 +136,7 @@ export async function findLocalSecretKey(cwd: string): Promise { const sdkApp = await readSdkKeylessApp(cwd); if (!sdkApp?.secretKey) return undefined; - return { secretKey: sdkApp.secretKey, source: SDK_KEYLESS_FILE.join("/") }; + return { secretKey: sdkApp.secretKey, source: SDK_KEYLESS_SOURCE }; } /** The publishable key a keyless project holds locally, when one can be found. */ diff --git a/packages/cli-core/src/lib/keyless.ts b/packages/cli-core/src/lib/keyless.ts index e8c769e2b..5994010f5 100644 --- a/packages/cli-core/src/lib/keyless.ts +++ b/packages/cli-core/src/lib/keyless.ts @@ -5,6 +5,7 @@ import { detectPublishableKeyName, detectSecretKeyName, detectEnvFile } from "./ import { parseEnvFile, mergeEnvVars, serializeEnvFile } from "./dotenv.ts"; import { BapiError } from "./errors.ts"; import { loggedFetch } from "./fetch.ts"; +import { ensureGitignoreEntry } from "./git.ts"; import { log } from "./log.ts"; const BREADCRUMB_DIR = ".clerk"; @@ -113,18 +114,6 @@ function breadcrumbPath(cwd: string): string { return join(cwd, BREADCRUMB_DIR, BREADCRUMB_FILE); } -async function ensureGitignoreEntry(cwd: string, entry: string): Promise { - const gitignorePath = join(cwd, ".gitignore"); - const content = await Bun.file(gitignorePath) - .text() - .catch(() => ""); - const lines = content.split("\n").map((l) => l.trim()); - if (lines.includes(entry)) return; - const separator = content && !content.endsWith("\n") ? "\n" : ""; - await Bun.write(gitignorePath, `${content}${separator}${entry}\n`); - log.debug(`Added ${entry} to .gitignore`); -} - export async function writeKeylessBreadcrumb(cwd: string, claimToken: string): Promise { await ensureGitignoreEntry(cwd, BREADCRUMB_DIR + "/"); await mkdir(join(cwd, BREADCRUMB_DIR), { recursive: true }); diff --git a/packages/cli-core/src/lib/next-steps.ts b/packages/cli-core/src/lib/next-steps.ts index f0967af8f..487b26ed1 100644 --- a/packages/cli-core/src/lib/next-steps.ts +++ b/packages/cli-core/src/lib/next-steps.ts @@ -75,6 +75,14 @@ export const NEXT_STEPS = { "Run `clerk apps list` to see your other applications", "Run `clerk config pull` to inspect the live configuration of this instance", ], + MIGRATE_DONE: (runId: string) => [ + `Run \`clerk migrate runs ${runId}\` to see what happened to each user`, + `Run \`clerk migrate undo ${runId}\` to delete the users it created`, + ], + MIGRATE_DONE_WITH_ERRORS: (runId: string) => [ + `Run \`clerk migrate runs ${runId}\` to see every user that failed and why`, + `Run \`clerk migrate undo ${runId}\` to delete the users it created`, + ], } as const; /** @@ -82,7 +90,30 @@ export const NEXT_STEPS = { * Only shown in human/interactive mode — agents get AGENT_PROMPT instead. */ export function printNextSteps(steps: readonly string[]): void { - if (!isHuman() || steps.length === 0) return; + if (!isHuman()) return; + renderNextSteps(steps); +} + +/** + * The same suggestions, on the paths a human never takes: agent mode and a + * non-TTY, where `printNextSteps` and `withGutter`'s outro both print nothing. + * + * A no-op for a human, who gets them from the outro — so a caller pairs this + * with `setNextSteps` rather than choosing between the two. + * + * Opt-in rather than folded into `printNextSteps`, because most steps are a + * nudge towards a command someone might like to run next. The migrate ones are + * not: they name the log file holding the per-user record of what landed, and + * the command that undoes the run. An agent that imported 10,000 users and lost + * 300 of them needs both, and has no gutter to read them from. + */ +export function printAgentNextSteps(steps: readonly string[]): void { + if (isHuman()) return; + renderNextSteps(steps); +} + +function renderNextSteps(steps: readonly string[]): void { + if (steps.length === 0) return; for (const step of steps) { log.info(` ${cyan("\u2192")} ${step}`); } diff --git a/packages/cli-core/src/lib/prompts-instructions.test.ts b/packages/cli-core/src/lib/prompts-instructions.test.ts new file mode 100644 index 000000000..b159c1a2e --- /dev/null +++ b/packages/cli-core/src/lib/prompts-instructions.test.ts @@ -0,0 +1,29 @@ +/** + * The multiselect footer hack, checked against the real @clack/prompts. + * + * Kept out of `prompts.test.ts`, which mocks the whole module — the one thing + * worth verifying here is that the real export is still a live array clack + * reads at render time. A clack upgrade that froze it, replaced it, or rendered + * a copy would drop `a: all` from the legend silently, and nothing else in the + * suite would notice. + */ + +import { test, expect } from "bun:test"; +import { MULTISELECT_INSTRUCTIONS } from "@clack/prompts"; + +// Importing for the module-level side effect is the point. +await import("./prompts.ts"); + +const legend = () => MULTISELECT_INSTRUCTIONS.join(" • ").replaceAll(/\[[0-9;]*m/g, ""); + +test("the multiselect legend advertises select-all", () => { + expect(legend()).toContain("a: all"); +}); + +test("confirm stays last, where readers expect it", () => { + expect(legend().endsWith("Enter: confirm")).toBe(true); +}); + +test("the keys clack actually binds are the ones named", () => { + expect(legend()).toBe("↑/↓ to navigate • Space: select • a: all • Enter: confirm"); +}); diff --git a/packages/cli-core/src/lib/prompts.ts b/packages/cli-core/src/lib/prompts.ts index 5d7ea6633..e4a0a9503 100644 --- a/packages/cli-core/src/lib/prompts.ts +++ b/packages/cli-core/src/lib/prompts.ts @@ -7,17 +7,35 @@ import { confirm as clackConfirm, isCancel, + MULTISELECT_INSTRUCTIONS, text as clackText, password as clackPassword, multiselect as clackMultiselect, type Option as ClackOption, } from "@clack/prompts"; import { editAsync } from "external-editor"; +import { dim } from "./color.ts"; import { throwUserAbort } from "./errors.ts"; import { ttyContext } from "./listage.ts"; import { log } from "./log.ts"; import { whileAwaitingUser } from "./signals.ts"; +/** + * Advertise select-all in the multiselect footer. + * + * `MultiSelectPrompt` binds `a` to toggle every option (and `i` to invert), but + * clack's instruction footer has never listed them and takes no override — the + * array below is the only seam, and it is read fresh on every render. So a + * genuinely useful key stays undiscoverable unless each call site spells it out + * in its own message, which is worse: it is a property of the prompt, not of + * any one question. + * + * Inserted second-to-last so `Enter: confirm` stays where readers expect it. + * `i` is left out deliberately — inverting is rarely what anyone wants, and a + * four-item legend stops being scannable. + */ +MULTISELECT_INSTRUCTIONS.splice(MULTISELECT_INSTRUCTIONS.length - 1, 0, `${dim("a:")} all`); + type ValidationResult = string | Error | true | undefined; type Validate = (value: string | undefined) => ValidationResult | Promise; type SyncValidate = (value: string | undefined) => string | Error | undefined; diff --git a/packages/cli-core/src/lib/spinner.ts b/packages/cli-core/src/lib/spinner.ts index 2222cb838..886248b33 100644 --- a/packages/cli-core/src/lib/spinner.ts +++ b/packages/cli-core/src/lib/spinner.ts @@ -107,7 +107,9 @@ export async function withGutter( let nextSteps: readonly string[] | undefined; const controls: GutterControls = { setNextSteps(steps) { - nextSteps = steps; + // Empty is ignored rather than stored: `outro([])` would render the + // "Next steps" header with no bullets under it. Matches printNextSteps. + if (steps.length > 0) nextSteps = steps; }, }; diff --git a/packages/cli-core/src/test/integration/completion.test.ts b/packages/cli-core/src/test/integration/completion.test.ts index e1e10d2e1..35a1152ba 100644 --- a/packages/cli-core/src/test/integration/completion.test.ts +++ b/packages/cli-core/src/test/integration/completion.test.ts @@ -220,6 +220,12 @@ describe("generateCompletions", () => { expect(names).toContain("DELETE"); }); + test("completes --source with the built-in migrate sources", () => { + const names = completionNames("migrate", "import", "--source", ""); + expect(names).toContain("clerk"); + expect(names).toContain("betterauth"); + }); + test("returns empty for options with unknown values (file paths)", () => { const result = complete("config", "pull", "--output", ""); expect(result.completions).toEqual([]); @@ -272,6 +278,12 @@ describe("generateCompletions", () => { expect(names).toContain("settings"); }); + test("migrate sources: suggests the built-in sources", () => { + const names = completionNames("migrate", "sources", ""); + expect(names).toContain("supabase"); + expect(names).toContain("workos"); + }); + test("open dashboard: filters subpaths by prefix", () => { const names = completionNames("open", "dashboard", "u"); expect(names).toContain("users"); diff --git a/packages/cli-core/src/test/integration/lib/harness.ts b/packages/cli-core/src/test/integration/lib/harness.ts index 6a99749fb..f4132d348 100644 --- a/packages/cli-core/src/test/integration/lib/harness.ts +++ b/packages/cli-core/src/test/integration/lib/harness.ts @@ -96,6 +96,7 @@ mock.module( getGitRepoIdentifier: async () => mockState.gitRepoIdentifier, getGitNormalizedRemote: async () => mockState.gitNormalizedRemote, normalizeGitRemoteUrl: (url: string) => url, + ensureGitignoreEntry: async () => {}, }) satisfies typeof import("../../../lib/git.ts"), ); @@ -115,7 +116,7 @@ mock.module( // ── Prompt queue (drives lib/prompts.ts and lib/listage.ts mocks) ──────────── -type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor"; +type PromptType = "select" | "search" | "input" | "confirm" | "password" | "editor" | "multiselect"; const promptQueues: Record = { select: [], @@ -124,6 +125,7 @@ const promptQueues: Record = { confirm: [], password: [], editor: [], + multiselect: [], }; function dequeuePrompt(name: PromptType) { @@ -164,6 +166,7 @@ export const mockPrompts = { input: (...responses: string[]) => promptQueues.input.push(...responses), password: (...responses: string[]) => promptQueues.password.push(...responses), editor: (...responses: string[]) => promptQueues.editor.push(...responses), + multiselect: (...responses: unknown[][]) => promptQueues.multiselect.push(...responses), }; function resetPromptQueues() { @@ -203,8 +206,12 @@ mock.module("../../../lib/listage.ts", () => ({ }, })); +// Every export of the real module must appear here: a missing one is a module +// link error at import time, not a failed prompt, so it takes down every test +// in the file the moment any command imports it. mock.module("../../../lib/prompts.ts", () => ({ confirm: dequeuePrompt("confirm"), + multiselect: dequeuePrompt("multiselect"), text: dequeuePrompt("input"), password: dequeuePrompt("password"), editor: dequeuePrompt("editor"), diff --git a/packages/cli-core/src/test/lib/stubs.ts b/packages/cli-core/src/test/lib/stubs.ts index d80d15e8d..0e780caf9 100644 --- a/packages/cli-core/src/test/lib/stubs.ts +++ b/packages/cli-core/src/test/lib/stubs.ts @@ -220,9 +220,14 @@ export const gitStubs = { * Stubs for `lib/prompts.ts` — the @clack/prompts-backed wrapper. Default * responses return benign values so tests can mock the module without * configuring each prompt explicitly. + * + * Must cover every export of the real module: an omission is a module link + * error at import time, which takes down the whole test file rather than + * failing one prompt. */ export const libPromptsStubs = { confirm: async () => true, + multiselect: async () => [], text: async () => "", password: async () => "", editor: async () => "{}", diff --git a/scripts/check-bun-version.ts b/scripts/check-bun-version.ts index a05b2d4b7..74f576892 100644 --- a/scripts/check-bun-version.ts +++ b/scripts/check-bun-version.ts @@ -9,6 +9,13 @@ * producing hundreds of order-dependent failures. Bun does not enforce * `engines.bun` at install time, so this preflight fails loudly instead. * + * A second, lower constraint rides along: the DB-backed `clerk migrate export` + * commands read MySQL through `Bun.sql` rather than `mysql2`. Verified against + * MySQL 8.4 -- the adapter landed in Bun 1.2.21, but VARBINARY/BLOB columns + * came back as lossily decoded strings until 1.3.6, which would silently + * corrupt exported password hashes. The 1.3.13 floor above already covers it; + * do not drop below 1.3.6 if the `--parallel` requirement ever goes away. + * * Usage: * bun run scripts/check-bun-version.ts */ diff --git a/slice-prs.md b/slice-prs.md new file mode 100644 index 000000000..1b2f19259 --- /dev/null +++ b/slice-prs.md @@ -0,0 +1,721 @@ +# `clerk migrate` in slices + +Ship `migrate` in small PRs instead of one 20k-line PR. #479 (`ra/integrate-migration-tool-into-cli`) stays open as the reference, and each slice ports from it. + +**Reference commit: `7a820406`.** That's #479's tip after the review fixes (`3eec00bd` to `d6201932`) and the merge of `main` at `a16595ff`. Because #479 now contains current `main`, a file copied from `7a820406` onto a branch off `origin/main` brings no stale `main` code with it. The earlier pin, `cea11672`, predates every review fix. Don't port from it. + +**The slices never write to #479:** + +- No slice branch starts from, cherry-picks from, merges, or rebases `ra/integrate-migration-tool-into-cli`. Code comes over by copying files from the reference commit (see "Reference implementation" below). +- #479 itself is never retargeted, marked ready, merged, or closed while the slices are in flight. Decide what to do with it after slice 6 merges. +- #479 can still take fixes. When one lands after the slices are cut, see "When #479 gets another fix" below. + +**Don't port these from #479:** `slice-prs.md` (this plan), `.changeset/integrate-migration-tool-into-cli.md` (each slice writes its own), and the `.gitignore` lines for `commands/migrate/logs/` (that folder no longer exists). + +**Size:** 6 slices, shipped as 10 PRs. Slice 4 is five PRs, one per provider. + +## How it ships: one PR at a time, gated until the last one + +The PRs are stacked, but they're reviewed and merged one at a time. Slice 1 merges to `main`, then slice 2 gets reviewed and merges, and so on. The stack is never reviewed or merged as a whole. + +**The gate stays on until slice 6 merges:** + +- Slice 1 adds `CLERK_EXPERIMENTAL=migrate`. Without it, `migrate` doesn't appear in `clerk --help` or completions, and it refuses to run. +- Slices 2 to 5 each add commands or sources behind that same gate. +- Slice 6 is the only PR that removes the gate. + +**What that means for every PR:** + +- **It stands alone on `main`.** Each merge reaches `@canary`, and a release from `main` between slices ships whatever has merged so far, still gated. So every PR passes CI by itself, and its `migrate/README.md` documents only the commands that have merged. `readme.test.ts` enforces this, because it fails on any documented flag the binary rejects. +- **It has its own changeset.** Create it with the `changesets` skill. Until slice 6, the text says the command is experimental and needs `CLERK_EXPERIMENTAL=migrate`. Slice 6's changeset announces `clerk migrate`. +- **It's tested against `@canary` before the next one is marked ready.** Run `npm i -g clerk@canary` with `CLERK_EXPERIMENTAL=migrate` set, then run that slice's testing process (to be defined). + +## Slice 1: `clerk migrate import ` for Clerk and Supabase files + +- **Ported from #479:** + - the Clerk and Supabase sources and the schema + - the checks against the real instance + - consent: `--yes` or a prompt + - `--dry-run`, `--allow-partial`, and `--skip-legal-checks` + - the run store, the target line, and `--json` + - throttling, 429 backoff, and the dev user limit + - the progress bar, and exit 130 on Ctrl-C +- **New:** the `CLERK_EXPERIMENTAL` gate. +- **Not in slice 1:** continuing a stopped run, `runs`, `undo`, `export`, importing by export run ID, other sources, and `--source ./file.ts`. + +Import goes first because the risky behavior lives there: writing to production, consent, quota, and silent data loss. It's also the smallest slice that's useful on its own. + +## After that + +| Slice | Scope | PRs | +| ----- | ----------------------------------------------------------------------------------------------------------------------- | --- | +| 2 | `runs`, `undo`, and continuing a stopped run (`--new-run`, adopting in-flight creates, finishing `pending` identifiers) | 1 | +| 3 | The export envelope, `export clerk`, `export supabase`, and importing by export run ID | 1 | +| 4 | One PR per provider, each with its export and import: Firebase, Auth0, WorkOS, Better Auth, Auth.js | 5 | +| 5 | Custom sources (`--source ./file.ts`) and `migrate sources` | 1 | +| 6 | Remove the gate, then the docs and skills written against the final help output | 1 | + +**Where the non-`migrate` files from #479 go:** + +- Slice 3: the `.gitignore` lines for `exports/` and `*service-account*.json`, and the MySQL note in `scripts/check-bun-version.ts`. +- Slice 4c (WorkOS): the WorkOS entry in `commands/init/scan.ts`. +- Slice 6: the root `README.md` entry. + +**How each slice is proven:** + +- unit and integration tests; +- a testing process for each slice (to be defined); +- once exports land, a weekly E2E run against real provider accounts, with credentials in 1Password, to catch providers changing underneath us. + +**Separate small PRs off `main`, not part of any slice.** These can merge in any order, before or during the slices: + +- `build:compile` in CI (`.github/workflows/ci.yml`, plus the root `build` script) +- `network_unreachable` for connection failures in `lib/fetch.ts` and `lib/errors.ts`, together with the `clerk update` fix from `0365fb84` that keeps reporting `registry_unreachable` when offline (`commands/update/index.ts` and its test) +- `bun --no-env-file` for `test:e2e` (`b0705144`: the root `package.json` and its line in `CLAUDE.md`) +- the `a: all` hint in the multiselect footer (`lib/prompts.ts`) +- the `describeBapiTarget` wording change (`lib/bapi-command.ts`) + +The Playwright pin is gone from this list. `main` moved Playwright to 1.62.1 in #409, and #479 no longer differs from it. + +## Branching + +Every branch below is new. `migrate/slice-1` starts from `origin/main`, and each branch above it starts from the branch below it. None of them starts from `ra/integrate-migration-tool-into-cli`. + +Stack the branches in one line: + +``` +main + └─ migrate/slice-1 import + └─ migrate/slice-2 runs, undo, continuing a run + └─ migrate/slice-3 exports + └─ migrate/slice-4a Firebase + └─ migrate/slice-4b Auth0 + └─ migrate/slice-4c WorkOS + └─ migrate/slice-4d Better Auth + └─ migrate/slice-4e Auth.js + └─ migrate/slice-5 custom sources, `migrate sources` + └─ migrate/slice-6 remove the gate, docs +``` + +The provider PRs don't depend on each other, but each one edits `sources/registry.ts`, `export/registry.ts`, `sources.test.ts`, and `README.md`. As siblings, every merge would conflict on those four files. In one line, nothing conflicts, and slice 5 sits on top of every provider. + +**Build only a little ahead.** Only one PR is in review at a time, and every review fix rebases every branch above it. Keep one or two draft branches above the PR in review, not all ten. A slice built early gets rebased through every review round below it. + +**Workflow:** + +1. Open slice 1 as a ready PR with `--base main`. Open each slice above it as a draft with `gh pr create --draft --base `. Don't reuse #479 for any slice. +2. Exactly one PR is ready at a time: the lowest one that hasn't merged. Its base is `main`. +3. For a review fix, commit on that PR's branch. Then run `git rebase --update-refs` from the highest branch you've built, and push each branch with `--force-with-lease`. +4. Merge the ready PR. The repo allows squash and rebase merges, not merge commits. Either way, `main` gets new commits, so the branch above still carries the old ones. Drop them, using the merged PR's head commit: + + ```bash + git fetch origin + MERGED=$(gh pr view --json headRefOid -q .headRefOid) + git rebase --onto origin/main "$MERGED" --update-refs + ``` + + Then push each remaining branch with `--force-with-lease`, and delete the merged branch locally. + +5. GitHub deletes the merged branch and moves the next PR's base to `main` on its own. Check that the next PR's diff shows only its own slice. +6. Test the merge on `@canary` with the slice's testing process (to be defined). Then mark the next PR ready. + +**When testing changes a behavior,** such as what the checks reject, fix it in the PR in review. Step 3 carries the fix into the branches above it. + +**When #479 gets another fix:** + +1. Note the new commit, and re-pin the reference (`git -C ../cli-pr479 checkout --detach `). +2. If the files it touches belong to a slice that hasn't merged, port the change into that slice. Then rebase the branches above it, as in step 3. +3. If that slice has already merged, open a small `fix(migrate):` PR off `main`. Rebase the stack onto `main` after it merges. + +--- + +# Slice 1 implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Ship a hidden, env-gated `clerk migrate import ` for Clerk and Supabase files. It checks every user against the real instance before writing, writes only with consent, records every user in a run, and speaks JSON. It merges to `main` on its own, ahead of slice 2. + +**Architecture:** A new `commands/migrate/` group in `packages/cli-core`, registered hidden and gated by `CLERK_EXPERIMENTAL=migrate`. An import goes through five stages: + +1. Settle the file, the source, and the target. Print the target. +2. Load the users: read the file, map it through the source, then normalize and validate it. +3. Run `checkImport()` against the instance. `--dry-run` stops here. +4. Refuse on any reject unless `--allow-partial` is passed, then ask for consent. +5. Import through the scheduler, writing one run line per user, under a progress bar. + +Every module is ported from #479. Slice 1 leaves out the parts that belong to later slices. + +**Tech stack:** Bun, TypeScript, Commander, `@clack/prompts` wrappers in `lib/prompts.ts`, zod 4, csv-parser, `bun:test`. + +**Spec:** [clerk migrate: CLI shape proposal](https://claude.ai/code/artifact/981d8d00-05ce-4545-bd19-1a886f2b788b) (Claude Doc). + +**Reference implementation:** clerk/cli#479 at `7a820406`. Below, `REF/` means `packages/cli-core/src/` in a checkout of that commit: + +```bash +git worktree add --detach ../cli-pr479 7a820406 +``` + +This worktree is read-only. It sits on a detached commit, so nothing done there can reach #479's branch. Don't commit in it. Copy files out of it, or run `git checkout 7a820406 -- ` from a slice branch. + +## Global constraints + +- **Branching:** branch fresh from `origin/main` in a new worktree, and open a new PR with `--base main`. Never build on, push to, or rebase `ra/integrate-migration-tool-into-cli`, and never run `gh pr edit`, `gh pr ready`, `gh pr merge`, or `gh pr close` on #479. +- **Stands alone:** slice 1 merges before slice 2 is reviewed, so nothing in it may depend on a later slice. The README, help, completion, and tests describe only `import`. +- **The gate:** + - `migrate` must not appear in `clerk --help`, completions, or the root README unless `CLERK_EXPERIMENTAL` includes `migrate`. + - With the experiment off, `migrate` is registered as a hidden stub with no subcommands, help disabled, and any arguments accepted. Every invocation exits 2 with code `experiment_disabled`, including `--help` and malformed arguments. Completion has no children to walk. + - `CLERK_EXPERIMENTAL` is a comma-separated list. Names are trimmed and case-insensitive, and unknown names are ignored. +- **The target prints first.** Once the target resolves, a human sees `Target: , instance ` and `Key from: ` before the checks. `--json` carries the same facts as `target`. +- **The target must agree with the key.** An `--instance` that the key from `--secret-key` or `CLERK_SECRET_KEY` doesn't address is refused with exit 2. +- **Consent.** Import writes to Clerk only when a human answers yes at a TTY prompt (`Import N users?`, default no), or when `--yes` is passed. Without either (an agent, a non-TTY run, or `--json`), it prints the checks and exits 2 with the exact command to run. Printed commands shell-quote their paths and keep `--json`. `--json` never prompts. +- **`--dry-run`** runs the checks against the real instance and writes nothing. It exits 2 when the real run would be refused, and 0 otherwise. +- **Rejects.** Any reject refuses the import (exit 2) and prints the command that adds `--allow-partial`. With `--allow-partial`, the rest import and each reject is recorded as `skipped` with its reason. +- **Unknown password hasher.** An unrecognized hasher aborts the whole run before anything is sent. +- **Exit codes:** + - `0`: every attempted user was created, or a dry run that would succeed. + - `1`: any user failed. + - `2`: a usage error or a refusal. + - `130`: Ctrl-C. In-flight requests abort and the process dies by SIGINT. This is why import shows a progress bar (`lib/progress.ts`) instead of a clack spinner: the spinner exits 0 on Ctrl-C. +- **Output streams.** UI goes to stderr. JSON goes to stdout through `log.data`. +- **Input files.** JSON, CSV, and NDJSON (Auth0's bulk export). A BOM prefix is read. A file that isn't valid JSON is named in the error. +- **The run store:** + - Location: `--runs-dir`, then `CLERK_MIGRATE_DIR`, then `/.clerk/migrate/`. The project root is the linked profile's directory, then the git toplevel, then the cwd. + - `.clerk/` is added to `.gitignore` only once there is consent to write a run, and only for the default location. + - Each run is a folder `YYYYMMDD-HHmmss-xxxx/` holding `run.json`, `users.ndjson`, and `lock`. Folders are `0700`. Folders and locks are created exclusively, so a run-ID collision or a lock race fails instead of sharing a folder. A lock holding this process's own PID is stale. + - A user's `creating` line is written as its `POST /v1/users` goes out. When the create gets no answer (an abort, a network error, or a 5xx), the line stays `creating` instead of becoming `failed`. + - A `created` line lists the extra emails and phones that haven't attached yet as `pending`. A later `created` line without `pending` means they're done. + - Every append is synchronous, and a failed append throws. + - The run format is final in this slice, because slice 2 reads it once slice 1 is on `main`. +- **Throughput:** + - The instance type comes from the key prefix: `sk_live_` is production, anything else is development. + - Default rate limits: 100 req/s for production and 10 req/s for development. Concurrency defaults to `floor(rate × 0.095)`. + - The env overrides are `CLERK_MIGRATE_RATE_LIMIT`, `CLERK_MIGRATE_CONCURRENCY_LIMIT`, and `CLERK_MIGRATE_DEV_USER_LIMIT`. A non-numeric or non-positive value falls back to the default. +- **Repo rules.** Follow `.claude/rules/*.md` (commands, errors, completion, promises, testing, changesets, interrupts). Run `bun run format && bun run lint && bun run typecheck && bun run test` before every commit. +- **Changeset.** One changeset for the slice (Task 9), created with the `changesets` skill. + +## File structure + +``` +packages/cli-core/src/ + lib/experimental.ts experimental.test.ts # CLERK_EXPERIMENTAL parsing (new) + lib/git.ts # + export ensureGitignoreEntry (moved from lib/keyless.ts) + lib/errors.ts # + ERROR_CODE.EXPERIMENT_DISABLED + lib/config.ts # export INSTANCE_ALIASES + lib/json-body.ts # export quoteArg + lib/next-steps.ts # + printAgentNextSteps, MIGRATE_DONE(_WITH_ERRORS) + lib/spinner.ts # ignore an empty setNextSteps([]) + commands/completion/__complete.ts # + --source values + commands/migrate/ + index.ts index.test.ts # gate stub, or the group with `import` + README.md readme.test.ts + types.ts # User, PASSWORD_HASHERS, SourceEntry, ImportSummary + validator.ts validator.test.ts # userSchema + run.ts run.test.ts run-interactive.test.ts # the `import` action + import-users.ts import-users.test.ts # create + attach, one run line per user + wizard.ts wizard.test.ts # promptForFile, promptForSource + sources/ + registry.ts # clerk + supabase; resolveSource, sourceKeys + shared.ts clerk.ts supabase.ts + sources.test.ts + lib/ + assume-yes.ts # -y read below the action + transform.ts transform.test.ts # read JSON/CSV/NDJSON → map → normalize → validate + instance.ts instance.test.ts # instance type, limits, dev user limit + retry.ts retry.test.ts # 429 backoff + scheduler.ts scheduler.test.ts # rate + concurrency + progress.ts progress.test.ts # the import progress bar + target.ts target.test.ts # resolveClerkTarget, printTarget + run-store.ts run-store.test.ts # write side of the run store + clerk-config.ts clerk-config.test.ts # FAPI settings, user count, Clerk's OAuth providers + user-lookup.ts # batched GET /v1/users + supabase-providers.ts supabase-providers.test.ts + analysis.ts analysis.test.ts + readiness.ts readiness.test.ts + modify-settings.ts modify-settings.test.ts # the `clerk config patch` offers + checks.ts checks.test.ts # checkImport +test/e2e/migrate.test.ts +``` + +## How to port a module + +Every task below ports files from `REF/`, using the same steps: + +1. Copy the test file from `REF/` and trim the cases the task names. +2. Run it with `bun test --isolate `. Expect it to fail with "Cannot find module". +3. Copy the module from `REF/` and apply the task's trims. +4. Run the test again. Expect it to pass. +5. Run the full CI set, then commit. + +When a task says "as-is", copy the file unchanged. + +--- + +### Task 1: Experimental gate and hidden `migrate` group + +**Files:** + +- Create: `lib/experimental.ts`, `lib/experimental.test.ts` +- Create: `commands/migrate/index.ts`, `commands/migrate/README.md` (a stub saying the command is experimental and gated) +- Modify: `lib/errors.ts` (add `EXPERIMENT_DISABLED: "experiment_disabled"`), `cli-program.ts` (append `registerMigrate` to `registrants`) +- Modify: `lib/git.ts`, `lib/keyless.ts`, `test/lib/stubs.ts`, `test/integration/lib/harness.ts`. Move `ensureGitignoreEntry` from `keyless.ts` into `git.ts` as an export, exactly as `REF/lib/git.ts` has it, and add it to `gitStubs` and to the harness's `git.ts` mock. +- Test: `commands/migrate/index.test.ts` (gate cases only), `cli-program.test.ts` (existing help-ordering test must still pass) + +**Interfaces:** + +- `isExperimentEnabled(name: string, env?: NodeJS.ProcessEnv): boolean` +- `requireExperiment(name: string, env?: NodeJS.ProcessEnv): void`: throws a `CliError` with code `experiment_disabled` and exit code 2 +- `registerMigrate(program: Program, env?: NodeJS.ProcessEnv): void` + +- [ ] **Step 1: Write `lib/experimental.test.ts`** + +```ts +import { describe, expect, test } from "bun:test"; +import { CliError, EXIT_CODE } from "./errors.ts"; +import { isExperimentEnabled, requireExperiment } from "./experimental.ts"; + +describe("isExperimentEnabled", () => { + test.each([ + { value: undefined, expected: false }, + { value: "", expected: false }, + { value: "migrate", expected: true }, + { value: " Migrate ", expected: true }, + { value: "other,migrate", expected: true }, + { value: "other, MIGRATE ,x", expected: true }, + { value: "migrates", expected: false }, + { value: "other", expected: false }, + ])("CLERK_EXPERIMENTAL=$value → $expected", ({ value, expected }) => { + expect(isExperimentEnabled("migrate", { CLERK_EXPERIMENTAL: value })).toBe(expected); + }); +}); + +describe("requireExperiment", () => { + test("throws a usage-level CliError naming the variable", () => { + try { + requireExperiment("migrate", {}); + throw new Error("expected throw"); + } catch (error) { + expect(error).toBeInstanceOf(CliError); + expect((error as CliError).code).toBe("experiment_disabled"); + expect((error as CliError).exitCode).toBe(EXIT_CODE.USAGE); + expect((error as CliError).message).toContain("CLERK_EXPERIMENTAL=migrate"); + } + }); + + test("passes when enabled", () => { + expect(() => requireExperiment("migrate", { CLERK_EXPERIMENTAL: "migrate" })).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run it and confirm it fails.** Run `bun test packages/cli-core/src/lib/experimental.test.ts`. It should fail with "Cannot find module './experimental.ts'". + +- [ ] **Step 3: Implement `lib/experimental.ts`** + +```ts +import { CliError, ERROR_CODE, EXIT_CODE } from "./errors.ts"; + +function enabledExperiments(env: NodeJS.ProcessEnv): Set { + return new Set( + (env.CLERK_EXPERIMENTAL ?? "") + .split(",") + .map((name) => name.trim().toLowerCase()) + .filter(Boolean), + ); +} + +export function isExperimentEnabled(name: string, env: NodeJS.ProcessEnv = process.env): boolean { + return enabledExperiments(env).has(name.toLowerCase()); +} + +export function requireExperiment(name: string, env: NodeJS.ProcessEnv = process.env): void { + if (isExperimentEnabled(name, env)) return; + throw new CliError( + `\`clerk ${name}\` is experimental. Set CLERK_EXPERIMENTAL=${name} to use it.`, + { code: ERROR_CODE.EXPERIMENT_DISABLED, exitCode: EXIT_CODE.USAGE }, + ); +} +``` + +- [ ] **Step 4: Write the gate tests** in `commands/migrate/index.test.ts`. Read `test/integration/lib/harness.ts` first to confirm that `clerk.raw` builds a fresh program per call, since the env must be set before the program is built. Cover: + - With the experiment off: + - `clerk --help` doesn't contain `migrate`. + - Each of these exits 2 with `"code":"experiment_disabled"` in agent mode: `migrate`, `migrate --help`, `migrate import users.json --yes`, `migrate --not-a-flag`, `help migrate`, and `--verbose migrate import x.json`. + - `__complete migrate ""` offers nothing. + - With the experiment on: `clerk --help` lists `migrate`. + + `clerk help migrate` goes through Commander's help path, not the stub's action. Override the stub's `helpInformation` to call `requireExperiment`, so that path exits 2 as well. + +- [ ] **Step 5: Implement `commands/migrate/index.ts`** + +```ts +import type { Program } from "../../cli-program.ts"; +import { isExperimentEnabled, requireExperiment } from "../../lib/experimental.ts"; + +export function registerMigrate(program: Program, env: NodeJS.ProcessEnv = process.env): void { + if (!isExperimentEnabled("migrate", env)) { + const stub = program + .command("migrate", { hidden: true }) + .helpOption(false) + .allowUnknownOption() + .allowExcessArguments() + .argument("[args...]") + .action(() => requireExperiment("migrate", env)); + stub.helpInformation = () => { + requireExperiment("migrate", env); + return ""; + }; + return; + } + // Task 9 registers the group and `import` here. +} +``` + +- [ ] **Step 6: Run the full suite.** `bun run test` must pass with `CLERK_EXPERIMENTAL` unset, including `cli-program.test.ts` help ordering. + +- [ ] **Step 7: Commit**: `feat(migrate): add hidden migrate group behind CLERK_EXPERIMENTAL` + +--- + +### Task 2: Schema, sources, and the transform pipeline + +**Files:** + +- Port as-is: `types.ts`, `validator.ts`, `validator.test.ts`, `sources/shared.ts`, `sources/clerk.ts`, `sources/supabase.ts` +- Port with trims: `sources/registry.ts`, `sources/sources.test.ts`, `lib/transform.ts`, `lib/transform.test.ts` +- Modify: `packages/cli-core/package.json`: add `"csv-parser": "^3.2.1"` and `"zod": "^4.4.3"`, then run `bun install` + +**Interfaces:** + +- `userSchema`, `passwordHasherEnum` (validator.ts) +- `User`, `PASSWORD_HASHERS`, `SourceEntry`, `ImportSummary`, `TransformContext` (types.ts) +- `sources`, `sourceKeys()`, `getSource(key)`, `resolveSource(value): Promise`, `ACCOUNT_LINKING_NOTE` (registry.ts) +- `loadUsersFromFile(file, key, options?)`, `readRawUsers(file, key)`, `transformUsers(...)`, `validatePreparedUsers(...)`, `resolveImportFilePath`, `fileExists`, `getFileType` (transform.ts) + +**Trims:** + +- `sources/registry.ts`: + - The `sources` array holds `clerkSource` and `supabaseSource` only. + - Remove `customSources`, `registerCustomSource`, `__resetCustomSourcesForTesting`, `isSourcePath`, and the `loadCustomSource` import. + - `allSources()` returns `sources`. + - `resolveSource` keeps only the built-in branch, and its unknown-key error drops the custom-source hint. +- `lib/transform.ts`: remove the `isEnvelope` branch in `readUsersFromFile` and the `export-file.ts` import. A JSON file must be an array (or whatever a source's `preTransform` unwraps). Keep NDJSON and BOM handling. +- `sources/sources.test.ts`: keep the `shared`, `clerk`, and `supabase` cases. Drop the other sources. +- `lib/transform.test.ts`: drop the envelope cases. + +`TransformContext` keeps its `firebaseHashConfig` field. It's a type, and slice 4 fills it in. `PASSWORD_HASHERS` stays the full list, including `phpass`. + +- [ ] **Step 1: Port the tests, then the modules, as described in "How to port a module".** Run `bun test --isolate packages/cli-core/src/commands/migrate/validator.test.ts packages/cli-core/src/commands/migrate/sources packages/cli-core/src/commands/migrate/lib/transform.test.ts`. +- [ ] **Step 2: Check the behaviors the review depends on.** These must pass in the ported tests; add any that are missing: + - Supabase: + - A `raw_user_meta_data` name splits into first and last names, and a one-word name becomes the first name. This also works on CSV input. + - A numeric phone gains a leading `+`. + - A soft-deleted user gets a `skipReason`. + - An active `banned_until` becomes `banned`. + - A bcrypt or argon2 hash keeps its hasher, and any other hash is dropped with `passwordDropped: true`. + - Clerk: + - Primary identifiers consolidate without nesting arrays. + - An unverified primary email or phone stays unverified. + - A Dashboard CSV says it carries no metadata, and the TAB the Dashboard puts before formula-like values is stripped. + - The pipeline: + - CSV values are coerced (lists, booleans, JSON metadata). Booleans read case-insensitively, plus `t` and `f`. + - A numeric user ID is read as a string. + - A BOM-prefixed CSV or JSON file reads, and NDJSON reads. + - A file that isn't valid JSON is named in the error. + - An unknown `passwordHasher` throws a `CliError` naming the row. + - Unknown fields are counted in `unknownFields`. +- [ ] **Step 3: Commit**: `feat(migrate): map Clerk and Supabase files onto the import schema` + +--- + +### Task 3: Throughput and progress + +**Files:** port as-is: `lib/instance.ts`, `lib/retry.ts`, `lib/scheduler.ts`, `lib/progress.ts`, and their tests. + +**Interfaces:** + +- `detectInstanceType(secretKey): "dev" | "prod"` +- `resolveLimits(secretKey, env?): ResolvedLimits` +- `resolveDevUserLimit(env?)`, `DEV_USER_LIMIT = 100`, `MAX_RETRIES = 5`, `RETRY_DELAY_MS` +- `retryOn429(fn, { onRetry?, maxRetries?, defaultDelayMs? })`, `RateLimitExceededError`, `readRetryAfter` +- `createApiScheduler(concurrencyLimit, rateLimit): ApiScheduler` +- `withProgress(...)`, `formatProgress(line)`, `formatRemaining(ms)`, `ProgressCounts`, `ProgressUpdate` + +**Progress bar:** it fills the line up to 80 columns, and the report under it adds an estimate of the time left. Without a terminal, the report prints at each 10%. It leaves stdin alone, so Ctrl-C reaches the CLI's handler. + +- [ ] **Step 1: Port the tests, then the modules.** Run `bun test --isolate` on the four test files. +- [ ] **Step 2: Commit**: `feat(migrate): pace Backend API calls, back off on 429, and show progress` + +--- + +### Task 4: Target resolution + +**Files:** + +- Port as-is: `lib/target.ts`, `lib/target.test.ts`. +- Modify: `lib/config.ts` (export `INSTANCE_ALIASES`). + +**Interfaces:** + +- `resolveClerkTarget(options): Promise<{ secretKey: string; target: ClerkTarget }>` +- `fetchInstanceIdentity(secretKey)`: reads `GET /v1/instance` and retries a 429. When the instance can't be read, it falls back to `key_`. +- `describeTarget(target)`, `printTarget(target)` + +`target.ts` imports `RunTarget` from `lib/run-store.ts`. Port Task 5 first, or move both into one commit. + +Key sources, in the order `resolveBapiSecretKey` checks them: + +1. `--secret-key` +2. `--app` +3. `CLERK_SECRET_KEY env var` +4. `accountless app ()` +5. `linked profile` + +- [ ] **Step 1: Port the test, then the module.** The test must cover: + - each key source; + - an exported `CLERK_SECRET_KEY` outranking the linked profile; + - an `--instance` that the key from `--secret-key` or `CLERK_SECRET_KEY` doesn't address, refused with exit 2. +- [ ] **Step 2: Commit**: `feat(migrate): resolve and print the import target` + +--- + +### Task 5: Run store (write side) + +**Files:** port `lib/run-store.ts` and `lib/run-store.test.ts` with trims. + +**Interfaces:** + +- Constants: `RUNS_DIR_ENV`, `RUNS_DIR_FLAG`, `RUNS_DIR_DESCRIPTION`, `RUN_ID_PATTERN` +- Types: `RunKind`, `RunStatus`, `UserStatus`, `RunTarget`, `RunFile`, `RunCounts`, `RunRecord`, `PendingIdentifier`, `UserLine`, `Run`, `StartRunInit` +- Functions: `resolveRunsDir(runsDir, { write?, cwd? })`, `runDir`, `newRunId`, `sha256File`, `startRun(runsDir, init): Run`, `readRun`, `readUserLines`, `latestUserLines`, `countLines`, `liveLockPid`, `lockFile` + +**Trims:** remove `continueRun`, `patchRun`, `runState`, `RunState`, `listRuns`, `clerkIdsCreatedByOtherRuns`, and their tests. Slice 2 adds them back. + +Keep the full `RunKind`, `RunStatus`, and `UserStatus` unions (`import | undo | export`, `undone`, plus `deleted` and `exported`), and keep `UserLine.pending`. The run format is final in this slice. + +- [ ] **Step 1: Port the test, then the module.** The tests must cover: + - the location order; + - `.clerk/` added to `.gitignore` once, and only when the run is written; + - the run ID shape; + - run folders created `0700` and exclusively, so a second run with the same ID fails; + - a live lock refusing a second writer with exit 2, naming the lock file; + - a lock holding this process's own PID read as stale; + - the last line per `sourceId` winning; + - a truncated last line being ignored; + - a failed append throwing; + - `finish()` setting `partial` when any user failed, was skipped, or is still `creating`, and `complete` otherwise. +- [ ] **Step 2: Commit**: `feat(migrate): keep every import as a run under .clerk/migrate` + +--- + +### Task 6: Instance reads and the readiness report + +**Files:** port as-is: `lib/clerk-config.ts`, `lib/user-lookup.ts`, `lib/supabase-providers.ts`, `lib/analysis.ts`, `lib/readiness.ts`, `lib/modify-settings.ts`, and their tests. + +**Interfaces:** + +- `fetchInstanceSettings(secretKey): Promise`: BAPI `/v1/domains`, then FAPI `/v1/environment`, bootstrapping a dev browser on development instances. Returns `null` when the settings can't be read. +- `fetchUserCount(secretKey): Promise` +- `toClerkStrategy(provider)`, `clerkOffersProvider(provider)`, `providerLabel(provider)`, `enabledSocialProviders(settings)`, `fetchEnabledSocialProviders(secretKey)`. Clerk's built-in OAuth strategies are copied from clerk_go's `api/shared/sso/oauth.go`, since nothing the CLI can call lists them. +- `lookupUsers({ filter, values, secretKey, schedule, spinner?, label? })`: batches of 100 through the scheduler and `retryOn429`. An `external_id` is looked up with a `+` prefix, so BAPI doesn't read a leading `-` as an exclusion. +- `readSupabaseRows(file)`, `findDisabledProviders`, `findUsersWithOnlyDisabledProviders`, `countSocialProviders` +- `analyzeFields(users)`, `buildReadinessReport(input)`, `buildSettingChanges(flagged)`, `buildChangePayload(changes)` + +**Behavior:** + +- Readiness counts users, not identifiers. +- A Supabase provider Clerk doesn't offer is named as not offered, and gets no fix. +- A fix names its instance with `--instance`, or points at the Dashboard when Clerk couldn't name the instance. This keeps a `--secret-key` import's fix from changing the linked profile's development instance. + +- [ ] **Step 1: Port the tests, then the modules.** +- [ ] **Step 2: Commit**: `feat(migrate): read the instance's settings, user count and existing users` + +--- + +### Task 7: Importer + +**Files:** port `import-users.ts` and `import-users.test.ts`, trimming only what continuing a run needs. Slice 2 adds adoption back. + +**Interfaces:** + +- `importUsers({ users, secretKey, limits, record, skipPasswordRequirement?, validationFailed?, spinner? }): Promise` +- `splitIdentifiers(user)`, `buildCreateUserBody(user, identifiers, skipPasswordRequirement)`, `normalizeErrorMessage(message)`, `outcomeUnknown(error)`, `pendingIdentifiers(identifiers)` + +**Behavior:** + +- Each user writes `creating` as its `POST /v1/users` goes out, not when it's queued. It writes `created` with its Clerk ID as soon as the create returns, with any extra identifiers listed as `pending`. +- A create with no answer (an abort, a network error, or a 5xx) leaves the user at `creating`. `outcomeUnknown` decides this. +- Only the first verified email and phone go on the create. An unverified primary stays unverified. Every other identifier attaches afterwards with its own request, ahead of queued creates. An attach retries on a 429. A failed attachment adds a note to the user's line, and the user still counts as created. +- Every create sends `skip_restriction_checks`, because the allowlist and blocklist police sign-ups and these users already signed up. With `--skip-legal-checks`, it also sends `skip_legal_checks`. +- When Clerk refuses the first phone (`unsupported_country_code`, or `param_name: phone_number`) and the user has an email, the create retries without the phone and logs a note. +- A 429 retries up to 5 times, honoring `Retry-After`. When the retries run out, the user is recorded as `failed` with code `429`. +- Any other error records `failed` with the API's long message and status, and the run continues. +- Progress shows through `withProgress`, not a spinner. + +- [ ] **Step 1: Port the test, then the module.** Include a test that an aborted create leaves the user at `creating`. +- [ ] **Step 2: Commit**: `feat(migrate): create users through the scheduler, one run line each` + +--- + +### Task 8: Checks + +**Files:** port `lib/checks.ts` and `lib/checks.test.ts` with trims. `checks.ts` imports `splitIdentifiers` from `import-users.ts`, which is why it comes after Task 7. + +**Interfaces:** + +- `checkImport(input: CheckInput): Promise` +- `hashShapeProblem(password, hasher)`, `passwordIsOnlySignIn(settings)` +- Types: `Reject`, `ReasonCount`, `Fix`, `Quota`, `ImportChecks`, `CheckInput` + +**Trims:** remove `adoptedClerkIds` from `CheckInput`, along with its use in `findInstanceDuplicates`. Slice 2 adds it back with continuing. Keep `skipLegalChecks`. + +**Rejects.** Each user gets the first reason that applies, in this order: + +1. it failed schema validation (`invalid: …`) +2. the source set a `skipReason` (Supabase: soft-deleted) +3. its only emails are ones Clerk refuses: malformed, or a private TLD such as `.local`, `.invalid`, `.test`, `.example`, or `.arpa`. Every such user gets one fixed reason, so they group and the address stays out of the run record. +4. it lacks a verified identifier the instance requires +5. it has no identifier left once the ones the instance has turned off are removed +6. it lacks a first or last name the instance requires +7. it has a TOTP secret or backup codes, and the instance has that feature off (with a fix to turn it on) +8. it has no password, and password is the instance's only way to sign in +9. it has no legal acceptance on record, and the instance requires one. `--skip-legal-checks`, or a yes at the prompt, imports these users instead. +10. its username breaks the instance's username rules +11. its password doesn't match its hasher's shape. `bcrypt` (cost up to 15), `scrypt_firebase`, `argon2i`/`argon2id`, and `scrypt_werkzeug` are checked. +12. Supabase: its only providers aren't enabled in Clerk, or aren't offered by Clerk. The reason names only that user's own providers. +13. it repeats an earlier passing user's source ID, email, phone, or username in the file. Usernames compare case-insensitively, and phones compare with punctuation stripped. Only users that pass checks 1 to 12 claim identifiers, so a rejected record doesn't cost a later one its email. The first record wins, and the reject names it (`kept: …`). +14. the instance already has a user with its source ID, email, phone, or username +15. on a development instance, it's past the headroom (`CLERK_MIGRATE_DEV_USER_LIMIT`, default 100, minus the live count), counted in file order. When the live count can't be read, the checks say so instead of treating it as zero. + +**Imported with warnings:** + +- fields the instance isn't set up to store +- `Clerk won't store: (N users)` for unknown fields +- passwords the source dropped +- refused emails and names, which are dropped. An email-shaped name is kept, as Clerk keeps it. +- a malformed secondary email, which is dropped. The shape check is looser than Zod's, so non-ASCII addresses Clerk accepts are kept. +- a password on an instance with passwords turned off, which is stored but works only once passwords are turned on + +**Identifiers the instance has turned off** (email, phone, or username) are removed from each importable user before the create. + +**Fixes** are `clerk config patch` offers built from the same readiness rows, with `--app` and `--instance` included when the target knows them. A provider Clerk doesn't offer gets no fix. + +- [ ] **Step 1: Port the test, then the module.** Run `bun test --isolate packages/cli-core/src/commands/migrate/lib/checks.test.ts`. +- [ ] **Step 2: Commit**: `feat(migrate): check every user against the instance before writing` + +--- + +### Task 9: `clerk migrate import` + +**Files:** + +- Port with trims: `run.ts`, `run.test.ts`, `run-interactive.test.ts`, `wizard.ts`, `wizard.test.ts`, `index.ts` (the enabled branch), `index.test.ts` +- Port as-is: `lib/assume-yes.ts` +- Modify: + - `lib/next-steps.ts`: add `printAgentNextSteps` from `REF/lib/next-steps.ts`, plus `MIGRATE_DONE` and `MIGRATE_DONE_WITH_ERRORS` (text below) + - `lib/spinner.ts`: ignore an empty `setNextSteps([])` + - `lib/json-body.ts`: export `quoteArg`, for the printed commands + - `commands/completion/__complete.ts`: complete `--source` from `sources` + - `test/integration/completion.test.ts` +- Create: the `commands/migrate/README.md` import sections, `readme.test.ts`, and the changeset + +**`index.ts`, enabled branch:** + +- `migrate` group: copy the description and examples from `REF/commands/migrate/index.ts`, keeping only the examples that use `import `. +- The group's `preAction` hook, as-is: it sets `-y` through `setAssumeYes`, and `--json` switches the run to agent mode. +- `import` subcommand: + - Argument: `[file]`, "A JSON or CSV export". Not `[file|export-run-id]`; slice 3 widens it. + - Options: `--source ` (no `.choices()`, so an unknown key reaches `resolveSource`'s error), `--dry-run`, `--allow-partial`, `--require-password`, `--skip-legal-checks`, `-y, --yes`, `--json`, `--secret-key `, `--app `, `--instance `, and `RUNS_DIR_FLAG`. + - Leave out `--new-run` (slice 2) and the `--firebase-*` flags (slice 4). + - Examples: keep the `users.json --source clerk --allow-partial --yes` one. Drop the export-run-ID and `./my-source.ts` examples. + +**`run.ts`. Keep:** + +- `ensureImportTarget`: a human who isn't signed in is signed in, then an unlinked directory gets linked. An agent gets the `AuthError` naming the missing half. +- `resolveInput`, without the export-run-ID branch +- `applySource`, `validateRunOptions`, `explainErrors`, `printChecks`, `checksJson`, `formatSummary`, `commandFor`, `recordRejects` +- The legal-checks prompt: a human with users lacking legal acceptance is asked whether to import them with `skip_legal_checks`. +- The main flow: print the target, load, read settings and the user count, check, then dry-run, refuse, nothing-to-import, consent, import under `withProgress`, and summary +- `--require-password` records the users it leaves out as `skipped`. + +**`run.ts`. Remove (later slices add them back):** + +- `findResume`, `ResumeCase`, the `complete` early return, the continued-run bookkeeping, adopting in-flight creates by `external_id`, finishing `pending` identifiers, `continueRun`, `--new-run`, and the refusal while an undo of the matching run is unfinished (slice 2) +- `readEnvelope`, `applyEnvelope`, and `fromExport` (slice 3) +- `cleanupLines` (slice 2, alongside `undo`) +- `FirebaseHashFlags`, `resolveFirebaseHashConfig`, `promptForFirebaseHashConfig`, and the `--firebase-*` placeholders in `commandFor` (slice 4) +- `MigrateRunOptions.sourceHash`, the custom-source log line in `applySource`, and keeping a custom `--source` path in `commandFor` (slice 5) + +**`wizard.ts`:** keep `promptForFile` and `promptForSource`. Remove `askNumber` and `promptForFirebaseHashConfig`. + +**JSON output:** `{ target, run, checks, result }`. When a run stops before importing, it adds one of these instead of `result`: `dryRun: true`, `refused: true`, `consent: "required"`, or `nothingToImport: true`. `result` is `{ created, failed, skipped, errors: [{ error, count }] }`. + +**Next steps**, until slice 2 adds `runs` and `undo`: + +```ts +MIGRATE_DONE: (runFolder: string) => [`See each user's outcome in ${runFolder}/users.ndjson`], +MIGRATE_DONE_WITH_ERRORS: (runFolder: string) => [ + `See every user that failed, and why, in ${runFolder}/users.ndjson`, +], +``` + +**Re-running in this slice:** a second import of the same file starts a new run. The users the first run created are rejected as already in the instance, so `--allow-partial --yes` imports the rest. Users an interrupted run left at `creating` may or may not exist in Clerk. The ones that do are rejected as duplicates, and the rest import. Slice 2 replaces this with continuing the run. + +- [ ] **Step 1: Port the tests with trims.** + - `run.test.ts`: keep `validateRunOptions`, "without a file or a source", `checks`, `consent`, `explainErrors`, `--skip-legal-checks`, the `--instance` mismatch, and the Clerk and Supabase cases of "per-platform imports". Drop "export envelopes", "continuing an earlier run", adoption, `--source `, and the other platforms. + - `index.test.ts`: keep the `import` registration and the `-y` hook cases, plus Task 1's gate cases. Set `CLERK_EXPERIMENTAL=migrate` in `beforeEach` for the enabled cases. + - `run-interactive.test.ts`, `wizard.test.ts`: drop the Firebase prompts. +- [ ] **Step 2: Add tests for the behavior this slice adds:** + 1. Importing the same file twice: the second run rejects every created user as already in the instance, and exits 2 without `--allow-partial`. + 2. `--source nope` exits 2 and lists `clerk, supabase`. + 3. In agent mode, stderr has no `\x1b[`. + 4. `--help` for `import` doesn't list `--new-run`, `--firebase-*`, or an export run ID. +- [ ] **Step 3: Run them and confirm they fail.** Run `bun test --isolate packages/cli-core/src/commands/migrate/run.test.ts packages/cli-core/src/commands/migrate/index.test.ts`. +- [ ] **Step 4: Port the modules with the trims above.** Keep `run.ts` to orchestration. Every decision it makes is already a tested function in Tasks 2 to 8. +- [ ] **Step 5: Write the README.** Port these sections from `REF/commands/migrate/README.md`, cut to slice 1: + - the rules + - targeting and auth + - the run store (without `runs`, `undo`, continuing, or exports) + - `clerk migrate import` (without re-running, export run IDs, Firebase flags, or `--new-run`) + - checks, additional identifiers, throughput, and the progress bar + - sources (Clerk and Supabase rows only) + - the schema fields + - the API endpoints `import` calls + + Put a blockquote at the top saying the command is experimental and needs `CLERK_EXPERIMENTAL=migrate`. Port `readme.test.ts`, and set `process.env.CLERK_EXPERIMENTAL = "migrate"` before it calls `createProgram()`, or it sees only the stub. It fails on any documented flag the binary rejects, and on any flag the README leaves out. + +- [ ] **Step 6: Create the changeset** by invoking the `changesets` skill. It's a minor bump, and the text says the command is experimental and gated. +- [ ] **Step 7: Run every CI check**: `bun run format && bun run lint && bun run typecheck && bun run test`, then `bun changeset status --since=origin/main`. +- [ ] **Step 8: Commit**: `feat(migrate): add experimental clerk migrate import for Clerk and Supabase files` + +--- + +### Task 10: Prove it with E2E + +**Files:** + +- Create: `test/e2e/migrate.test.ts`. Port the "a user whose only email is unverified is refused where email is required" case from `test/e2e/migrate.test.ts` at `7a820406`. It reads each user's latest run line per source ID, because a user's first line is `creating`. Leave the Better Auth case for slice 4d. + +- [ ] **Step 1: Add a Supabase round trip to the E2E test.** Follow `.claude/rules/e2e.md`: + - Write a 2-user Supabase JSON file with fresh bcrypt hashes (`Bun.password.hash("", { algorithm: "bcrypt", cost: 10 })`) and unique `+e2e-` emails. + - Run `import --source supabase --dry-run --json` and assert `checks.importable === 2`. + - Run `--yes --json` and assert `result.created === 2`. + - Verify each password with `POST /v1/users/{id}/verify_password`. + - Delete both users in `afterAll`. + - Set `CLERK_EXPERIMENTAL=migrate` and `CLERK_TELEMETRY_DISABLED=1` on the subprocess env. + + Run `bun run test:e2e:op -- -t "migrate"`. It should pass. Until the `--no-env-file` PR merges to `main`, move any `.env.local` that points at a local `clerk_go` stack aside first, or the run sends the test secrets there. + +- [ ] **Step 2: Commit the E2E test**: `test(migrate): e2e import with dry-run, consent and password verification` +- [ ] **Step 3: After the PR merges,** install `clerk@canary` with `CLERK_EXPERIMENTAL=migrate` and test it with the slice's testing process (to be defined). Then rebase slice 2 onto `main` (Branching, step 4) and mark it ready. + +--- + +## Out of scope for slice 1 + +Each of these lands in a later slice: + +- Continuing a stopped run, `--new-run`, adopting in-flight creates, finishing `pending` identifiers, `runs`, `undo`, and the cleanup lines after a complete import (slice 2) +- The export envelope, importing by export run ID, `export clerk`, and `export supabase` (slice 3) +- Firebase, Auth0, WorkOS, Better Auth (including Drizzle's snake_case and plural schemas), and Auth.js, including the `--firebase-*` flags and the Firebase hash prompt (slice 4) +- `--source ./file.ts` and `migrate sources` (slice 5) +- Removing the gate, the root README entry, and the docs and skills (slice 6) diff --git a/test/e2e/migrate.test.ts b/test/e2e/migrate.test.ts new file mode 100644 index 000000000..caeb110e3 --- /dev/null +++ b/test/e2e/migrate.test.ts @@ -0,0 +1,316 @@ +/** + * Live-BAPI tests for `clerk migrate import`, covering what only a real + * instance can answer: + * + * - A Supabase export round-trips: the dry run counts it without writing, an + * import without `--yes` writes nothing, the import creates every user and + * records the run, each bcrypt hash verifies against the password it was + * made from, and `undo` deletes them again. + * - A Better Auth scrypt hash, sent as `scrypt_werkzeug`, verifies against the + * password it was made from. A unit test can only check the string shape. + * - A user whose only email is unverified, imported into an instance that + * requires an email, is refused by Clerk. The import's checks reject that + * user up front on the strength of this test, so it checks both halves. + * - The same user imported with `--reserve-unverified` is created, with that + * email reserved and primary. The checks let them through on the strength + * of this test. + * + * Requires `CLERK_PLATFORM_API_KEY` and `CLERK_CLI_TEST_APP_ID`. Locally, run + * via `bun run test:e2e:op` so 1Password resolves both in-memory. + */ + +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { randomBytes, scryptSync } from "node:crypto"; +import { mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const CLI_PATH = join(import.meta.dir, "../../packages/cli-core/src/cli.ts"); + +let APP_ID: string; +let workDir: string; +let emailRequired = false; +/** Every Clerk user an import created, deleted in `afterAll`. */ +const createdIds: string[] = []; + +async function cli(args: string[]) { + return Bun.$`bun ${CLI_PATH} ${args} --app ${APP_ID}` + .env({ + ...process.env, + CLERK_CONFIG_DIR: join(workDir, "config"), + CLERK_MIGRATE_DIR: join(workDir, "runs"), + CLERK_TELEMETRY_DISABLED: "1", + }) + .cwd(workDir) + .quiet() + .nothrow(); +} + +beforeAll(async () => { + const appId = process.env.CLERK_CLI_TEST_APP_ID; + if (!appId || !process.env.CLERK_PLATFORM_API_KEY) { + throw new Error( + "CLERK_CLI_TEST_APP_ID and CLERK_PLATFORM_API_KEY are required. " + + "Run via `bun run test:e2e:op` for local 1Password injection.", + ); + } + APP_ID = appId; + workDir = mkdtempSync(join(tmpdir(), "clerk-cli-e2e-migrate-")); + + const config = await cli(["config", "pull", "--keys", "auth_email"]); + const authEmail = ( + JSON.parse(config.stdout.toString()) as { + auth_email?: { required_for_sign_up?: boolean }; + } + ).auth_email; + emailRequired = Boolean(authEmail?.required_for_sign_up); +}); + +// Delete every imported user, so the test app does not fill up. +afterAll(async () => { + await Promise.all(createdIds.map(async (id) => cli(["api", `/users/${id}`, "-X", "DELETE"]))); + rmSync(workDir, { recursive: true, force: true }); +}, 60_000); + +/** + * Each user's latest line in run `runId`. A user has several (`creating`, + * then `created`); the last one wins. + */ +function latestLines(runId: string): Record[] { + const latest = new Map>(); + for (const line of readFileSync(join(workDir, "runs", runId, "users.ndjson"), "utf-8") + .trim() + .split("\n")) { + const parsed = JSON.parse(line) as Record; + latest.set(parsed.sourceId, parsed); + } + for (const line of latest.values()) { + if (line.status === "created" && typeof line.clerkId === "string") + createdIds.push(line.clerkId); + } + return [...latest.values()]; +} + +function writeExport(users: Record[]): string { + const file = join(workDir, `export-${randomBytes(4).toString("hex")}.json`); + writeFileSync(file, JSON.stringify(users)); + return file; +} + +test("a Supabase export dry-runs, imports, and its passwords verify", async () => { + const users = await Promise.all( + [0, 1].map(async () => { + const hex = randomBytes(6).toString("hex"); + const password = `Migrate${hex}!1`; + return { + password, + record: { + id: `sb_${hex}`, + email: `e2e-${hex}+clerk_test@clerkcookie.com`, + email_confirmed_at: "2024-06-29 20:25:06+00", + encrypted_password: await Bun.password.hash(password, { algorithm: "bcrypt", cost: 10 }), + }, + }; + }), + ); + const file = writeExport(users.map((user) => user.record)); + + const dryRun = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--dry-run", + "--json", + ]); + expect(dryRun.exitCode).toBe(0); + expect(JSON.parse(dryRun.stdout.toString())).toMatchObject({ + dryRun: true, + checks: { importable: 2 }, + }); + expect(readdirSync(workDir)).not.toContain("runs"); + + // Rule 1, against a real instance: without --yes nobody has consented, so + // nothing is written, here or in Clerk. + const unconsented = await cli(["migrate", "import", file, "--source", "supabase", "--json"]); + expect(unconsented.exitCode).toBe(2); + expect(JSON.parse(unconsented.stdout.toString())).toMatchObject({ + consent: "required", + checks: { importable: 2 }, + }); + expect(readdirSync(workDir)).not.toContain("runs"); + for (const { record } of users) { + const found = await cli(["api", `/users?external_id=${record.id}`]); + expect(JSON.parse(found.stdout.toString())).toEqual([]); + } + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--yes", + "--json", + ]); + const result = JSON.parse(imported.stdout.toString()) as { + run: { id: string }; + result: { created: number }; + }; + // Read first: it registers the created users for cleanup, which a failed + // assertion would otherwise skip, leaving them in the shared test app. + const lines = latestLines(result.run.id); + expect(imported.exitCode).toBe(0); + expect(result.result.created).toBe(2); + expect( + JSON.parse(readFileSync(join(workDir, "runs", result.run.id, "run.json"), "utf-8")), + ).toMatchObject({ status: "complete", counts: { total: 2, created: 2 } }); + for (const { password, record } of users) { + const line = lines.find((candidate) => candidate.sourceId === record.id); + expect(line).toMatchObject({ status: "created" }); + const verify = await cli([ + "api", + `/users/${line?.clerkId as string}/verify_password`, + "-X", + "POST", + "-d", + JSON.stringify({ password }), + ]); + expect(JSON.parse(verify.stdout.toString())).toMatchObject({ verified: true }); + } + + const undo = await cli(["migrate", "undo", result.run.id, "--yes", "--json"]); + expect(undo.exitCode).toBe(0); + for (const { record } of users) { + const line = lines.find((candidate) => candidate.sourceId === record.id); + expect(typeof line?.clerkId).toBe("string"); + // Clerk's own answer, not just any failure: a wrong path or an auth error + // would exit non-zero too. `clerk api` prints the error body to stdout. + const gone = await cli(["api", `/users/${line?.clerkId as string}`]); + expect(gone.exitCode).not.toBe(0); + expect(JSON.parse(gone.stdout.toString())).toMatchObject({ + errors: [{ code: "resource_not_found" }], + }); + } +}, 60_000); + +/** A password hashed exactly the way Better Auth's default hasher does it. */ +function betterAuthHash(password: string): string { + const salt = randomBytes(16).toString("hex"); + const key = scryptSync(password.normalize("NFKC"), salt, 64, { + N: 16384, + r: 16, + p: 1, + maxmem: 128 * 16384 * 16 * 2, + }); + return `${salt}:${key.toString("hex")}`; +} + +test("a Better Auth scrypt hash imports and verifies against its password", async () => { + const hex = randomBytes(6).toString("hex"); + const password = `Migrate${hex}!1`; + const file = writeExport([ + { + user_id: `ba_${hex}`, + email: `e2e-${hex}+clerk_test@clerkcookie.com`, + email_verified: true, + password_hash: betterAuthHash(password), + }, + ]); + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "betterauth", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + const [line] = latestLines(run.id); + expect(line).toMatchObject({ status: "created" }); + + const verify = await cli([ + "api", + `/users/${line?.clerkId as string}/verify_password`, + "-X", + "POST", + "-d", + JSON.stringify({ password }), + ]); + expect(JSON.parse(verify.stdout.toString())).toMatchObject({ verified: true }); +}, 60_000); + +test("a user whose only email is unverified is refused where email is required", async () => { + if (!emailRequired) { + // The test app decides this, not the test; say so rather than pass silently. + console.warn("Skipped: the test app's development instance does not require an email."); + return; + } + const hex = randomBytes(6).toString("hex"); + + // The premise: Clerk itself refuses a user created with no email, which is + // what an unverified-only user is at `POST /v1/users`. + const direct = await cli([ + "api", + "/users", + "-d", + JSON.stringify({ external_id: `direct_${hex}`, skip_password_requirement: true }), + ]); + expect(direct.exitCode).not.toBe(0); + + // And the import's checks reject them before asking Clerk. + const file = writeExport([{ id: `sb_${hex}`, email: `e2e-${hex}+clerk_test@clerkcookie.com` }]); + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--allow-partial", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + expect(latestLines(run.id)).toEqual([ + expect.objectContaining({ + status: "skipped", + reason: "only has an unverified email, and this instance requires an email", + }), + ]); +}, 60_000); + +test("--reserve-unverified creates an unverified-only user with the email reserved", async () => { + const hex = randomBytes(6).toString("hex"); + const email = `e2e-${hex}+clerk_test@clerkcookie.com`; + const file = writeExport([{ id: `sb_${hex}`, email }]); + + const imported = await cli([ + "migrate", + "import", + file, + "--source", + "supabase", + "--reserve-unverified", + "--yes", + "--json", + ]); + const { run } = JSON.parse(imported.stdout.toString()) as { run: { id: string } }; + const [line] = latestLines(run.id); + expect(line).toMatchObject({ status: "created" }); + + const fetched = await cli(["api", `/users/${line?.clerkId as string}`]); + const user = JSON.parse(fetched.stdout.toString()) as { + primary_email_address_id: string; + email_addresses: { id: string; email_address: string; reserved: boolean }[]; + }; + expect(user.email_addresses).toEqual([ + expect.objectContaining({ + id: user.primary_email_address_id, + email_address: email, + reserved: true, + }), + ]); +}, 60_000);